diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/README.md b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/README.md
index 893b9c9..7e8452d 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/README.md
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/README.md
@@ -1,7 +1,7 @@
 # B300 DeepSeek-V4-Flash / GLM-5.3 SGLang text matrix
 
-This directory archives the test code and all files written by the active B300
-text-matrix run as of the snapshot time.
+This directory archives the test code and all files written by the B300
+text-matrix run through its manual cutoff and cleanup.
 
 ## Snapshot
 
@@ -20,9 +20,23 @@ Incremental synchronization:
 - At this point DeepSeek-V4-Flash had produced 53 Low-Latency and 18
   Balanced point JSON files.
 
-The DeepSeek-V4-Flash matrix was still running when this snapshot was taken.
-Consequently, this is a complete snapshot of files present at that time, not
-the final completed run archive. GLM-5.3 had completed its first pass.
+Final synchronization:
+
+- Stop time: `2026-09-10 07:46:07 UTC` (`2026-09-10 15:46:07 Asia/Shanghai`)
+- Final delta archive SHA-256: `f6ba8dbce5db850d43e7583a40eb8fc7cc95aab91c08fc5352fa69ec1d7d1117`
+- Final status: 185/236 points reached a terminal state (78.39%): 164 pass,
+  3 timeout-pass, 18 fail markers, and 51 pending.
+- Mode progress: DSV4 Low-Latency 54/59, DSV4 Balanced 31/59,
+  GLM-5.3 Low-Latency 54/59, and GLM-5.3 High-Throughput 46/59.
+- Both benchmark tmux sessions were stopped, the test container was removed,
+  and `nvidia-smi` reported no remaining compute process.
+
+The 17 GLM-5.3 fail markers on OSL=1 are validation false negatives: the
+benchmark command returned zero and produced complete JSON, while TPOT is
+mathematically zero because a one-token response has no inter-token interval.
+The DSV4 Low-Latency `512k_1_c1` failure is a real GPU OOM. Raw status markers
+are preserved exactly as emitted; use `final-summary/` together with the point
+JSON and logs when interpreting them.
 
 ## Layout
 
@@ -32,6 +46,8 @@ the final completed run archive. GLM-5.3 had completed its first pass.
 - `snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/`: raw run
   outputs, point logs, service logs, commands, status markers, and summaries.
 - `snapshot-20260910-024355/*.log`: top-level orchestration logs.
+- `snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/`:
+  final `summary.md`, `summary.csv`, `comparison.md`, and `progress.json`.
 
 The repository normally excludes raw benchmark logs. They are intentionally
 included here because this snapshot is an explicit full-run archive.
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-dev4-20260909-084427-followup.log b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-dev4-20260909-084427-followup.log
index 4c9ee66..f6ea699 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-dev4-20260909-084427-followup.log
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-dev4-20260909-084427-followup.log
@@ -44,3 +44,188 @@
 [2026-09-10T02:42:10+00:00] warning profile=dsv4-low-latency point=16k_1_c128 zero_ttft_count=4
 [2026-09-10T02:42:10+00:00] pass profile=dsv4-low-latency point=16k_1_c128
 [2026-09-10T02:42:11+00:00] run profile=dsv4-low-latency shape=16k_1 isl=16384 osl=1 c=256 prompts=1280
+[2026-09-10T02:47:47+00:00] warning profile=dsv4-low-latency point=16k_1_c256 zero_ttft_count=7
+[2026-09-10T02:47:47+00:00] pass profile=dsv4-low-latency point=16k_1_c256
+[2026-09-10T02:47:48+00:00] run profile=dsv4-low-latency shape=1k_128 isl=1024 osl=128 c=1 prompts=64
+[2026-09-10T02:48:36+00:00] pass profile=dsv4-low-latency point=1k_128_c1
+[2026-09-10T02:48:37+00:00] run profile=dsv4-low-latency shape=1k_128 isl=1024 osl=128 c=8 prompts=64
+[2026-09-10T02:49:08+00:00] pass profile=dsv4-low-latency point=1k_128_c8
+[2026-09-10T02:49:09+00:00] run profile=dsv4-low-latency shape=1k_128 isl=1024 osl=128 c=32 prompts=160
+[2026-09-10T02:49:45+00:00] pass profile=dsv4-low-latency point=1k_128_c32
+[2026-09-10T02:49:46+00:00] run profile=dsv4-low-latency shape=1k_128 isl=1024 osl=128 c=64 prompts=320
+[2026-09-10T02:50:28+00:00] pass profile=dsv4-low-latency point=1k_128_c64
+[2026-09-10T02:50:29+00:00] run profile=dsv4-low-latency shape=1k_128 isl=1024 osl=128 c=128 prompts=640
+[2026-09-10T02:51:13+00:00] pass profile=dsv4-low-latency point=1k_128_c128
+[2026-09-10T02:51:14+00:00] run profile=dsv4-low-latency shape=1k_128 isl=1024 osl=128 c=256 prompts=1280
+[2026-09-10T02:52:12+00:00] pass profile=dsv4-low-latency point=1k_128_c256
+[2026-09-10T02:52:13+00:00] run profile=dsv4-low-latency shape=1k_4k isl=1024 osl=4096 c=1 prompts=64
+[2026-09-10T02:59:40+00:00] pass profile=dsv4-low-latency point=1k_4k_c1
+[2026-09-10T02:59:41+00:00] run profile=dsv4-low-latency shape=1k_4k isl=1024 osl=4096 c=8 prompts=64
+[2026-09-10T03:01:28+00:00] pass profile=dsv4-low-latency point=1k_4k_c8
+[2026-09-10T03:01:29+00:00] run profile=dsv4-low-latency shape=1k_4k isl=1024 osl=4096 c=32 prompts=160
+[2026-09-10T03:03:21+00:00] pass profile=dsv4-low-latency point=1k_4k_c32
+[2026-09-10T03:03:22+00:00] run profile=dsv4-low-latency shape=1k_4k isl=1024 osl=4096 c=64 prompts=320
+[2026-09-10T03:05:53+00:00] pass profile=dsv4-low-latency point=1k_4k_c64
+[2026-09-10T03:05:54+00:00] run profile=dsv4-low-latency shape=1k_4k isl=1024 osl=4096 c=128 prompts=640
+[2026-09-10T03:09:39+00:00] pass profile=dsv4-low-latency point=1k_4k_c128
+[2026-09-10T03:09:40+00:00] run profile=dsv4-low-latency shape=1k_4k isl=1024 osl=4096 c=256 prompts=1280
+[2026-09-10T03:14:18+00:00] pass profile=dsv4-low-latency point=1k_4k_c256
+[2026-09-10T03:14:19+00:00] run profile=dsv4-low-latency shape=64k_1 isl=65536 osl=1 c=1 prompts=64
+[2026-09-10T03:15:49+00:00] warning profile=dsv4-low-latency point=64k_1_c1 zero_ttft_count=1
+[2026-09-10T03:15:49+00:00] pass profile=dsv4-low-latency point=64k_1_c1
+[2026-09-10T03:15:50+00:00] run profile=dsv4-low-latency shape=64k_1 isl=65536 osl=1 c=4 prompts=64
+[2026-09-10T03:17:15+00:00] warning profile=dsv4-low-latency point=64k_1_c4 zero_ttft_count=1
+[2026-09-10T03:17:15+00:00] pass profile=dsv4-low-latency point=64k_1_c4
+[2026-09-10T03:17:16+00:00] run profile=dsv4-low-latency shape=64k_1 isl=65536 osl=1 c=8 prompts=64
+[2026-09-10T03:18:42+00:00] warning profile=dsv4-low-latency point=64k_1_c8 zero_ttft_count=1
+[2026-09-10T03:18:42+00:00] pass profile=dsv4-low-latency point=64k_1_c8
+[2026-09-10T03:18:43+00:00] run profile=dsv4-low-latency shape=64k_1 isl=65536 osl=1 c=16 prompts=80
+[2026-09-10T03:20:27+00:00] warning profile=dsv4-low-latency point=64k_1_c16 zero_ttft_count=1
+[2026-09-10T03:20:27+00:00] pass profile=dsv4-low-latency point=64k_1_c16
+[2026-09-10T03:20:28+00:00] run profile=dsv4-low-latency shape=64k_1 isl=65536 osl=1 c=32 prompts=160
+[2026-09-10T03:23:43+00:00] warning profile=dsv4-low-latency point=64k_1_c32 zero_ttft_count=1
+[2026-09-10T03:23:43+00:00] pass profile=dsv4-low-latency point=64k_1_c32
+[2026-09-10T03:23:44+00:00] run profile=dsv4-low-latency shape=64k_1 isl=65536 osl=1 c=64 prompts=320
+[2026-09-10T03:30:02+00:00] warning profile=dsv4-low-latency point=64k_1_c64 zero_ttft_count=3
+[2026-09-10T03:30:02+00:00] pass profile=dsv4-low-latency point=64k_1_c64
+[2026-09-10T03:30:03+00:00] run profile=dsv4-low-latency shape=64k_1 isl=65536 osl=1 c=128 prompts=640
+[2026-09-10T03:42:28+00:00] warning profile=dsv4-low-latency point=64k_1_c128 zero_ttft_count=5
+[2026-09-10T03:42:28+00:00] pass profile=dsv4-low-latency point=64k_1_c128
+[2026-09-10T03:42:29+00:00] run profile=dsv4-low-latency shape=64k_512 isl=65536 osl=512 c=1 prompts=64
+[2026-09-10T03:44:52+00:00] pass profile=dsv4-low-latency point=64k_512_c1
+[2026-09-10T03:44:53+00:00] run profile=dsv4-low-latency shape=64k_512 isl=65536 osl=512 c=4 prompts=64
+[2026-09-10T03:46:37+00:00] pass profile=dsv4-low-latency point=64k_512_c4
+[2026-09-10T03:46:38+00:00] run profile=dsv4-low-latency shape=64k_512 isl=65536 osl=512 c=8 prompts=64
+[2026-09-10T03:48:14+00:00] pass profile=dsv4-low-latency point=64k_512_c8
+[2026-09-10T03:48:15+00:00] run profile=dsv4-low-latency shape=64k_512 isl=65536 osl=512 c=16 prompts=80
+[2026-09-10T03:50:06+00:00] pass profile=dsv4-low-latency point=64k_512_c16
+[2026-09-10T03:50:07+00:00] run profile=dsv4-low-latency shape=64k_512 isl=65536 osl=512 c=32 prompts=160
+[2026-09-10T03:53:33+00:00] pass profile=dsv4-low-latency point=64k_512_c32
+[2026-09-10T03:53:34+00:00] run profile=dsv4-low-latency shape=64k_512 isl=65536 osl=512 c=64 prompts=320
+[2026-09-10T04:00:06+00:00] pass profile=dsv4-low-latency point=64k_512_c64
+[2026-09-10T04:00:07+00:00] run profile=dsv4-low-latency shape=128k_1 isl=131072 osl=1 c=1 prompts=64
+[2026-09-10T04:03:11+00:00] pass profile=dsv4-low-latency point=128k_1_c1
+[2026-09-10T04:03:12+00:00] run profile=dsv4-low-latency shape=128k_1 isl=131072 osl=1 c=2 prompts=64
+[2026-09-10T04:06:11+00:00] pass profile=dsv4-low-latency point=128k_1_c2
+[2026-09-10T04:06:12+00:00] run profile=dsv4-low-latency shape=128k_1 isl=131072 osl=1 c=4 prompts=64
+[2026-09-10T04:09:12+00:00] pass profile=dsv4-low-latency point=128k_1_c4
+[2026-09-10T04:09:13+00:00] run profile=dsv4-low-latency shape=128k_1 isl=131072 osl=1 c=8 prompts=64
+[2026-09-10T04:12:12+00:00] pass profile=dsv4-low-latency point=128k_1_c8
+[2026-09-10T04:12:13+00:00] run profile=dsv4-low-latency shape=128k_1 isl=131072 osl=1 c=16 prompts=80
+[2026-09-10T04:15:55+00:00] pass profile=dsv4-low-latency point=128k_1_c16
+[2026-09-10T04:15:56+00:00] run profile=dsv4-low-latency shape=128k_1 isl=131072 osl=1 c=32 prompts=160
+[2026-09-10T04:23:05+00:00] pass profile=dsv4-low-latency point=128k_1_c32
+[2026-09-10T04:23:06+00:00] run profile=dsv4-low-latency shape=128k_1 isl=131072 osl=1 c=64 prompts=320
+[2026-09-10T04:37:16+00:00] warning profile=dsv4-low-latency point=128k_1_c64 zero_ttft_count=2
+[2026-09-10T04:37:16+00:00] pass profile=dsv4-low-latency point=128k_1_c64
+[2026-09-10T04:37:17+00:00] run profile=dsv4-low-latency shape=128k_512 isl=131072 osl=512 c=1 prompts=64
+[2026-09-10T04:42:26+00:00] pass profile=dsv4-low-latency point=128k_512_c1
+[2026-09-10T04:42:27+00:00] run profile=dsv4-low-latency shape=128k_512 isl=131072 osl=512 c=2 prompts=64
+[2026-09-10T04:46:39+00:00] pass profile=dsv4-low-latency point=128k_512_c2
+[2026-09-10T04:46:40+00:00] run profile=dsv4-low-latency shape=128k_512 isl=131072 osl=512 c=4 prompts=64
+[2026-09-10T04:50:19+00:00] pass profile=dsv4-low-latency point=128k_512_c4
+[2026-09-10T04:50:21+00:00] run profile=dsv4-low-latency shape=128k_512 isl=131072 osl=512 c=8 prompts=64
+[2026-09-10T04:53:45+00:00] pass profile=dsv4-low-latency point=128k_512_c8
+[2026-09-10T04:53:46+00:00] run profile=dsv4-low-latency shape=128k_512 isl=131072 osl=512 c=16 prompts=80
+[2026-09-10T04:57:47+00:00] pass profile=dsv4-low-latency point=128k_512_c16
+[2026-09-10T04:57:48+00:00] run profile=dsv4-low-latency shape=128k_512 isl=131072 osl=512 c=32 prompts=160
+[2026-09-10T05:05:25+00:00] pass profile=dsv4-low-latency point=128k_512_c32
+[2026-09-10T05:05:26+00:00] run profile=dsv4-low-latency shape=256k_1 isl=262144 osl=1 c=1 prompts=16
+[2026-09-10T05:07:29+00:00] pass profile=dsv4-low-latency point=256k_1_c1
+[2026-09-10T05:07:31+00:00] run profile=dsv4-low-latency shape=256k_1 isl=262144 osl=1 c=2 prompts=16
+[2026-09-10T05:09:29+00:00] pass profile=dsv4-low-latency point=256k_1_c2
+[2026-09-10T05:09:30+00:00] run profile=dsv4-low-latency shape=256k_1 isl=262144 osl=1 c=4 prompts=16
+[2026-09-10T05:11:28+00:00] pass profile=dsv4-low-latency point=256k_1_c4
+[2026-09-10T05:11:29+00:00] run profile=dsv4-low-latency shape=512k_1 isl=524288 osl=1 c=1 prompts=8
+[2026-09-10T05:12:50+00:00] failed profile=dsv4-low-latency point=512k_1_c1 rc=1
+[2026-09-10T05:12:50+00:00] capacity short-circuit armed profile=dsv4-low-latency shape=512k_1 after c=1
+[2026-09-10T05:12:50+00:00] server failure aborts profile=dsv4-low-latency shape=512k_1
+[2026-09-10T05:12:50+00:00] ERROR: unhealthy before profile=dsv4-low-latency point=1m_boundary_1_c1
+[2026-09-10T05:12:50+00:00] server failure aborts profile=dsv4-low-latency shape=1m_boundary_1
+[2026-09-10T05:13:22+00:00] starting profile=dsv4-balanced model=dsv4-flash-0731
+[2026-09-10T05:13:23+00:00] waiting for server profile=dsv4-balanced elapsed=0s
+[2026-09-10T05:14:24+00:00] waiting for server profile=dsv4-balanced elapsed=60s
+[2026-09-10T05:15:25+00:00] waiting for server profile=dsv4-balanced elapsed=120s
+[2026-09-10T05:16:25+00:00] waiting for server profile=dsv4-balanced elapsed=180s
+[2026-09-10T05:17:26+00:00] waiting for server profile=dsv4-balanced elapsed=240s
+[2026-09-10T05:18:26+00:00] waiting for server profile=dsv4-balanced elapsed=300s
+[2026-09-10T05:19:27+00:00] waiting for server profile=dsv4-balanced elapsed=360s
+[2026-09-10T05:20:27+00:00] waiting for server profile=dsv4-balanced elapsed=420s
+[2026-09-10T05:21:28+00:00] waiting for server profile=dsv4-balanced elapsed=480s
+[2026-09-10T05:22:29+00:00] waiting for server profile=dsv4-balanced elapsed=540s
+[2026-09-10T05:23:00+00:00] server healthy profile=dsv4-balanced elapsed=570s
+[2026-09-10T05:23:18+00:00] run profile=dsv4-balanced shape=16k_512 isl=16384 osl=512 c=1 prompts=64
+[2026-09-10T05:30:32+00:00] pass profile=dsv4-balanced point=16k_512_c1
+[2026-09-10T05:30:33+00:00] run profile=dsv4-balanced shape=16k_512 isl=16384 osl=512 c=8 prompts=64
+[2026-09-10T05:31:50+00:00] pass profile=dsv4-balanced point=16k_512_c8
+[2026-09-10T05:31:51+00:00] run profile=dsv4-balanced shape=16k_512 isl=16384 osl=512 c=32 prompts=160
+[2026-09-10T05:33:19+00:00] pass profile=dsv4-balanced point=16k_512_c32
+[2026-09-10T05:33:20+00:00] run profile=dsv4-balanced shape=16k_512 isl=16384 osl=512 c=64 prompts=320
+[2026-09-10T05:35:32+00:00] pass profile=dsv4-balanced point=16k_512_c64
+[2026-09-10T05:35:33+00:00] run profile=dsv4-balanced shape=16k_512 isl=16384 osl=512 c=128 prompts=640
+[2026-09-10T05:39:14+00:00] pass profile=dsv4-balanced point=16k_512_c128
+[2026-09-10T05:39:15+00:00] run profile=dsv4-balanced shape=16k_512 isl=16384 osl=512 c=256 prompts=1280
+[2026-09-10T05:45:44+00:00] pass profile=dsv4-balanced point=16k_512_c256
+[2026-09-10T05:45:45+00:00] run profile=dsv4-balanced shape=16k_1 isl=16384 osl=1 c=1 prompts=64
+[2026-09-10T05:47:02+00:00] pass profile=dsv4-balanced point=16k_1_c1
+[2026-09-10T05:47:03+00:00] run profile=dsv4-balanced shape=16k_1 isl=16384 osl=1 c=8 prompts=64
+[2026-09-10T05:47:34+00:00] pass profile=dsv4-balanced point=16k_1_c8
+[2026-09-10T05:47:35+00:00] run profile=dsv4-balanced shape=16k_1 isl=16384 osl=1 c=32 prompts=160
+[2026-09-10T05:48:31+00:00] warning profile=dsv4-balanced point=16k_1_c32 zero_ttft_count=2
+[2026-09-10T05:48:31+00:00] pass profile=dsv4-balanced point=16k_1_c32
+[2026-09-10T05:48:32+00:00] run profile=dsv4-balanced shape=16k_1 isl=16384 osl=1 c=64 prompts=320
+[2026-09-10T05:50:09+00:00] warning profile=dsv4-balanced point=16k_1_c64 zero_ttft_count=2
+[2026-09-10T05:50:09+00:00] pass profile=dsv4-balanced point=16k_1_c64
+[2026-09-10T05:50:10+00:00] run profile=dsv4-balanced shape=16k_1 isl=16384 osl=1 c=128 prompts=640
+[2026-09-10T05:53:09+00:00] warning profile=dsv4-balanced point=16k_1_c128 zero_ttft_count=4
+[2026-09-10T05:53:09+00:00] pass profile=dsv4-balanced point=16k_1_c128
+[2026-09-10T05:53:10+00:00] run profile=dsv4-balanced shape=16k_1 isl=16384 osl=1 c=256 prompts=1280
+[2026-09-10T05:58:59+00:00] warning profile=dsv4-balanced point=16k_1_c256 zero_ttft_count=7
+[2026-09-10T05:58:59+00:00] pass profile=dsv4-balanced point=16k_1_c256
+[2026-09-10T05:59:00+00:00] run profile=dsv4-balanced shape=1k_128 isl=1024 osl=128 c=1 prompts=64
+[2026-09-10T06:00:58+00:00] pass profile=dsv4-balanced point=1k_128_c1
+[2026-09-10T06:00:59+00:00] run profile=dsv4-balanced shape=1k_128 isl=1024 osl=128 c=8 prompts=64
+[2026-09-10T06:01:28+00:00] pass profile=dsv4-balanced point=1k_128_c8
+[2026-09-10T06:01:29+00:00] run profile=dsv4-balanced shape=1k_128 isl=1024 osl=128 c=32 prompts=160
+[2026-09-10T06:01:55+00:00] pass profile=dsv4-balanced point=1k_128_c32
+[2026-09-10T06:01:56+00:00] run profile=dsv4-balanced shape=1k_128 isl=1024 osl=128 c=64 prompts=320
+[2026-09-10T06:02:25+00:00] pass profile=dsv4-balanced point=1k_128_c64
+[2026-09-10T06:02:26+00:00] run profile=dsv4-balanced shape=1k_128 isl=1024 osl=128 c=128 prompts=640
+[2026-09-10T06:03:03+00:00] pass profile=dsv4-balanced point=1k_128_c128
+[2026-09-10T06:03:04+00:00] run profile=dsv4-balanced shape=1k_128 isl=1024 osl=128 c=256 prompts=1280
+[2026-09-10T06:03:55+00:00] pass profile=dsv4-balanced point=1k_128_c256
+[2026-09-10T06:03:56+00:00] run profile=dsv4-balanced shape=1k_4k isl=1024 osl=4096 c=1 prompts=64
+run_full_text_matrix.sh: line 247: 1797795 Killed                  timeout --signal=TERM --kill-after=120 "${MAX_POINT_SECONDS}" sudo docker exec "${CONTAINER}" python3 -m sglang.bench_serving --backend sglang --host 127.0.0.1 --port "${PORT}" --model "${MODEL_CONTAINER}" --tokenizer "${MODEL_CONTAINER}" --dataset-name random-ids --tokenize-prompt --random-input-len "${isl}" --random-output-len "${osl}" --random-range-ratio 1.0 --num-prompts "${prompts}" --max-concurrency "${concurrency}" --seed "${SEED}" --disable-tqdm --flush-cache --output-details --output-file "/results/points/${point}.json" > "${console}" 2>&1
+[2026-09-10T06:35:57+00:00] timeout pass profile=dsv4-balanced point=1k_4k_c1
+[2026-09-10T06:35:58+00:00] run profile=dsv4-balanced shape=1k_4k isl=1024 osl=4096 c=8 prompts=64
+[2026-09-10T06:42:24+00:00] pass profile=dsv4-balanced point=1k_4k_c8
+[2026-09-10T06:42:25+00:00] run profile=dsv4-balanced shape=1k_4k isl=1024 osl=4096 c=32 prompts=160
+[2026-09-10T06:47:31+00:00] pass profile=dsv4-balanced point=1k_4k_c32
+[2026-09-10T06:47:32+00:00] run profile=dsv4-balanced shape=1k_4k isl=1024 osl=4096 c=64 prompts=320
+[2026-09-10T06:53:11+00:00] pass profile=dsv4-balanced point=1k_4k_c64
+[2026-09-10T06:53:12+00:00] run profile=dsv4-balanced shape=1k_4k isl=1024 osl=4096 c=128 prompts=640
+[2026-09-10T06:59:43+00:00] pass profile=dsv4-balanced point=1k_4k_c128
+[2026-09-10T06:59:44+00:00] run profile=dsv4-balanced shape=1k_4k isl=1024 osl=4096 c=256 prompts=1280
+[2026-09-10T07:09:13+00:00] pass profile=dsv4-balanced point=1k_4k_c256
+[2026-09-10T07:09:14+00:00] run profile=dsv4-balanced shape=64k_1 isl=65536 osl=1 c=1 prompts=64
+[2026-09-10T07:13:38+00:00] warning profile=dsv4-balanced point=64k_1_c1 zero_ttft_count=1
+[2026-09-10T07:13:38+00:00] pass profile=dsv4-balanced point=64k_1_c1
+[2026-09-10T07:13:39+00:00] run profile=dsv4-balanced shape=64k_1 isl=65536 osl=1 c=4 prompts=64
+[2026-09-10T07:15:06+00:00] warning profile=dsv4-balanced point=64k_1_c4 zero_ttft_count=1
+[2026-09-10T07:15:06+00:00] pass profile=dsv4-balanced point=64k_1_c4
+[2026-09-10T07:15:07+00:00] run profile=dsv4-balanced shape=64k_1 isl=65536 osl=1 c=8 prompts=64
+[2026-09-10T07:16:30+00:00] warning profile=dsv4-balanced point=64k_1_c8 zero_ttft_count=1
+[2026-09-10T07:16:30+00:00] pass profile=dsv4-balanced point=64k_1_c8
+[2026-09-10T07:16:31+00:00] run profile=dsv4-balanced shape=64k_1 isl=65536 osl=1 c=16 prompts=80
+[2026-09-10T07:18:09+00:00] warning profile=dsv4-balanced point=64k_1_c16 zero_ttft_count=1
+[2026-09-10T07:18:09+00:00] pass profile=dsv4-balanced point=64k_1_c16
+[2026-09-10T07:18:10+00:00] run profile=dsv4-balanced shape=64k_1 isl=65536 osl=1 c=32 prompts=160
+[2026-09-10T07:21:11+00:00] warning profile=dsv4-balanced point=64k_1_c32 zero_ttft_count=1
+[2026-09-10T07:21:11+00:00] pass profile=dsv4-balanced point=64k_1_c32
+[2026-09-10T07:21:12+00:00] run profile=dsv4-balanced shape=64k_1 isl=65536 osl=1 c=64 prompts=320
+[2026-09-10T07:26:57+00:00] warning profile=dsv4-balanced point=64k_1_c64 zero_ttft_count=3
+[2026-09-10T07:26:57+00:00] pass profile=dsv4-balanced point=64k_1_c64
+[2026-09-10T07:26:58+00:00] run profile=dsv4-balanced shape=64k_1 isl=65536 osl=1 c=128 prompts=640
+[2026-09-10T07:38:13+00:00] warning profile=dsv4-balanced point=64k_1_c128 zero_ttft_count=5
+[2026-09-10T07:38:13+00:00] pass profile=dsv4-balanced point=64k_1_c128
+[2026-09-10T07:38:14+00:00] run profile=dsv4-balanced shape=64k_512 isl=65536 osl=512 c=1 prompts=64
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-cleanup.complete b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-cleanup.complete
new file mode 100644
index 0000000..3cfed3f
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-cleanup.complete
@@ -0,0 +1 @@
+2026-09-10T07:45:20+00:00
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-summary/comparison.md b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-summary/comparison.md
new file mode 100644
index 0000000..87ef16a
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-summary/comparison.md
@@ -0,0 +1,121 @@
+# B300 DeepSeek-V4-Flash / GLM-5.3 Comparison: b300-dsv4-glm53-dev4-20260909-084427
+
+## Run Status
+
+- Finished: 78.39%
+- Pass: 164
+- Timeout pass: 3
+- Invalid: 0
+- Fail: 18
+
+## Service Configuration And Capacity
+
+| Profile | TP | DP | EP | Max running | Max tokens/rank | Chunk | KV dtype | Attention | MoE runner | MoE A2A | Speculative |
+|---|---:|---:|---:|---:|---:|---:|---|---|---|---|---|
+| dsv4-low-latency | - | - | - | - | - | - | - | - | - | - | - |
+| dsv4-balanced | - | - | - | - | - | - | - | - | - | - | - |
+| glm53-low-latency | 8 | 1 | 1 | 48 | 2696000 | 16384 | fp8_e4m3 | dsa | flashinfer_trtllm | none | EAGLE |
+| glm53-high-throughput | 8 | 8 | 8 | 256 | 2542208 | 4096 | fp8_e4m3 | dsa | auto | deepep | EAGLE |
+
+## Throughput Peaks And Their Latency
+
+Only validated PASS points are considered.
+
+| Shape | Mode | Peak input TPS @ C | TTFT P95 (ms) | Peak output TPS @ C | TPOT P95 (ms) |
+|---|---|---:|---:|---:|---:|
+| 16k_512 | dsv4-low-latency | 60485.71 @ C256 | 48385.38 | 1890.18 @ C256 | 226.23 |
+| 16k_512 | dsv4-balanced | 57436.40 @ C256 | 62849.59 | 1794.89 @ C256 | 134.28 |
+| 16k_512 | glm53-low-latency | 24021.20 @ C256 | 144384.42 | 750.66 @ C256 | 102.36 |
+| 16k_512 | glm53-high-throughput | 31889.21 @ C128 | 30890.29 | 996.54 @ C128 | 165.33 |
+| 16k_1 | dsv4-low-latency | 66726.27 @ C32 | 7810.68 | 4.07 @ C32 | 0.00 |
+| 16k_1 | dsv4-balanced | 65093.70 @ C32 | 7943.21 | 3.97 @ C32 | 0.00 |
+| 1k_128 | dsv4-low-latency | 29913.48 @ C256 | 3544.47 | 3739.19 @ C256 | 99.14 |
+| 1k_128 | dsv4-balanced | 36265.26 @ C256 | 4505.51 | 4533.16 @ C256 | 59.65 |
+| 1k_128 | glm53-low-latency | 14308.93 @ C64 | 1848.75 | 1788.62 @ C64 | 40.49 |
+| 1k_128 | glm53-high-throughput | 15193.07 @ C128 | 2569.96 | 1899.13 @ C128 | 87.74 |
+| 1k_4k | dsv4-low-latency | 5283.08 @ C256 | 3658.46 | 21132.32 @ C256 | 17.31 |
+| 1k_4k | dsv4-balanced | 2441.07 @ C256 | 4386.00 | 9764.26 @ C256 | 26.51 |
+| 1k_4k | glm53-low-latency | 1530.74 @ C128 | 56046.13 | 6122.95 @ C128 | 11.08 |
+| 1k_4k | glm53-high-throughput | 2653.31 @ C256 | 4540.61 | 10613.25 @ C256 | 22.36 |
+| 64k_1 | dsv4-low-latency | 58913.97 @ C4 | 4440.74 | 0.90 @ C4 | 0.00 |
+| 64k_1 | dsv4-balanced | 65864.61 @ C16 | 15823.61 | 1.01 @ C16 | 0.00 |
+| 64k_1 | glm53-low-latency | 25336.86 @ C8 | 20633.79 | 0.39 @ C8 | 0.00 |
+| 64k_1 | glm53-high-throughput | 53720.01 @ C8 | 9891.34 | 0.82 @ C8 | 0.00 |
+| 64k_512 | dsv4-low-latency | 56712.78 @ C64 | 55290.66 | 443.07 @ C64 | 238.49 |
+| 64k_512 | glm53-low-latency | 24220.17 @ C64 | 128260.27 | 189.22 @ C64 | 323.81 |
+| 64k_512 | glm53-high-throughput | 33437.72 @ C64 | 57344.18 | 261.23 @ C64 | 329.77 |
+| 128k_1 | dsv4-low-latency | 51351.81 @ C2 | 5095.93 | 0.39 @ C2 | 0.00 |
+| 128k_1 | glm53-low-latency | 22544.03 @ C2 | 11620.01 | 0.17 @ C2 | 0.00 |
+| 128k_512 | dsv4-low-latency | 48253.24 @ C32 | 62228.62 | 188.49 @ C32 | 214.99 |
+| 128k_512 | glm53-low-latency | 21787.55 @ C32 | 142627.99 | 85.11 @ C32 | 339.34 |
+| 128k_512 | glm53-high-throughput | 27503.67 @ C32 | 59905.46 | 107.44 @ C32 | 401.75 |
+| 256k_1 | dsv4-low-latency | 40426.30 @ C2 | 13069.41 | 0.15 @ C2 | 0.00 |
+| 256k_1 | glm53-low-latency | 17620.27 @ C2 | 29920.79 | 0.07 @ C2 | 0.00 |
+| 512k_1 | glm53-low-latency | 12497.10 @ C2 | 84423.37 | 0.02 @ C2 | 0.00 |
+| 512k_1 | glm53-high-throughput | 24677.77 @ C4 | 86777.89 | 0.05 @ C4 | 0.00 |
+| 1m_boundary_1 | glm53-low-latency | 7914.10 @ C4 | 509514.56 | 0.01 @ C4 | 0.00 |
+| 1m_boundary_1 | glm53-high-throughput | 18580.87 @ C4 | 225402.20 | 0.02 @ C4 | 0.00 |
+
+## Capacity And Failure Boundaries
+
+| Mode | Shape | C | Status | Evidence |
+|---|---|---:|---|---|
+| dsv4-low-latency | 512k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-low-latency/points/512k_1_c1.failed` |
+| dsv4-balanced | 1k_4k | 1 | TIMEOUT_PASS | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/1k_4k_c1.json` |
+| glm53-low-latency | 16k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/16k_1_c1.json` |
+| glm53-low-latency | 64k_1 | 4 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c4.json` |
+| glm53-low-latency | 64k_1 | 16 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c16.json` |
+| glm53-low-latency | 64k_1 | 32 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c32.json` |
+| glm53-low-latency | 64k_1 | 64 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c64.json` |
+| glm53-low-latency | 64k_1 | 128 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c128.json` |
+| glm53-low-latency | 128k_1 | 8 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/128k_1_c8.json` |
+| glm53-low-latency | 128k_1 | 16 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/128k_1_c16.json` |
+| glm53-low-latency | 128k_1 | 32 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/128k_1_c32.json` |
+| glm53-low-latency | 128k_1 | 64 | TIMEOUT_PASS | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/128k_1_c64.json` |
+| glm53-low-latency | 256k_1 | 4 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/256k_1_c4.json` |
+| glm53-high-throughput | 16k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/16k_1_c1.json` |
+| glm53-high-throughput | 1k_4k | 1 | TIMEOUT_PASS | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/1k_4k_c1.json` |
+| glm53-high-throughput | 64k_1 | 16 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/64k_1_c16.json` |
+| glm53-high-throughput | 64k_1 | 32 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/64k_1_c32.json` |
+| glm53-high-throughput | 64k_1 | 64 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/64k_1_c64.json` |
+| glm53-high-throughput | 64k_1 | 128 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/64k_1_c128.json` |
+| glm53-high-throughput | 128k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/128k_1_c1.json` |
+| glm53-high-throughput | 256k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/256k_1_c1.json` |
+
+## Service Error Summary
+
+### dsv4-low-latency
+
+- Matched lines: 319
+- Source: `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-low-latency/server.final.log`
+- `[rank0]:[W910 05:05:37.307446434 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2415919104 bytes (free: 753532928, total: 287428771840).`
+- `[rank1]:[W910 05:05:37.312576349 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 1 while trying to allocate 2415919104 bytes (free: 724172800, total: 287428771840).`
+- `[rank3]:[W910 05:05:37.312868926 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 3 while trying to allocate 2415919104 bytes (free: 1066008576, total: 287428771840).`
+- `[rank2]:[W910 05:05:37.314989750 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 2 while trying to allocate 2415919104 bytes (free: 724172800, total: 287428771840).`
+- `[rank1]:[W910 05:05:38.764535635 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 1 while trying to allocate 3221225472 bytes (free: 870973440, total: 287428771840).`
+
+### dsv4-balanced
+
+- Matched lines: 0
+- Source: `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.final.log`
+
+### glm53-low-latency
+
+- Matched lines: 141
+- Source: `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/server.final.log`
+- `[rank2]:[W909 15:01:40.129936229 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 2 while trying to allocate 8858370048 bytes (free: 2053701632, total: 287428771840).`
+- `[rank3]:[W909 15:01:40.130091786 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 3 while trying to allocate 8858370048 bytes (free: 2116616192, total: 287428771840).`
+- `[rank1]:[W909 15:01:40.130433377 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 1 while trying to allocate 8858370048 bytes (free: 2053701632, total: 287428771840).`
+- `[rank0]:[W909 15:01:40.130656418 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 8885633024 bytes (free: 2229862400, total: 287428771840).`
+- `[rank7]:[W909 15:01:40.131055935 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 7 while trying to allocate 8994684928 bytes (free: 2714304512, total: 287428771840).`
+
+### glm53-high-throughput
+
+- Matched lines: 586
+- Source: `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/server.final.log`
+- `/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: /dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDAinit failed for transport: IBGDA`
+- `/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA`
+- `[rank1]:[W909 19:58:27.770722283 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 1 while trying to allocate 1950351360 bytes (free: 1160314880, total: 287428771840).`
+- `[rank2]:[W909 19:58:42.743872031 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 2 while trying to allocate 1950351360 bytes (free: 724107264, total: 287428771840).`
+- `[rank3]:[W909 19:58:55.386477067 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 3 while trying to allocate 1883242496 bytes (free: 1875443712, total: 287428771840).`
+
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-summary/progress.json b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-summary/progress.json
new file mode 100644
index 0000000..7e243b1
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-summary/progress.json
@@ -0,0 +1,47 @@
+{
+  "expected": 236,
+  "fail": 18,
+  "finished_percent": 78.39,
+  "invalid": 0,
+  "modes": {
+    "dsv4-balanced": {
+      "expected": 59,
+      "fail": 0,
+      "finished_percent": 52.54,
+      "invalid": 0,
+      "pass": 30,
+      "pending": 28,
+      "timeout_pass": 1
+    },
+    "dsv4-low-latency": {
+      "expected": 59,
+      "fail": 1,
+      "finished_percent": 91.53,
+      "invalid": 0,
+      "pass": 53,
+      "pending": 5,
+      "timeout_pass": 0
+    },
+    "glm53-high-throughput": {
+      "expected": 59,
+      "fail": 7,
+      "finished_percent": 77.97,
+      "invalid": 0,
+      "pass": 38,
+      "pending": 13,
+      "timeout_pass": 1
+    },
+    "glm53-low-latency": {
+      "expected": 59,
+      "fail": 10,
+      "finished_percent": 91.53,
+      "invalid": 0,
+      "pass": 43,
+      "pending": 5,
+      "timeout_pass": 1
+    }
+  },
+  "pass": 164,
+  "pending": 51,
+  "timeout_pass": 3
+}
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-summary/summary.md b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-summary/summary.md
new file mode 100644
index 0000000..0bff236
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/deadline-summary/summary.md
@@ -0,0 +1,200 @@
+# B300 DeepSeek-V4-Flash / GLM-5.3 Text Matrix: b300-dsv4-glm53-dev4-20260909-084427
+
+## Progress
+
+| Mode | Finished | Expected | Pass | Timeout pass | Invalid | Fail | Progress |
+|---|---:|---:|---:|---:|---:|---:|---:|
+| dsv4-low-latency | 54 | 59 | 53 | 0 | 0 | 1 | 91.53% |
+| dsv4-balanced | 31 | 59 | 30 | 1 | 0 | 0 | 52.54% |
+| glm53-low-latency | 54 | 59 | 43 | 1 | 0 | 10 | 91.53% |
+| glm53-high-throughput | 46 | 59 | 38 | 1 | 0 | 7 | 77.97% |
+
+## Results
+
+| Mode | Shape | C | Status | Input TPS | Output TPS | TTFT P95 (ms) | TPOT P95 (ms) |
+|---|---|---:|---|---:|---:|---:|---:|
+| dsv4-low-latency | 16k_512 | 1 | PASS | 13128.54 | 410.27 | 265.87 | 3.07 |
+| dsv4-low-latency | 16k_512 | 8 | PASS | 33936.55 | 1060.52 | 1517.12 | 10.23 |
+| dsv4-low-latency | 16k_512 | 32 | PASS | 49777.89 | 1555.56 | 6230.65 | 31.87 |
+| dsv4-low-latency | 16k_512 | 64 | PASS | 55645.12 | 1738.91 | 12091.63 | 57.79 |
+| dsv4-low-latency | 16k_512 | 128 | PASS | 59108.19 | 1847.13 | 24712.64 | 114.28 |
+| dsv4-low-latency | 16k_512 | 256 | PASS | 60485.71 | 1890.18 | 48385.38 | 226.23 |
+| dsv4-low-latency | 16k_1 | 1 | PASS | 61163.57 | 3.73 | 266.91 | 0.00 |
+| dsv4-low-latency | 16k_1 | 8 | PASS | 66664.03 | 4.07 | 1950.41 | 0.00 |
+| dsv4-low-latency | 16k_1 | 32 | PASS | 66726.27 | 4.07 | 7810.68 | 0.00 |
+| dsv4-low-latency | 16k_1 | 64 | PASS | 66288.99 | 4.05 | 16147.44 | 0.00 |
+| dsv4-low-latency | 16k_1 | 128 | PASS | 66596.05 | 4.06 | 31378.16 | 0.00 |
+| dsv4-low-latency | 16k_1 | 256 | PASS | 66600.10 | 4.06 | 62914.28 | 0.00 |
+| dsv4-low-latency | 1k_128 | 1 | PASS | 2041.91 | 255.24 | 183.12 | 3.53 |
+| dsv4-low-latency | 1k_128 | 8 | PASS | 3578.88 | 447.36 | 3387.56 | 31.29 |
+| dsv4-low-latency | 1k_128 | 32 | PASS | 7184.95 | 898.12 | 1708.71 | 49.89 |
+| dsv4-low-latency | 1k_128 | 64 | PASS | 11724.62 | 1465.58 | 1205.29 | 62.99 |
+| dsv4-low-latency | 1k_128 | 128 | PASS | 20916.51 | 2614.56 | 2101.48 | 68.93 |
+| dsv4-low-latency | 1k_128 | 256 | PASS | 29913.48 | 3739.19 | 3544.47 | 99.14 |
+| dsv4-low-latency | 1k_4k | 1 | PASS | 151.34 | 605.35 | 183.93 | 2.28 |
+| dsv4-low-latency | 1k_4k | 8 | PASS | 709.27 | 2837.08 | 451.52 | 3.52 |
+| dsv4-low-latency | 1k_4k | 32 | PASS | 1679.25 | 6717.00 | 754.10 | 6.34 |
+| dsv4-low-latency | 1k_4k | 64 | PASS | 2459.31 | 9837.23 | 1200.11 | 9.86 |
+| dsv4-low-latency | 1k_4k | 128 | PASS | 3217.13 | 12868.50 | 2188.22 | 14.14 |
+| dsv4-low-latency | 1k_4k | 256 | PASS | 5283.08 | 21132.32 | 3658.46 | 17.31 |
+| dsv4-low-latency | 64k_1 | 1 | PASS | 57079.88 | 0.87 | 1180.60 | 0.00 |
+| dsv4-low-latency | 64k_1 | 4 | PASS | 58913.97 | 0.90 | 4440.74 | 0.00 |
+| dsv4-low-latency | 64k_1 | 8 | PASS | 58818.80 | 0.90 | 8883.26 | 0.00 |
+| dsv4-low-latency | 64k_1 | 16 | PASS | 58871.47 | 0.90 | 17757.38 | 0.00 |
+| dsv4-low-latency | 64k_1 | 32 | PASS | 58821.54 | 0.90 | 35523.52 | 0.00 |
+| dsv4-low-latency | 64k_1 | 64 | PASS | 58852.44 | 0.90 | 71226.16 | 0.00 |
+| dsv4-low-latency | 64k_1 | 128 | PASS | 58701.82 | 0.90 | 142742.12 | 0.00 |
+| dsv4-low-latency | 64k_512 | 1 | PASS | 32785.22 | 256.13 | 1146.10 | 2.91 |
+| dsv4-low-latency | 64k_512 | 4 | PASS | 47367.52 | 370.06 | 3651.92 | 13.00 |
+| dsv4-low-latency | 64k_512 | 8 | PASS | 51470.88 | 402.12 | 8142.53 | 27.01 |
+| dsv4-low-latency | 64k_512 | 16 | PASS | 54390.07 | 424.92 | 13764.60 | 53.61 |
+| dsv4-low-latency | 64k_512 | 32 | PASS | 55570.81 | 434.15 | 27317.01 | 113.22 |
+| dsv4-low-latency | 64k_512 | 64 | PASS | 56712.78 | 443.07 | 55290.66 | 238.49 |
+| dsv4-low-latency | 128k_1 | 1 | PASS | 50299.31 | 0.38 | 2606.88 | 0.00 |
+| dsv4-low-latency | 128k_1 | 2 | PASS | 51351.81 | 0.39 | 5095.93 | 0.00 |
+| dsv4-low-latency | 128k_1 | 4 | PASS | 51297.32 | 0.39 | 10307.95 | 0.00 |
+| dsv4-low-latency | 128k_1 | 8 | PASS | 51293.08 | 0.39 | 20440.51 | 0.00 |
+| dsv4-low-latency | 128k_1 | 16 | PASS | 51305.80 | 0.39 | 40813.75 | 0.00 |
+| dsv4-low-latency | 128k_1 | 32 | PASS | 51293.67 | 0.39 | 81674.88 | 0.00 |
+| dsv4-low-latency | 128k_1 | 64 | PASS | 51257.15 | 0.39 | 163688.22 | 0.00 |
+| dsv4-low-latency | 128k_512 | 1 | PASS | 28968.88 | 113.16 | 2613.09 | 4.47 |
+| dsv4-low-latency | 128k_512 | 2 | PASS | 35668.55 | 139.33 | 2886.10 | 14.56 |
+| dsv4-low-latency | 128k_512 | 4 | PASS | 41221.32 | 161.02 | 5149.64 | 25.35 |
+| dsv4-low-latency | 128k_512 | 8 | PASS | 44544.04 | 174.00 | 13131.55 | 55.16 |
+| dsv4-low-latency | 128k_512 | 16 | PASS | 46962.57 | 183.45 | 31281.83 | 107.27 |
+| dsv4-low-latency | 128k_512 | 32 | PASS | 48253.24 | 188.49 | 62228.62 | 214.99 |
+| dsv4-low-latency | 256k_1 | 1 | PASS | 39875.71 | 0.15 | 6610.03 | 0.00 |
+| dsv4-low-latency | 256k_1 | 2 | PASS | 40426.30 | 0.15 | 13069.41 | 0.00 |
+| dsv4-low-latency | 256k_1 | 4 | PASS | 40371.52 | 0.15 | 26029.44 | 0.00 |
+| dsv4-low-latency | 512k_1 | 1 | FAIL | - | - | - | - |
+| dsv4-balanced | 16k_512 | 1 | PASS | 2517.53 | 78.67 | 1067.18 | 11.26 |
+| dsv4-balanced | 16k_512 | 8 | PASS | 16954.74 | 529.84 | 2578.91 | 12.69 |
+| dsv4-balanced | 16k_512 | 32 | PASS | 36100.44 | 1128.14 | 8087.71 | 25.54 |
+| dsv4-balanced | 16k_512 | 64 | PASS | 45336.60 | 1416.77 | 16042.56 | 41.92 |
+| dsv4-balanced | 16k_512 | 128 | PASS | 52115.96 | 1628.62 | 31955.08 | 73.50 |
+| dsv4-balanced | 16k_512 | 256 | PASS | 57436.40 | 1794.89 | 62849.59 | 134.28 |
+| dsv4-balanced | 16k_1 | 1 | PASS | 16704.37 | 1.02 | 1001.65 | 0.00 |
+| dsv4-balanced | 16k_1 | 8 | PASS | 63912.26 | 3.90 | 2334.27 | 0.00 |
+| dsv4-balanced | 16k_1 | 32 | PASS | 65093.70 | 3.97 | 7943.21 | 0.00 |
+| dsv4-balanced | 16k_1 | 64 | PASS | 64887.20 | 3.96 | 16398.83 | 0.00 |
+| dsv4-balanced | 16k_1 | 128 | PASS | 64875.87 | 3.96 | 32514.97 | 0.00 |
+| dsv4-balanced | 16k_1 | 256 | PASS | 64245.92 | 3.92 | 65033.17 | 0.00 |
+| dsv4-balanced | 1k_128 | 1 | PASS | 643.09 | 80.39 | 342.42 | 10.62 |
+| dsv4-balanced | 1k_128 | 8 | PASS | 4391.78 | 548.97 | 649.01 | 10.73 |
+| dsv4-balanced | 1k_128 | 32 | PASS | 13757.02 | 1719.63 | 943.32 | 12.48 |
+| dsv4-balanced | 1k_128 | 64 | PASS | 21219.84 | 2652.48 | 1451.23 | 17.88 |
+| dsv4-balanced | 1k_128 | 128 | PASS | 29472.67 | 3684.08 | 2841.20 | 27.68 |
+| dsv4-balanced | 1k_128 | 256 | PASS | 36265.26 | 4533.16 | 4505.51 | 59.65 |
+| dsv4-balanced | 1k_4k | 1 | TIMEOUT_PASS | 21.98 | 87.94 | 370.83 | 14.15 |
+| dsv4-balanced | 1k_4k | 8 | PASS | 187.97 | 751.87 | 799.79 | 10.62 |
+| dsv4-balanced | 1k_4k | 32 | PASS | 631.65 | 2526.62 | 916.72 | 12.59 |
+| dsv4-balanced | 1k_4k | 64 | PASS | 1112.31 | 4449.26 | 1374.52 | 14.35 |
+| dsv4-balanced | 1k_4k | 128 | PASS | 1913.20 | 7652.81 | 3162.08 | 16.57 |
+| dsv4-balanced | 1k_4k | 256 | PASS | 2441.07 | 9764.26 | 4386.00 | 26.51 |
+| dsv4-balanced | 64k_1 | 1 | PASS | 17118.25 | 0.26 | 3874.73 | 0.00 |
+| dsv4-balanced | 64k_1 | 4 | PASS | 61303.08 | 0.94 | 4407.47 | 0.00 |
+| dsv4-balanced | 64k_1 | 8 | PASS | 65564.19 | 1.00 | 8356.27 | 0.00 |
+| dsv4-balanced | 64k_1 | 16 | PASS | 65864.61 | 1.01 | 15823.61 | 0.00 |
+| dsv4-balanced | 64k_1 | 32 | PASS | 65504.60 | 1.00 | 32070.29 | 0.00 |
+| dsv4-balanced | 64k_1 | 64 | PASS | 65616.93 | 1.00 | 64081.25 | 0.00 |
+| dsv4-balanced | 64k_1 | 128 | PASS | 65401.37 | 1.00 | 128671.94 | 0.00 |
+| glm53-low-latency | 16k_512 | 1 | PASS | 7882.31 | 246.32 | 612.89 | 4.36 |
+| glm53-low-latency | 16k_512 | 8 | PASS | 16384.46 | 512.01 | 3892.78 | 22.46 |
+| glm53-low-latency | 16k_512 | 32 | PASS | 21303.26 | 665.73 | 15127.36 | 69.03 |
+| glm53-low-latency | 16k_512 | 64 | PASS | 23502.93 | 734.47 | 29490.35 | 105.62 |
+| glm53-low-latency | 16k_512 | 128 | PASS | 23920.46 | 747.51 | 63407.51 | 101.14 |
+| glm53-low-latency | 16k_512 | 256 | PASS | 24021.20 | 750.66 | 144384.42 | 102.36 |
+| glm53-low-latency | 16k_1 | 1 | FAIL | 26686.87 | 1.63 | 606.55 | 0.00 |
+| glm53-low-latency | 1k_128 | 1 | PASS | 2005.88 | 250.73 | 70.19 | 5.55 |
+| glm53-low-latency | 1k_128 | 8 | PASS | 6062.38 | 757.80 | 813.10 | 15.41 |
+| glm53-low-latency | 1k_128 | 32 | PASS | 11946.48 | 1493.31 | 1204.54 | 31.01 |
+| glm53-low-latency | 1k_128 | 64 | PASS | 14308.93 | 1788.62 | 1848.75 | 40.49 |
+| glm53-low-latency | 1k_128 | 128 | PASS | 14124.94 | 1765.62 | 6332.63 | 42.03 |
+| glm53-low-latency | 1k_128 | 256 | PASS | 13944.40 | 1743.05 | 15823.61 | 43.56 |
+| glm53-low-latency | 1k_4k | 1 | PASS | 109.08 | 436.33 | 70.69 | 3.32 |
+| glm53-low-latency | 1k_4k | 8 | PASS | 477.85 | 1911.42 | 435.51 | 5.56 |
+| glm53-low-latency | 1k_4k | 32 | PASS | 1191.77 | 4767.10 | 892.67 | 9.33 |
+| glm53-low-latency | 1k_4k | 64 | PASS | 1458.93 | 5835.71 | 19807.67 | 10.84 |
+| glm53-low-latency | 1k_4k | 128 | PASS | 1530.74 | 6122.95 | 56046.13 | 11.08 |
+| glm53-low-latency | 1k_4k | 256 | PASS | 1526.22 | 6104.87 | 140390.00 | 11.26 |
+| glm53-low-latency | 64k_1 | 1 | PASS | 24918.24 | 0.38 | 2625.80 | 0.00 |
+| glm53-low-latency | 64k_1 | 4 | FAIL | 25330.44 | 0.39 | 10319.69 | 0.00 |
+| glm53-low-latency | 64k_1 | 8 | PASS | 25336.86 | 0.39 | 20633.79 | 0.00 |
+| glm53-low-latency | 64k_1 | 16 | FAIL | 25368.13 | 0.39 | 41257.75 | 0.00 |
+| glm53-low-latency | 64k_1 | 32 | FAIL | 25376.71 | 0.39 | 82506.66 | 0.00 |
+| glm53-low-latency | 64k_1 | 64 | FAIL | 25399.08 | 0.39 | 164948.88 | 0.00 |
+| glm53-low-latency | 64k_1 | 128 | FAIL | 25411.76 | 0.39 | 329925.47 | 0.00 |
+| glm53-low-latency | 64k_512 | 1 | PASS | 15809.05 | 123.51 | 2626.48 | 4.40 |
+| glm53-low-latency | 64k_512 | 4 | PASS | 20932.05 | 163.53 | 5658.47 | 26.76 |
+| glm53-low-latency | 64k_512 | 8 | PASS | 22427.36 | 175.21 | 13072.49 | 61.58 |
+| glm53-low-latency | 64k_512 | 16 | PASS | 23295.48 | 182.00 | 31634.83 | 127.45 |
+| glm53-low-latency | 64k_512 | 32 | PASS | 23917.39 | 186.85 | 62694.72 | 243.87 |
+| glm53-low-latency | 64k_512 | 64 | PASS | 24220.17 | 189.22 | 128260.27 | 323.81 |
+| glm53-low-latency | 128k_1 | 1 | PASS | 22283.97 | 0.17 | 5887.63 | 0.00 |
+| glm53-low-latency | 128k_1 | 2 | PASS | 22544.03 | 0.17 | 11620.01 | 0.00 |
+| glm53-low-latency | 128k_1 | 4 | PASS | 22540.08 | 0.17 | 23234.84 | 0.00 |
+| glm53-low-latency | 128k_1 | 8 | FAIL | 22556.35 | 0.17 | 46447.98 | 0.00 |
+| glm53-low-latency | 128k_1 | 16 | FAIL | 22547.53 | 0.17 | 92876.53 | 0.00 |
+| glm53-low-latency | 128k_1 | 32 | FAIL | 22549.85 | 0.17 | 185810.07 | 0.00 |
+| glm53-low-latency | 128k_1 | 64 | TIMEOUT_PASS | 22567.39 | 0.17 | 371590.24 | 0.00 |
+| glm53-low-latency | 128k_512 | 1 | PASS | 17525.45 | 68.46 | 5894.95 | 4.62 |
+| glm53-low-latency | 128k_512 | 2 | PASS | 19570.06 | 76.45 | 6325.56 | 27.21 |
+| glm53-low-latency | 128k_512 | 4 | PASS | 20526.29 | 80.18 | 11700.22 | 62.53 |
+| glm53-low-latency | 128k_512 | 8 | PASS | 21132.30 | 82.55 | 28726.77 | 145.04 |
+| glm53-low-latency | 128k_512 | 16 | PASS | 21643.46 | 84.54 | 70612.08 | 282.81 |
+| glm53-low-latency | 128k_512 | 32 | PASS | 21787.55 | 85.11 | 142627.99 | 339.34 |
+| glm53-low-latency | 256k_1 | 1 | PASS | 17456.68 | 0.07 | 15141.04 | 0.00 |
+| glm53-low-latency | 256k_1 | 2 | PASS | 17620.27 | 0.07 | 29920.79 | 0.00 |
+| glm53-low-latency | 256k_1 | 4 | FAIL | 17584.82 | 0.07 | 59691.73 | 0.00 |
+| glm53-low-latency | 512k_1 | 1 | PASS | 12420.94 | 0.02 | 42470.03 | 0.00 |
+| glm53-low-latency | 512k_1 | 2 | PASS | 12497.10 | 0.02 | 84423.37 | 0.00 |
+| glm53-low-latency | 512k_1 | 4 | PASS | 12494.82 | 0.02 | 168091.37 | 0.00 |
+| glm53-low-latency | 1m_boundary_1 | 1 | PASS | 7897.30 | 0.01 | 132967.28 | 0.00 |
+| glm53-low-latency | 1m_boundary_1 | 2 | PASS | 7846.52 | 0.01 | 269850.67 | 0.00 |
+| glm53-low-latency | 1m_boundary_1 | 4 | PASS | 7914.10 | 0.01 | 509514.56 | 0.00 |
+| glm53-high-throughput | 16k_512 | 1 | PASS | 1753.70 | 54.80 | 1706.01 | 16.40 |
+| glm53-high-throughput | 16k_512 | 8 | PASS | 7602.48 | 237.58 | 2952.39 | 38.86 |
+| glm53-high-throughput | 16k_512 | 32 | PASS | 15478.02 | 483.69 | 10709.46 | 82.99 |
+| glm53-high-throughput | 16k_512 | 64 | PASS | 22864.39 | 714.51 | 17177.98 | 115.88 |
+| glm53-high-throughput | 16k_512 | 128 | PASS | 31889.21 | 996.54 | 30890.29 | 165.33 |
+| glm53-high-throughput | 16k_512 | 256 | PASS | 25722.47 | 803.83 | 58208.90 | 435.35 |
+| glm53-high-throughput | 16k_1 | 1 | FAIL | 10265.87 | 0.63 | 1635.66 | 0.00 |
+| glm53-high-throughput | 1k_128 | 1 | PASS | 440.61 | 55.08 | 354.14 | 17.25 |
+| glm53-high-throughput | 1k_128 | 8 | PASS | 2216.13 | 277.02 | 658.97 | 33.33 |
+| glm53-high-throughput | 1k_128 | 32 | PASS | 5512.82 | 689.10 | 1351.79 | 55.76 |
+| glm53-high-throughput | 1k_128 | 64 | PASS | 8781.68 | 1097.71 | 1544.35 | 75.33 |
+| glm53-high-throughput | 1k_128 | 128 | PASS | 15193.07 | 1899.13 | 2569.96 | 87.74 |
+| glm53-high-throughput | 1k_128 | 256 | PASS | 14521.72 | 1815.21 | 4585.68 | 192.57 |
+| glm53-high-throughput | 1k_4k | 1 | TIMEOUT_PASS | 15.31 | 61.25 | 722.89 | 21.05 |
+| glm53-high-throughput | 1k_4k | 8 | PASS | 122.82 | 491.26 | 585.67 | 17.31 |
+| glm53-high-throughput | 1k_4k | 32 | PASS | 458.53 | 1834.10 | 964.13 | 18.18 |
+| glm53-high-throughput | 1k_4k | 64 | PASS | 772.99 | 3091.95 | 1480.98 | 20.33 |
+| glm53-high-throughput | 1k_4k | 128 | PASS | 1421.86 | 5687.44 | 2492.89 | 23.05 |
+| glm53-high-throughput | 1k_4k | 256 | PASS | 2653.31 | 10613.25 | 4540.61 | 22.36 |
+| glm53-high-throughput | 64k_1 | 1 | PASS | 10461.13 | 0.16 | 6387.30 | 0.00 |
+| glm53-high-throughput | 64k_1 | 4 | PASS | 31751.47 | 0.48 | 8371.38 | 0.00 |
+| glm53-high-throughput | 64k_1 | 8 | PASS | 53720.01 | 0.82 | 9891.34 | 0.00 |
+| glm53-high-throughput | 64k_1 | 16 | FAIL | 56023.39 | 0.85 | 19477.62 | 0.00 |
+| glm53-high-throughput | 64k_1 | 32 | FAIL | 56599.60 | 0.86 | 36913.14 | 0.00 |
+| glm53-high-throughput | 64k_1 | 64 | FAIL | 56554.68 | 0.86 | 74260.73 | 0.00 |
+| glm53-high-throughput | 64k_1 | 128 | FAIL | 56357.44 | 0.86 | 153394.58 | 0.00 |
+| glm53-high-throughput | 64k_512 | 1 | PASS | 4707.22 | 36.78 | 6471.79 | 16.01 |
+| glm53-high-throughput | 64k_512 | 4 | PASS | 8844.03 | 69.09 | 8240.16 | 54.88 |
+| glm53-high-throughput | 64k_512 | 8 | PASS | 13286.74 | 103.80 | 9928.84 | 96.89 |
+| glm53-high-throughput | 64k_512 | 16 | PASS | 18863.70 | 147.37 | 19171.29 | 135.82 |
+| glm53-high-throughput | 64k_512 | 32 | PASS | 25109.46 | 196.17 | 29462.53 | 206.58 |
+| glm53-high-throughput | 64k_512 | 64 | PASS | 33437.72 | 261.23 | 57344.18 | 329.77 |
+| glm53-high-throughput | 128k_1 | 1 | FAIL | 10100.97 | 0.08 | 13350.24 | 0.00 |
+| glm53-high-throughput | 128k_512 | 1 | PASS | 6350.91 | 24.81 | 13384.88 | 16.13 |
+| glm53-high-throughput | 128k_512 | 2 | PASS | 7862.65 | 30.71 | 13400.23 | 41.95 |
+| glm53-high-throughput | 128k_512 | 4 | PASS | 9986.29 | 39.01 | 16714.05 | 92.60 |
+| glm53-high-throughput | 128k_512 | 8 | PASS | 13503.28 | 52.75 | 20219.46 | 171.97 |
+| glm53-high-throughput | 128k_512 | 16 | PASS | 19923.94 | 77.83 | 39164.93 | 264.45 |
+| glm53-high-throughput | 128k_512 | 32 | PASS | 27503.67 | 107.44 | 59905.46 | 401.75 |
+| glm53-high-throughput | 256k_1 | 1 | FAIL | 9204.74 | 0.04 | 29442.21 | 0.00 |
+| glm53-high-throughput | 512k_1 | 1 | PASS | 7532.88 | 0.01 | 71779.28 | 0.00 |
+| glm53-high-throughput | 512k_1 | 2 | PASS | 14141.46 | 0.03 | 77207.47 | 0.00 |
+| glm53-high-throughput | 512k_1 | 4 | PASS | 24677.77 | 0.05 | 86777.89 | 0.00 |
+| glm53-high-throughput | 1m_boundary_1 | 1 | PASS | 5330.26 | 0.01 | 204815.71 | 0.00 |
+| glm53-high-throughput | 1m_boundary_1 | 2 | PASS | 9956.15 | 0.01 | 214992.66 | 0.00 |
+| glm53-high-throughput | 1m_boundary_1 | 4 | PASS | 18580.87 | 0.02 | 225402.20 | 0.00 |
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/container.inspect.json b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/container.inspect.json
index 09e9622..1432832 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/container.inspect.json
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/container.inspect.json
@@ -52,7 +52,9 @@
         "MountLabel": "",
         "ProcessLabel": "",
         "AppArmorProfile": "docker-default",
-        "ExecIDs": null,
+        "ExecIDs": [
+            "f1a2dea8eb25c373aca73053e98b7f35e7a9ca3852f88b68bec48d293884b2b1"
+        ],
         "HostConfig": {
             "Binds": [
                 "/data/model/deepseek-ai--DeepSeek-V4-Flash-0731:/models:ro",
@@ -445,14 +447,6 @@
             "Name": "overlay2"
         },
         "Mounts": [
-            {
-                "Type": "bind",
-                "Source": "/data/model/deepseek-ai--DeepSeek-V4-Flash-0731",
-                "Destination": "/models",
-                "Mode": "ro",
-                "RW": false,
-                "Propagation": "rprivate"
-            },
             {
                 "Type": "bind",
                 "Source": "/data/b300-sglang-cache",
@@ -476,6 +470,14 @@
                 "Mode": "ro",
                 "RW": false,
                 "Propagation": "rprivate"
+            },
+            {
+                "Type": "bind",
+                "Source": "/data/model/deepseek-ai--DeepSeek-V4-Flash-0731",
+                "Destination": "/models",
+                "Mode": "ro",
+                "RW": false,
+                "Propagation": "rprivate"
             }
         ],
         "Config": {
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/gpu_state.csv b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/gpu_state.csv
index 89fbe19..926612a 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/gpu_state.csv
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/gpu_state.csv
@@ -1,9 +1,9 @@
 index, name, memory.total [MiB], memory.used [MiB], memory.free [MiB], utilization.gpu [%], power.draw [W]
-0, NVIDIA Graphics Device, 275040 MiB, 259411 MiB, 14703 MiB, 0 %, 242.62 W
-1, NVIDIA Graphics Device, 275040 MiB, 259603 MiB, 14511 MiB, 0 %, 234.74 W
-2, NVIDIA Graphics Device, 275040 MiB, 259617 MiB, 14497 MiB, 0 %, 235.85 W
-3, NVIDIA Graphics Device, 275040 MiB, 258643 MiB, 15471 MiB, 0 %, 243.30 W
-4, NVIDIA Graphics Device, 275040 MiB, 4 MiB, 274110 MiB, 0 %, 183.98 W
-5, NVIDIA Graphics Device, 275040 MiB, 4 MiB, 274110 MiB, 0 %, 182.88 W
-6, NVIDIA Graphics Device, 275040 MiB, 4 MiB, 274110 MiB, 0 %, 184.01 W
-7, NVIDIA Graphics Device, 275040 MiB, 4 MiB, 274110 MiB, 0 %, 184.09 W
+0, NVIDIA Graphics Device, 275040 MiB, 262083 MiB, 12031 MiB, 0 %, 304.84 W
+1, NVIDIA Graphics Device, 275040 MiB, 261373 MiB, 12741 MiB, 0 %, 295.27 W
+2, NVIDIA Graphics Device, 275040 MiB, 261495 MiB, 12619 MiB, 0 %, 294.86 W
+3, NVIDIA Graphics Device, 275040 MiB, 260091 MiB, 14023 MiB, 0 %, 306.36 W
+4, NVIDIA Graphics Device, 275040 MiB, 4 MiB, 274110 MiB, 0 %, 183.70 W
+5, NVIDIA Graphics Device, 275040 MiB, 4 MiB, 274110 MiB, 0 %, 182.13 W
+6, NVIDIA Graphics Device, 275040 MiB, 4 MiB, 274110 MiB, 0 %, 183.44 W
+7, NVIDIA Graphics Device, 275040 MiB, 4 MiB, 274110 MiB, 0 %, 183.97 W
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c128.done b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c128.done
new file mode 100644
index 0000000..6e5e30c
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c128.done
@@ -0,0 +1 @@
+2026-09-10T07:38:13+00:00
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c128.json b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c128.json
new file mode 100644
index 0000000..0dec65c
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c128.json
@@ -0,0 +1 @@
+{"tag": null, "backend": "sglang", "dataset_name": "random-ids", "request_rate": Infinity, "max_concurrency": 128, "sharegpt_output_len": null, "random_input_len": 65536, "random_output_len": 1, "random_range_ratio": 1.0, "server_info": {"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": false, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": null, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": null, "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "status": "ready", "max_total_num_tokens": 25820928, "max_req_input_len": 1048570, "startup_time": {"load_weight": 348.83619413909037, "kv_cache_allocation": 0.8970330059528351, "scheduler_e2e": 508.1397862319136, "cuda_graph": {"prefill": 0.0, "decode": 89.43910579301883, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}, "tokenizer_e2e": 526.9549640880432}, "scheduler_pids": [440, 441, 442, 443], "internal_states": [{"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 64, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "_resolved_overrides": [["_deepseek_v4_overrides", {"attention_backend": "dsv4", "page_size": 256, "swa_full_tokens_ratio": 0.1}], ["_deepseek_v4_kv_cache_dtype", {"kv_cache_dtype": "fp8_e4m3"}], ["_sampling_backend_default", {"sampling_backend": "flashinfer"}], ["_tp_lm_head_all_to_all_default", {"enable_tp_lm_head_all_to_all": false}], ["_a2a_ep_size", {"ep_size": 4}], ["_speculative_moe_runner_default", {"speculative_moe_runner_backend": "auto"}]], "grpc_worker_threads": 4, "_quantization_explicitly_unset": false, "_cuda_graph_config_locked": [], "_declarations_materialized": true, "_mx_config_cache": {}, "max_speculative_num_draft_tokens": null, "last_gen_throughput": 3655.157740872784, "memory_usage": {"weight": 42.971, "kvcache": 0, "startup_available": 14.438, "token_capacity": 25820928, "token_capacity_swa": 2582016, "graph": {"prefill": 0.0, "decode": 19.176, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 348.49362135794945, "kv_cache_allocation": 0.8837988879531622, "scheduler_e2e": 507.92600555799436, "cuda_graph": {"prefill": 0.0, "decode": 89.42433235095814, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 64}, {"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 64, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "_resolved_overrides": [["_deepseek_v4_overrides", {"attention_backend": "dsv4", "page_size": 256, "swa_full_tokens_ratio": 0.1}], ["_deepseek_v4_kv_cache_dtype", {"kv_cache_dtype": "fp8_e4m3"}], ["_sampling_backend_default", {"sampling_backend": "flashinfer"}], ["_tp_lm_head_all_to_all_default", {"enable_tp_lm_head_all_to_all": false}], ["_a2a_ep_size", {"ep_size": 4}], ["_speculative_moe_runner_default", {"speculative_moe_runner_backend": "auto"}]], "grpc_worker_threads": 4, "_quantization_explicitly_unset": false, "_cuda_graph_config_locked": [], "_declarations_materialized": true, "_mx_config_cache": {}, "max_speculative_num_draft_tokens": null, "last_gen_throughput": 3809.2647395807885, "memory_usage": {"weight": 42.971, "kvcache": 0, "startup_available": 15.19, "token_capacity": 25820928, "token_capacity_swa": 2582016, "graph": {"prefill": 0.0, "decode": 18.549, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 348.5880510379793, "kv_cache_allocation": 0.8970330059528351, "scheduler_e2e": 508.1053619449958, "cuda_graph": {"prefill": 0.0, "decode": 89.43910579301883, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 64}, {"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 64, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "_resolved_overrides": [["_deepseek_v4_overrides", {"attention_backend": "dsv4", "page_size": 256, "swa_full_tokens_ratio": 0.1}], ["_deepseek_v4_kv_cache_dtype", {"kv_cache_dtype": "fp8_e4m3"}], ["_sampling_backend_default", {"sampling_backend": "flashinfer"}], ["_tp_lm_head_all_to_all_default", {"enable_tp_lm_head_all_to_all": false}], ["_a2a_ep_size", {"ep_size": 4}], ["_speculative_moe_runner_default", {"speculative_moe_runner_backend": "auto"}]], "grpc_worker_threads": 4, "_quantization_explicitly_unset": false, "_cuda_graph_config_locked": [], "_declarations_materialized": true, "_mx_config_cache": {}, "max_speculative_num_draft_tokens": null, "last_gen_throughput": 3813.8562249765855, "memory_usage": {"weight": 42.971, "kvcache": 0, "startup_available": 14.253, "token_capacity": 25820928, "token_capacity_swa": 2582016, "graph": {"prefill": 0.0, "decode": 19.33, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 348.83619413909037, "kv_cache_allocation": 0.8781842190073803, "scheduler_e2e": 507.96765940997284, "cuda_graph": {"prefill": 0.0, "decode": 89.42852122697514, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 64}, {"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 64, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "_resolved_overrides": [["_deepseek_v4_overrides", {"attention_backend": "dsv4", "page_size": 256, "swa_full_tokens_ratio": 0.1}], ["_deepseek_v4_kv_cache_dtype", {"kv_cache_dtype": "fp8_e4m3"}], ["_sampling_backend_default", {"sampling_backend": "flashinfer"}], ["_tp_lm_head_all_to_all_default", {"enable_tp_lm_head_all_to_all": false}], ["_a2a_ep_size", {"ep_size": 4}], ["_speculative_moe_runner_default", {"speculative_moe_runner_backend": "auto"}]], "grpc_worker_threads": 4, "_quantization_explicitly_unset": false, "_cuda_graph_config_locked": [], "_declarations_materialized": true, "_mx_config_cache": {}, "max_speculative_num_draft_tokens": null, "last_gen_throughput": 3809.8602166968576, "memory_usage": {"weight": 42.971, "kvcache": 0, "startup_available": 14.253, "token_capacity": 25820928, "token_capacity_swa": 2582016, "graph": {"prefill": 0.0, "decode": 19.33, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 348.43866326799616, "kv_cache_allocation": 0.8657742600189522, "scheduler_e2e": 508.1397862319136, "cuda_graph": {"prefill": 0.0, "decode": 89.43686953000724, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 64}], "version": "0.5.18", "kv_events": null}, "duration": 641.3174964149948, "completed": 640, "total_input_tokens": 41943040, "total_input_text_tokens": 41943040, "total_input_vision_tokens": 0, "total_output_tokens": 640, "total_output_tokens_retokenized": 635, "request_throughput": 0.9979456409308031, "input_throughput": 65401.36552404111, "output_throughput": 0.9979456409308031, "total_throughput": 65402.36346968204, "mean_e2e_latency_ms": 115550.41285510379, "median_e2e_latency_ms": 127346.22153197415, "std_e2e_latency_ms": 29309.505262483864, "p90_e2e_latency_ms": 128318.6184164486, "p95_e2e_latency_ms": 128671.9865934283, "p99_e2e_latency_ms": 128993.9503739588, "mean_ttft_ms": 114558.38273423178, "median_ttft_ms": 127340.6884399592, "std_ttft_ms": 31005.598780711167, "p90_ttft_ms": 128318.57239600504, "p95_ttft_ms": 128671.9423642906, "p99_ttft_ms": 128993.88444155455, "mean_tpot_ms": 0.0, "median_tpot_ms": 0.0, "std_tpot_ms": 0.0, "p90_tpot_ms": 0.0, "p95_tpot_ms": 0.0, "p99_tpot_ms": 0.0, "mean_itl_ms": 0.0, "median_itl_ms": 0.0, "std_itl_ms": 0.0, "p90_itl_ms": 0.0, "p95_itl_ms": 0.0, "p99_itl_ms": 0.0, "concurrency": 115.31303081650546, "accept_length": null, "max_output_tokens_per_s": 5.0, "max_concurrent_requests": 132, "input_lens": [65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536], "output_lens": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], "ttfts": [5.599223662051372, 9.611336312023923, 9.84511902590748, 13.586102794040926, 13.59786448709201, 13.580463196034543, 77.8123869599076, 81.54827413498424, 81.5303853310179, 81.52640387695283, 81.77209602703806, 85.49879830295686, 85.47984657692723, 85.47710943000857, 85.72255788207985, 89.43387201207224, 89.44552756298799, 13.761062472010963, 89.39468626410235, 89.63923702400643, 101.31895185902249, 93.35776170901954, 101.3113588889828, 101.32350661500823, 101.55390589800663, 105.28996918606572, 93.34076679905411, 105.28369524690788, 105.26508093392476, 105.51235767500475, 109.22681930300314, 109.23779415606987, 109.21947919402737, 93.33228789607529, 109.4603746549692, 113.17502454703208, 113.18582319992129, 113.18306313606445, 113.41426464600954, 117.15917249000631, 117.14137722400483, 117.13752433599439, 117.38408580201212, 93.53544972499367, 121.11645721190143, 121.09909935295582, 121.0950231939787, 121.34064720699098, 97.24800305010285, 125.04787893500179, 125.04398380103521, 0.0, 125.28533996501938, 97.23090800305363, 129.00538203306496, 128.98737355903722, 128.98490046802908, 97.23298746999353, 129.2259062039666, 97.46262054296676, 17.47388687497005, 17.45647601399105, 17.452322802040726, 17.703199587995186, 21.545828646980226, 21.55577827396337, 21.537216217024252, 21.784390076994896, 25.53824376699049, 25.534293039003387, 25.516813199967146, 25.214406694984064, 28.957314608967863, 28.969823876977898, 28.951510371058248, 29.197652896982618, 32.97846525500063, 32.975263779982924, 32.957263416959904, 33.20479817793239, 36.94451802200638, 36.928617925033905, 36.910352892940864, 37.16247645195108, 40.913128726999275, 40.89518824906554, 4.719999682041816, 40.88805020600557, 4.698442125925794, 4.945559482905082, 41.1286512700608, 44.86830084200483, 44.849885902018286, 44.84734587999992, 45.09322730801068, 48.82404653902631, 48.80653623701073, 48.80257871490903, 49.04812743002549, 52.77000148105435, 52.76742072694469, 52.79219530289993, 53.01168458699249, 56.74422147194855, 56.75507489708252, 8.642608766909689, 56.733776867971756, 56.98146685794927, 60.72463841899298, 60.70650064002257, 60.703840987058356, 60.950140521046706, 64.68723772303201, 64.68372189300135, 64.66592787799891, 64.91394737397786, 68.64817596192006, 68.62996869895142, 68.62739162193611, 68.87330818001647, 72.60144516697619, 72.59736836492084, 72.60904915397987, 72.84109821391758, 76.57732784899417, 8.591948317945935, 76.55584783898667, 76.56689955492038, 127.57919908501208, 127.54977222799789, 127.54996688198298, 127.56282432307489, 127.54455573495943, 127.51435367600061, 127.50120409892406, 127.52839591598604, 127.51257147500291, 127.4831694700988, 127.46921042411122, 127.4948857439449, 127.3756477109855, 127.36044437799137, 127.36118616303429, 127.3724781560013, 127.26551478810143, 127.2359991039848, 127.22206546703819, 127.24896686407737, 127.2331193840364, 127.20326161698904, 127.18833410507068, 127.2164366840152, 127.19852706906386, 127.16878822201397, 127.15497803501785, 127.18151990300976, 127.1435826550005, 127.11325220298022, 127.10018669697456, 127.12601067509968, 127.11647670692764, 127.10186436399817, 127.07241926703136, 127.0951982929837, 127.08565560495481, 127.0569334620377, 127.0416526820045, 127.07022695394699, 127.07696169998962, 127.04750754206907, 127.03371186298318, 127.06143114797305, 127.06023688102141, 127.04526128806174, 127.04590789403301, 127.06082079198677, 127.06756672007032, 127.05325391306542, 127.02317606203724, 127.05552913795691, 127.03969021199737, 127.03848868701607, 127.00831106002443, 127.03678144095466, 127.01992530794814, 127.02014825202059, 126.99063743709121, 127.0179326820653, 127.00859528500587, 126.99239903502166, 0.0, 127.00440064095892, 127.0024374150671, 126.98896183399484, 126.98876827897038, 127.00208613602445, 126.98858738900162, 126.9894347799709, 126.9744801990455, 126.98766890005209, 126.99437020497862, 126.96446755900979, 126.95138325402513, 126.98001859802753, 126.98592559702229, 126.9563391940901, 126.94253449502867, 126.97131303802598, 126.99014600098599, 126.95997704099864, 126.946268633008, 126.9737463000929, 126.9928714199923, 126.96397735201754, 126.94910170999356, 126.97779469494708, 126.97070533595979, 126.9557340699248, 126.95623360807076, 126.96736561099533, 126.94833557796665, 126.9480861669872, 126.91845878597815, 126.94518864608835, 126.93839404790197, 126.92440075206105, 126.93782371794805, 126.93997002800461, 126.94056703010574, 126.92414698691573, 126.92543484596536, 126.9389856599737, 126.95834585605189, 126.93006599496584, 126.9148742480902, 126.94458536093589, 126.9378945180215, 126.93671833293047, 126.90855384594761, 126.93705924798269, 126.92216142208781, 126.90841465105768, 126.90844229096547, 126.92160253191832, 126.92420268303249, 126.89454590098467, 126.88043907296378, 126.90939109190367, 126.93008432700299, 126.89994658890646, 126.88638271205127, 126.91573593299836, 126.93456927593797, 126.90510359802283, 126.8912366440054, 126.92196125199553, 127.2184882259462, 127.21846843103413, 127.19049155700486, 127.22212870710064, 127.25877060904168, 127.22974796895869, 127.21470141899772, 127.245302904048, 127.26889404200483, 127.25550958805252, 127.25489949004259, 127.27347446000203, 127.2895158980973, 127.27523373998702, 127.24535577103961, 127.27279035898391, 127.26642493309919, 127.26664135500323, 127.23197024699766, 127.26704586704727, 127.27975738502573, 127.28032315894961, 127.25148014805745, 127.2864399390528, 127.29888226697221, 127.29887665703427, 127.2687462560134, 127.30069023207761, 127.29783935297746, 127.29801310203038, 127.26798131398391, 127.30040341697168, 127.4875894789584, 127.45740254502743, 127.44301164895296, 127.4746582810767, 127.51929561200086, 127.50479086302221, 127.4758658161154, 127.5066067520529, 127.52257858903613, 127.52257329598069, 127.49265674606431, 127.52321142994333, 127.69801407598425, 127.66815595899243, 127.65301076194737, 127.68440868298057, 127.71273030200973, 127.71228822798003, 0.0, 127.71063766197767, 127.76488209702075, 127.76504715601914, 127.7362046930939, 127.76978271000553, 127.8048383849673, 127.78950515808538, 127.79007668804843, 127.80782130593434, 127.84609584196005, 127.8465154709993, 127.81804543500766, 127.85115584603045, 127.87045525410213, 127.87067613296676, 0.0, 127.8726263770368, 127.89666951808613, 127.89619601902086, 127.86833783891052, 127.89898180996533, 127.92127199994866, 127.92119520506822, 127.89202848507557, 127.92181917908601, 127.94158395600971, 127.9416450340068, 127.91200919402763, 127.94306462397799, 127.95840370899532, 127.96400248201098, 127.9299321109429, 127.96471104503144, 127.99499138107058, 127.99517192307394, 127.96555777499452, 127.9978412979981, 128.0684764700709, 128.05448793596588, 128.02442667004652, 128.09765344799962, 128.12940039497335, 128.13055028801318, 128.1013036890654, 128.1330104030203, 128.15201176900882, 128.1384656430455, 128.13778234506026, 128.15228141611442, 128.16454982606228, 128.15053257893305, 128.12084534903988, 128.15229964489117, 128.170432309038, 128.15630752802826, 128.126469128998, 128.15881086094305, 128.1904687579954, 128.19080777803902, 128.1541644850513, 128.19322560797445, 128.20361866196617, 128.20395534101408, 128.1720657380065, 128.20495100796688, 128.234489590046, 128.2205094489036, 128.1904881959781, 128.2211977809202, 128.24327195598744, 128.2136948739644, 128.19854045298416, 128.22981188993435, 128.2312991679646, 128.23191547300667, 128.20176286494825, 128.23020038905088, 127.93656350998208, 127.93633217201568, 127.90636735910084, 127.93357665208168, 127.93097906990442, 127.90168213495053, 127.8863849180052, 127.91398822504561, 127.89536283607595, 127.89533278904855, 127.86565245105885, 127.89159994199872, 127.8877790030092, 127.88648106891196, 127.85770278505515, 127.88740809995215, 127.8933335270267, 127.89435220998712, 127.86496583104599, 127.8954187349882, 127.89124645292759, 127.8918016969692, 127.86172410205472, 127.88536840793677, 127.87614533794113, 127.87611262791324, 127.84646483894903, 127.87752688501496, 127.88176844106056, 127.88169418496545, 127.85184979799669, 127.88092636095826, 127.71279466000851, 127.71320915000979, 127.68332122196443, 127.71177066292148, 127.69575375900604, 127.69535573106259, 127.66590202902444, 127.69574226404075, 127.69380400993396, 127.6938590790378, 127.6633267638972, 127.69401961704716, 127.54669297405053, 127.5467412120197, 127.51693423802499, 127.54506004892755, 127.5420753180515, 127.52713750198018, 127.49765247805044, 127.52494350099005, 127.46826346800663, 127.46986427204683, 127.45522565289866, 127.46478318294976, 127.43336251797155, 127.41991834598593, 127.42052708298434, 127.43231289298274, 127.40136633091606, 127.40365061897319, 127.37315456697252, 127.39952955301851, 127.38587631098926, 127.38818205706775, 127.35766455798876, 127.3843862140784, 127.36561564891599, 127.36787960806396, 127.33681270398665, 127.36460179300047, 127.34531268198043, 127.34705391700845, 127.33218320901506, 127.34456417593174, 127.32769439101685, 127.32973671192303, 127.29944699304178, 127.32637728599366, 127.31687596894335, 127.31363881996367, 0.0, 127.31296973500866, 127.28349549509585, 127.28477157000452, 127.25323762104381, 127.28181876789313, 127.23565319296904, 127.23550777800847, 127.20494261593558, 127.19315039401408, 127.17576480901334, 127.17691034101881, 127.14566652989015, 127.17201306903735, 127.15484160999767, 127.15603802003898, 127.1247037970461, 127.15333730808925, 127.1577272239374, 127.14289854303934, 127.14469097997062, 127.15540388296358, 127.16228340403177, 127.13074518390931, 127.1162932709558, 127.14323577296454, 127.13519709894899, 127.12125850305893, 127.08979032293428, 127.11710392392706, 127.11303752602544, 127.09853055898566, 127.0671670739539, 127.09596928092651, 127.08113506692462, 127.0648807019461, 127.06718166498467, 127.0789132269565, 127.19262025796343, 127.19383563403971, 127.16229054599535, 127.19965024199337, 127.5228297059657, 127.52403871703427, 127.49235632305499, 127.52696973097045, 127.58909165894147, 127.5730708859628, 127.5572606890928, 127.59248397697229, 127.64549502008595, 127.63192498299759, 127.61505872197449, 127.65081102505792, 127.6988369160099, 127.686025641975, 127.66835497308057, 127.704871314927, 127.75545712700114, 127.7416679339949, 127.72518366994336, 127.7638483470073, 127.80990214494523, 127.79620654403698, 127.77959940303117, 127.81086073501501, 127.86720187705941, 127.85307070997078, 127.83671854098793, 127.86960624402855, 127.91456920804922, 127.90109716402367, 127.88436047302093, 127.91554211603943, 127.95669874106534, 127.94328756898176, 127.92619569494855, 127.96002577908803, 127.9989013139857, 127.98603867797647, 127.96999288909137, 128.00047004397493, 128.0353841050528, 128.0219077100046, 128.0064314280171, 128.0359473299468, 128.06944824801758, 128.0556230309885, 128.0388076780364, 128.06914769404102, 128.09566122107208, 128.08211812202353, 128.06545784091577, 128.09967911907006, 128.13317490997724, 128.11616301501635, 128.10157372395042, 128.13518691493664, 128.1861624380108, 128.17044263496064, 128.152902976959, 128.18856206093915, 128.23861841007601, 128.2236480790889, 128.20888039807323, 128.24221399996895, 128.2861220400082, 128.2717729510041, 128.25355012889486, 128.28699375793803, 128.3143334799679, 128.29687153804116, 128.28008978406433, 128.31749708100688, 128.36100595700555, 128.34625199297443, 128.3282502309885, 128.36255277798045, 128.40657347801607, 128.3910607950529, 128.3738584719831, 128.408507226035, 128.4471972300671, 128.43307766399812, 128.41603552596644, 128.44995291007217, 128.49114977393765, 128.47561201197095, 128.45871504792012, 128.49395880301017, 128.53661448601633, 128.52159134007525, 128.5036269709235, 128.539279220975, 128.57761412498076, 128.56291080010124, 128.54500277608167, 128.5788260420086, 128.61925750307273, 128.60473902896047, 128.58666183892637, 128.62387288792524, 128.67150301509537, 128.6568874020595, 128.6415654689772, 128.6802899989998, 128.71713253797498, 128.70224607398268, 128.68409106798936, 128.72079574305099, 128.76889763202053, 128.7524237940088, 128.7381631199969, 128.77166196191683, 128.8963895359775, 128.8802285399288, 128.86316535796504, 128.8978736179415, 128.94899907894433, 128.93311767303385, 128.9157573740231, 128.95071046892554, 128.99998325109482, 128.98559849499725, 128.96833874005824, 129.0033338360954, 129.01305143604986, 128.99804713693447, 128.97970908496063, 129.00511134206317, 128.72299299691804, 128.70617926609702, 128.69166463403963, 128.4846618770389], "itls": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "generated_texts": ["\u0432", "\u73fe", " ", " ", "\u0432", "\ufffd", "\u0432", "\u0432", "\u0432", ".\n\n", "UM", "\u0432", "ices", ".\n\n", "\ufffd", "\u0432", "\u043f", " ", "\u0435\u043d", "\u05ea", " ", "\u0627", "\u0432", "\n", "ric", ".\n\n", "\u6d3e", "\u0432", "o", "\u7684", "ar", "ident", "\u0430", "\u043e\u0433", "\u0432", "c", "\u0432", "\n", "r", "\u0442", " ", "_", "\n", "el", "als", "\u518c", "\u0432", "\u0432", "ry", "\u0432", " ", "", "\u0434", "ament", ".\n\n", "\u0432", "ION", "q", "hasa", "s", "U", "\u0432", ".\n\n", "(", " ", "\u043e\u0432", " ", "\u0431", "\u0434", "\u7684", " ", "\u043f", ".\n\n", " ", "g", "os", "\u0432", " ", "\u5c4a", " ", "istics", "\u96f7", ".\n\n", "\u0432", "\u0432", "\u7684", " ", " ", " ", "\u0432", ".\n\n", "\u0432", " ", "\u0432", "\u0432", " ", "\u0432", ".\n\n", "\u0432", "\u062d", "\u0432", "\u0436", "ings", "View", "ul", "\u0432", " ", "et", "\u0432", " ", "\u0432", "stand", "ident", ".\n\n", "\u043f", "e", "\u09c7", "ia", "\u0432", " ", ".\n\n", "\u09bf\u0995", "n", "\u0441\u044f", ".\n\n", "\u09c7", "\u0442", "}", "\n", "\u7684", "\u0434", "\u0447\u0435\u0441\u043a\u0438\u0445", " ", ".\n\n", "\u0444", "Name", "t", "s", " ", "\u0432", "\u9879", ".\n\n", " ", "\u0432", " ", " ", "\u043f", "\u0432", ".\n\n", "\u064a", " ", ".\n\n", "s", "ity", " ", ".\n\n", "quake", "\u0442", ".\n\n", "\u63a7", "\u043d\u043e", "e", "rette", "\u092a", "\u0432", "ard", "\u0432", "\u0434", " ", ".\n\n", ".\n\n", "\u0447", " ", "\u043f", "\u0432", " ", "\u0432", "\u0434", "entation", "\u7684", ".\n\n", "\u7684", " ", "ce", "\u2588\u2588\u2588\u2588", "\u064a\u0628", "\u3002\n\n", "K", ")", "\u0432", "", " ", " ", "\u0432", "\u0432", " ", "ip", ".\n\n", "\u7684", "\u9633", "\u0432", "\u0e35", "e", " ", "\u5417", " ", ".\n\n", " ", "\u0434", "\u0432", "\u0432", "\ub85d", "\u0432", " ", "\u0434", "\u0440\u043e\u0432", "\u043f", " ", "\u03bd", "\u0432", " ", "\u0432", "ian", ".\n\n", "\u0432", "ity", "R", ".\n\n", "able", "\uff09", ".\n\n", "\u0434", ".\n\n", "\u0432", " ", "s", "m", ".\n", ".\n\n", "\u0444", "\u0432", "\u09cd\u09af", " ", "ie", "\u0434", ".\n\n", "ida", "\u0432", "c", " ", "\u0432", " ", "\u0432", "orn", "as", " ", "ac", "\u0432", " ", "\u0432", "arn", " ", "ant", "\u308a", "\u0432", "\u0434", "\u0432", "ure", "\u0432", ".\n\n", ".\n\n", "\u7684", "\u0432", " ", "\u4e0a", "s", "r", ".\n\n", "\u0432", "board", ".\n\n", "ales", "\u0432", "\u0434", "um", ".\n\n", "\u306e", " ", "\u0432", "\u52a8\u7269", "\u0432", "\u0432", " ", "\u0432", "\u0434", "\u00f3n", "\u0432", "imum", "al", "\u0e04", "ion", "\u5929", "\u7684", "\u0432", " ", "\u0434", "", " ", "m", "e", ".\n\n", ".\n\n", " ", "s", "\u0432", " ", ".\n\n", "ation", "\u7684", "ary", " ", " ", "", "es", "\n", "ing", " ", "\ufffd", "\ufffd", " ", "\u043f", "ion", ".\n\n", "\u7684", "\u043d\u0438\u043a", "160", "\u0432", " the", "run", " ", "\u968f", "\u00fc", "\u0432", ".\n\n", " ", ".\n\n", " ", "\n", "ob", "5", "\u5bbe", "con", " ", ".\n\n", "\u0432", "\u0432", "\u043f", "\u5316", " ", " ", "\u0432", "um", "\u0432", ".\n\n", " ", "\u0e22", "e", "e", "indows", ".\n\n", "ity", "estic", "\u0432", " ", "\u0432", ".\n\n", "rig", ".\n\n", "\u0432", "\u0432", ".", " ", "oca", "\u0430", "\u0432", " ", ".\n\n", "ite", "\u0432", "\u0432", " ", "S", "\u0432", "\u0442", "v", "\u0432", "ART", "\u0432", "w", "ologies", "\u0432", ".", "ic", " ", " ", " ", "\u0432", "\u0442", "ation", " ", ".\n\n", " ", "e", ".\n\n", "e", ".\n\n", "\u0432", " ", " ", " ", "\u0432", "ion", ".\n\n", "\u0432", " ", "s", ".\n\n", " ", "ull", " ", "n", "\u0432", ".\n\n", "\ufffd", "r", "\u0432", "ness", " ", "\u535a", "rop", " ", "ity", "\u0432", "\u0432", " ", " ", "\u0432", "\u043c", "o", ".\n\n", " ", " ", "adas", "\u0442", "\u5b9e", " ", "\t", "ra", "uno", ".\n\n", " ", "\u590d", "oration", "\u308a", " ", "ork", "", ".\n\n", ".\n\n", "ol", "\uc5b4", "\u043c", ".\n\n", ".\n\n", " ", " ", "\u0432", "(", "pole", "ection", "strate", "\u0432", "zip", ".\n\n", "\u0432", "h", " ", " ", "\u0627\u062a", "\u0432", "ren", "at", "\u0e02", " ", "\u7684", ".\n\n", "ator", "\u0bcd", " ", " ", "\u043e\u0442\u044b", "\u043f", " M", "\u0432", "\u0432", "\u043f", " ", "\u0432", "\ufffd", " ", " ", "\u0432", " ", "st", " ", " ", "\u0432", "\u0432", "\u0432", "pler", "ft", ".\n\n", "ity", " ", ".\n\n", "\u7684", "ik", "lev", "ive", " ", "\u0446\u0438\u0438", "\u0432", "\u7684", "ANCE", "ect", ".\n\n", "os", "ated", "ir", "\u0432", "\u0432", " ", "\u0432", "\u0627\u062a", "\u0442", " ", "\u043f", "\u0432", "\u0441\u0442\u0432", "\u5206", "as", "\ufffd", ".\n\n", "\u043c\u0435\u043d\u0442", ".\n\n", ".\n\n", ".\n\n", "\u043f", "as", "\u0432", ".\n\n", "\u4f5c", "\u0432", "ction", "\u0434", "\u0432", "\u0432", " ", "\u7684", ".\n\n", "\u0432", " ", "la", "\u0432", "\u0432", "\u5417", " ", "]", "\u09a8", "\u0432", "ang", "\u03bc", "\u0435", "\n", "\u7684", "\u0432", "</think>", "hots", "\u09bf", " ", "\u0443\u0447", "\u0432", "\u043f", "\u0432", "\u0442", ".\n\n", " ", " ", "e", "\u0432", "\u0432", " ", "\u0432", ".\n\n", "s", "\u0432", " ", "e", ".", ".\n\n", ";", " ", ".\n\n", "\u043f", "\u00e2nt", "\u0432", ".\n\n", " ", "c", "ical", "\u09c7", "\u0446", "\u062e", "\u0435", " ", "\u0432", "\u0432", "\u0432", " ", "9", ".\n\n", "\u0442", " ", "\u0432", " ", "\u0432", "ch", "\u043d\u044f", "\u043f", "\u0434"], "errors": ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""]}
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.done b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.done
new file mode 100644
index 0000000..fe7ad29
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.done
@@ -0,0 +1 @@
+2026-09-10T07:26:57+00:00
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.json b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.json
new file mode 100644
index 0000000..3ebbb5f
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.json
@@ -0,0 +1 @@
+{"tag": null, "backend": "sglang", "dataset_name": "random-ids", "request_rate": Infinity, "max_concurrency": 64, "sharegpt_output_len": null, "random_input_len": 65536, "random_output_len": 1, "random_range_ratio": 1.0, "server_info": {"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": false, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": null, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": null, "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "status": "ready", "max_total_num_tokens": 25820928, "max_req_input_len": 1048570, "startup_time": {"load_weight": 348.83619413909037, "kv_cache_allocation": 0.8970330059528351, "scheduler_e2e": 508.1397862319136, "cuda_graph": {"prefill": 0.0, "decode": 89.43910579301883, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}, "tokenizer_e2e": 526.9549640880432}, "scheduler_pids": [440, 441, 442, 443], "internal_states": [{"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 64, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "_resolved_overrides": [["_deepseek_v4_overrides", {"attention_backend": "dsv4", "page_size": 256, "swa_full_tokens_ratio": 0.1}], ["_deepseek_v4_kv_cache_dtype", {"kv_cache_dtype": "fp8_e4m3"}], ["_sampling_backend_default", {"sampling_backend": "flashinfer"}], ["_tp_lm_head_all_to_all_default", {"enable_tp_lm_head_all_to_all": false}], ["_a2a_ep_size", {"ep_size": 4}], ["_speculative_moe_runner_default", {"speculative_moe_runner_backend": "auto"}]], "grpc_worker_threads": 4, "_quantization_explicitly_unset": false, "_cuda_graph_config_locked": [], "_declarations_materialized": true, "_mx_config_cache": {}, "max_speculative_num_draft_tokens": null, "last_gen_throughput": 3655.157740872784, "memory_usage": {"weight": 42.971, "kvcache": 0, "startup_available": 14.438, "token_capacity": 25820928, "token_capacity_swa": 2582016, "graph": {"prefill": 0.0, "decode": 19.176, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 348.49362135794945, "kv_cache_allocation": 0.8837988879531622, "scheduler_e2e": 507.92600555799436, "cuda_graph": {"prefill": 0.0, "decode": 89.42433235095814, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 64}, {"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 64, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "_resolved_overrides": [["_deepseek_v4_overrides", {"attention_backend": "dsv4", "page_size": 256, "swa_full_tokens_ratio": 0.1}], ["_deepseek_v4_kv_cache_dtype", {"kv_cache_dtype": "fp8_e4m3"}], ["_sampling_backend_default", {"sampling_backend": "flashinfer"}], ["_tp_lm_head_all_to_all_default", {"enable_tp_lm_head_all_to_all": false}], ["_a2a_ep_size", {"ep_size": 4}], ["_speculative_moe_runner_default", {"speculative_moe_runner_backend": "auto"}]], "grpc_worker_threads": 4, "_quantization_explicitly_unset": false, "_cuda_graph_config_locked": [], "_declarations_materialized": true, "_mx_config_cache": {}, "max_speculative_num_draft_tokens": null, "last_gen_throughput": 3813.8562249765855, "memory_usage": {"weight": 42.971, "kvcache": 0, "startup_available": 14.253, "token_capacity": 25820928, "token_capacity_swa": 2582016, "graph": {"prefill": 0.0, "decode": 19.33, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 348.83619413909037, "kv_cache_allocation": 0.8781842190073803, "scheduler_e2e": 507.96765940997284, "cuda_graph": {"prefill": 0.0, "decode": 89.42852122697514, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 64}, {"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 64, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "_resolved_overrides": [["_deepseek_v4_overrides", {"attention_backend": "dsv4", "page_size": 256, "swa_full_tokens_ratio": 0.1}], ["_deepseek_v4_kv_cache_dtype", {"kv_cache_dtype": "fp8_e4m3"}], ["_sampling_backend_default", {"sampling_backend": "flashinfer"}], ["_tp_lm_head_all_to_all_default", {"enable_tp_lm_head_all_to_all": false}], ["_a2a_ep_size", {"ep_size": 4}], ["_speculative_moe_runner_default", {"speculative_moe_runner_backend": "auto"}]], "grpc_worker_threads": 4, "_quantization_explicitly_unset": false, "_cuda_graph_config_locked": [], "_declarations_materialized": true, "_mx_config_cache": {}, "max_speculative_num_draft_tokens": null, "last_gen_throughput": 3809.8602166968576, "memory_usage": {"weight": 42.971, "kvcache": 0, "startup_available": 14.253, "token_capacity": 25820928, "token_capacity_swa": 2582016, "graph": {"prefill": 0.0, "decode": 19.33, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 348.43866326799616, "kv_cache_allocation": 0.8657742600189522, "scheduler_e2e": 508.1397862319136, "cuda_graph": {"prefill": 0.0, "decode": 89.43686953000724, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 64}, {"model_path": "/models", "tokenizer_path": "/models", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "fp8_e4m3", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.862, "max_running_requests": 256, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 4096, "enable_dynamic_chunking": false, "prefill_reserve_budget": 0, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 0.3, "page_size": 256, "c128_page_size": 16, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": false, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": null, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 64, "pp_async_batch_depth": 0, "dp_size": 4, "load_balance_method": "round_robin", "prefix_affinity_fallback": "total_tokens", "prefix_affinity_max_load_skew": 1.5, "prefix_affinity_hash_tokens": 4096, "prefix_affinity_disable_token_fallback": false, "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": true, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 903930395, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "/models", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": false, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": false, "kv_events_config": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "dsv4", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 512, "bs": [1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 16384, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": null, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": null, "speculative_draft_model_path": null, "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": null, "speculative_eagle_topk": null, "speculative_num_draft_tokens": null, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_draft_kv_cache_dtype": null, "speculative_draft_window_size": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "deepep", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": "{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}", "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.9, "mamba_radix_cache_strategy": "auto", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": null, "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "_resolved_overrides": [["_deepseek_v4_overrides", {"attention_backend": "dsv4", "page_size": 256, "swa_full_tokens_ratio": 0.1}], ["_deepseek_v4_kv_cache_dtype", {"kv_cache_dtype": "fp8_e4m3"}], ["_sampling_backend_default", {"sampling_backend": "flashinfer"}], ["_tp_lm_head_all_to_all_default", {"enable_tp_lm_head_all_to_all": false}], ["_a2a_ep_size", {"ep_size": 4}], ["_speculative_moe_runner_default", {"speculative_moe_runner_backend": "auto"}]], "grpc_worker_threads": 4, "_quantization_explicitly_unset": false, "_cuda_graph_config_locked": [], "_declarations_materialized": true, "_mx_config_cache": {}, "max_speculative_num_draft_tokens": null, "last_gen_throughput": 3809.2647395807885, "memory_usage": {"weight": 42.971, "kvcache": 0, "startup_available": 15.19, "token_capacity": 25820928, "token_capacity_swa": 2582016, "graph": {"prefill": 0.0, "decode": 18.549, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 348.5880510379793, "kv_cache_allocation": 0.8970330059528351, "scheduler_e2e": 508.1053619449958, "cuda_graph": {"prefill": 0.0, "decode": 89.43910579301883, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 64}], "version": "0.5.18", "kv_events": null}, "duration": 319.6053244450595, "completed": 320, "total_input_tokens": 20971520, "total_input_text_tokens": 20971520, "total_input_vision_tokens": 0, "total_output_tokens": 320, "total_output_tokens_retokenized": 317, "request_throughput": 1.0012348841672953, "input_throughput": 65616.92936878787, "output_throughput": 1.0012348841672953, "total_throughput": 65617.93060367204, "mean_e2e_latency_ms": 57846.30506912799, "median_e2e_latency_ms": 63517.853102472145, "std_e2e_latency_ms": 14141.959911824717, "p90_e2e_latency_ms": 63880.27745159343, "p95_e2e_latency_ms": 64084.943865140674, "p99_e2e_latency_ms": 64357.0819141611, "mean_ttft_ms": 57407.25224736743, "median_ttft_ms": 63517.115516995545, "std_ttft_ms": 14988.300308626836, "p90_ttft_ms": 63877.81410876196, "p95_ttft_ms": 64081.24834221671, "p99_ttft_ms": 64357.0507246838, "mean_tpot_ms": 0.0, "median_tpot_ms": 0.0, "std_tpot_ms": 0.0, "p90_tpot_ms": 0.0, "p95_tpot_ms": 0.0, "p99_tpot_ms": 0.0, "mean_itl_ms": 0.0, "median_itl_ms": 0.0, "std_itl_ms": 0.0, "p90_itl_ms": 0.0, "p95_itl_ms": 0.0, "p99_itl_ms": 0.0, "concurrency": 57.91773855539439, "accept_length": null, "max_output_tokens_per_s": 5.0, "max_concurrent_requests": 68, "input_lens": [65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536, 65536], "output_lens": [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], "ttfts": [4.928420259035192, 5.165332081029192, 5.154045335017145, 8.974404316046275, 8.96500277693849, 9.205475082970224, 25.231856881990097, 28.948293809895404, 28.941542627988383, 29.18399595702067, 29.17829895892646, 32.89789493801072, 32.906659568077885, 33.13549124903511, 33.130994820035994, 36.85266923601739, 36.8482431009179, 37.061381259933114, 41.02433585806284, 44.73331565700937, 44.74121015705168, 37.03323919698596, 44.96559585200157, 44.95924109604675, 48.68740635900758, 48.68256424902938, 40.72715238702949, 48.80859575397335, 48.80445218796376, 52.53364543791395, 52.52987463097088, 52.77710331999697, 52.77325115201529, 40.58628014405258, 56.47976242203731, 56.477223708992824, 56.722768355044536, 56.71883936203085, 60.439241606974974, 60.436620173975825, 60.681998754967935, 60.6782310300041, 64.39134328695945, 40.80148278095294, 64.38545298203826, 64.63038230792154, 64.62643994402606, 8.856755592976697, 12.694432032993063, 12.691805190988816, 12.938666015979834, 0.0, 16.65888248803094, 4.505039983079769, 16.65295668190811, 16.900459229946136, 16.896443967008963, 20.620409275055863, 20.617792751058005, 20.865742383990437, 20.861778163933195, 24.594245421001688, 24.591386977001093, 24.837517041014507, 63.816133163985796, 63.80142477399204, 63.81460351601709, 63.80119810893666, 63.7015155289555, 63.688274862011895, 63.70151122903917, 63.68837338301819, 63.577554122079164, 63.564649975975044, 63.575592148001306, 63.56232353602536, 63.56742983299773, 63.55285120906774, 63.56626369105652, 63.55163973104209, 63.5585651199799, 63.54439179506153, 63.558494983939454, 63.54550663009286, 63.540635950048454, 63.527113049058244, 63.53917742497288, 63.52576608606614, 63.53476641699672, 63.52176741906442, 63.53395303199068, 63.52099928900134, 63.52935421396978, 63.51507274899632, 63.52944456203841, 63.51450890500564, 63.52648008905817, 63.512713594012894, 63.5272292989539, 63.51238940807525, 63.52525752398651, 63.51041296403855, 63.523865971015766, 63.51102477905806, 63.53524669900071, 63.519457089016214, 63.53185124602169, 63.51901206490584, 63.52784016402438, 63.51470717799384, 63.52502052905038, 63.51003668503836, 63.51819714694284, 63.505045913974755, 63.518668969976716, 63.50414535193704, 63.53822200000286, 63.52499110309873, 63.537632399005815, 63.524480439024046, 63.537269802065566, 63.52265446202364, 63.53705835796427, 63.52385826606769, 63.54932207404636, 63.53613438899629, 63.55226115009282, 63.53736115107313, 63.55571834708098, 63.5413035020465, 63.55512661894318, 63.54197898507118, 63.55106430104934, 63.53764788398985, 63.548386105918325, 63.534818893065676, 63.56055188702885, 63.546986137051135, 63.56218445894774, 63.548899284913205, 63.571796285919845, 63.55880377499852, 63.571764074033126, 63.55823278892785, 63.57723477703985, 63.56315247097518, 63.577245441963896, 63.564317018026486, 63.577768287970684, 63.562945798039436, 63.57593931199517, 63.56271668395493, 63.560378502937965, 63.547390038962476, 63.55935906700324, 63.545061929035, 63.55178082198836, 63.53864407597575, 63.55191173404455, 63.553028668044135, 63.53349907894153, 63.519003418972716, 63.532286904985085, 63.51914326904807, 63.52694086602423, 63.50600359798409, 63.527799917035736, 63.513096529990435, 63.516812021960504, 63.50211013003718, 63.51741901203059, 63.504123819991946, 63.50311865308322, 63.4897342500044, 63.501740500913, 63.48673207999673, 63.494784097070806, 63.47972970304545, 63.49263572297059, 63.479431106941774, 63.4675337059889, 63.45295481593348, 63.46810593793634, 63.45176446100231, 63.46398503391538, 63.45060718199238, 63.46381077100523, 63.4491079460131, 63.45397484092973, 63.43889521097299, 0.0, 63.43772427702788, 63.441834950004704, 63.4271540310001, 63.44011305994354, 63.42640200606547, 63.44023769488558, 63.42641228903085, 63.44006305304356, 63.426337972981855, 63.42472995398566, 63.410965798888355, 63.42364634992555, 63.41027790401131, 63.41369445901364, 63.400461590033956, 63.41171852091793, 63.41382088395767, 63.41259838605765, 63.399321928969584, 63.409850964089856, 63.396548856049776, 63.421799369039945, 63.406728968024254, 63.42210151394829, 63.42418046609964, 63.44040324690286, 63.42522204807028, 63.43962379998993, 63.42609085398726, 63.44290538993664, 63.42800303502008, 63.442256666952744, 63.427502762991935, 63.4564328690758, 63.458541256026365, 63.46130939293653, 63.44753046997357, 63.474042864050716, 63.46023807302117, 63.472048223018646, 63.458798736915924, 63.47118210699409, 63.45621929108165, 63.46947551507037, 63.45487335999496, 63.46839285898022, 63.4708600230515, 63.46967813500669, 63.45610781700816, 63.45942733704578, 63.446066214935854, 63.45963323605247, 63.44340994802769, 63.45836129307281, 63.44213038496673, 63.45724856609013, 63.442281658994034, 63.45120999903884, 63.43602210201789, 63.45206977589987, 63.437563942046836, 63.43669822602533, 63.42187818593811, 63.443238358944654, 63.42811152001377, 63.74491504090838, 63.72979047195986, 63.747449537040666, 63.73150347603951, 63.772939958027564, 63.75702227209695, 63.77448165905662, 63.75834855798166, 63.78577181289438, 63.770427916082554, 63.7871125000529, 63.77106713503599, 63.7996679689968, 63.784825994982384, 63.80316311202478, 63.78572138806339, 63.79845308500808, 63.78364052705001, 63.798602121998556, 63.782651902060024, 63.77428718097508, 63.75940753996838, 63.77477067592554, 63.75865500199143, 63.758918764069676, 63.742713206098415, 63.7604319070233, 63.74445181991905, 63.767966700019315, 63.75312002899591, 63.77059867908247, 63.7560587200569, 63.87964968394954, 63.863703200011514, 63.87761015596334, 63.86158011201769, 63.90032716700807, 63.88550540700089, 63.90745037503075, 63.89138789707795, 64.02499531803187, 64.01017862604931, 64.02741951902863, 64.01230710407253, 64.0650260290131, 64.04871847399045, 64.0666779599851, 64.05064553802367, 64.09617322601844, 64.08108654397074, 0.0, 64.08432250889018, 64.16949315997772, 64.15165626408998, 64.17145933001302, 64.15685554000083, 64.20534273597877, 64.18875942600425, 64.20652511308435, 64.19042331003584, 64.23596741701476, 64.21984994201921, 63.99505920999218, 63.980718559003435], "itls": [[], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], [], []], "generated_texts": ["\u0432", "\u73fe", " ", " ", "\u0432", " ", "\u0432", "\u0432", " ", ".\n\n", "oun", "\u0432", "ices", ".\n\n", " ", "\u0432", "\u043f", " ", "\u0435\u043d", "\u05ea", " ", "\u0627", "\u0432", " ", "rick", ".\n\n", "\u6d3e", "\u0432", " ", "\u7684", "ar", "ident", "\u0430", "\u043e\u0433", "\u0432", "m", "\u0432", "\n", "r", "\u0442", " ", " ", " ", "el", "als", "\u518c", "\u0432", "\u0432", "ry", "\u0432", " ", "", "\u0432", "ament", ".\n\n", "\u306e", "ION", " ", "hasa", "s", "U", "\u0432", ".\n\n", "(", " ", "\u043e\u0432", " ", "\u0431", "\u0434", "\u7684", " ", "\u043f", ".\n\n", " ", "g", "os", "\u0432", ".\n\n", "\u5c4a", " ", "istics", "\u5229", ".\n\n", "\u0432", "\u0432", "\u7684", " ", " ", " ", "\u0434", ".\n\n", "\u0432", " ", "\u0432", "\u0432", " ", "\u0432", " ", "\u0432", "\u062d", "\u0432", "\u0441", "ings", "View", "ul", "\u0432", "y", " ", "\u0432", " ", " ", "stand", "ident", ".\n\n", "\u043f", "e", "\u09c7", "ia", "\u0432", " ", ".\n\n", "\u09bf\u0995", "n", "\u0441\u044f", ".\n\n", "\u09c7", "\u0442", "}", "\n", " ", "ould", "\u0447\u0435\u0441\u043a\u0438\u0445", "\u043f", ".\n\n", "\u0444", "Name", "t", "s", " ", "\u0432", "\u9879", ".\n\n", " ", "\u043f", " ", " ", " ", "\u0432", ".\n\n", "\u064e", " ", ".\n\n", "\u0438", "ity", " ", ".\n\n", "quake", "\u0442", ".\n\n", "\u63a7", "\u043d\u043e", "e", "rette", "\u092a", " ", "\u0432", "\u0432", "\u0434", " ", ".\n\n", ".\n\n", "\u0447", " ", "\u043f", "\u0432", " ", "\u0432", "\u0434", "entation", " ", ".\n\n", "te", " ", "ce", "\u2588\u2588\u2588\u2588", "\u064a\u0628", "\u0442", "K", ")", "\u0432", "", " ", " ", "\u0432", "\u0432", " \u043f\u0435", "ip", ".\n\n", ".\n\n", "\u9633", ".\n\n", "\u0e38\u0e14", "e", " ", "\u5417", "\u0432", ".\n\n", " ", "\u0434", "\u0432", "\u0432", "\ub85d", " ", " ", "\u0432", "\u0440\u043e\u0432", "\u043f", " ", "\u03bd", "\u0432", " ", "\u0432", "ian", ".\n\n", "\u0432", ".\n\n", "R", ".\n\n", "or", "\uff09", ".\n\n", "\u0434", "\u884c", "\u0432", ".\n\n", "s", " ", ".\n", ".\n\n", "\u0444", " ", "\u09cd\u09af", " ", "ie", "\u0434", ".\n\n", "ida", "\u0432", "c", " ", "\u0432", " ", "\u0432", "orn", "as", " ", "\u0430", "\u0432", " ", "\u0432", "arn", " ", "ant", "\u308a", "\u7684", "\u0434", "\u0432", "ure", "\u0432", ".\n\n", ".\n\n", "\u7684", ".\n\n", " ", "\u4e0a", "s", "r", "\u043f", "\u0432", "board", ".\n\n", "ales", "\u0432", " ", "um", ".\n\n", " ", "\u7684", "\u0432", "\u52a8\u7269", "\u0432", "\u0432", " ", "\u0432", " ", "\u00f3n", "\u0432", "imum", "al", "\u0e04", "ion", "\u4e2a", "\u7684", "\u0434", "\u0432", "\u0434", "", " ", "m", "e", "s", ".\n\n", " ", "s", "\u0432", " ", ".\n\n", "ation", "\u7684", "ary"], "errors": ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""]}
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.log b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.log
index 092ac50..760681c 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.log
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/64k_1_c64.log
@@ -1,2 +1,59 @@
 /sgl-workspace/sglang/python/sglang/bench_serving.py:13: FutureWarning: `sglang.bench_serving` is deprecated and will be removed in a future release; use `sglang.benchmark.serving` instead (e.g. `python -m sglang.benchmark.serving`).
   warnings.warn(
+benchmark_args=Namespace(backend='sglang', base_url=None, host='127.0.0.1', port=30000, ready_check_timeout_sec=60, dataset_name='random-ids', dataset_path='', dataset_offset=0, agentic_max_turns=None, speed_bench_category=None, speed_bench_output_len=512, model='/models', served_model_name=None, tokenizer='/models', num_prompts=320, sharegpt_output_len=None, sharegpt_context_len=None, random_input_len=65536, random_output_len=1, random_range_ratio=1.0, image_count=1, image_resolution='1080p', random_image_count=False, image_format='jpeg', image_content='random', request_rate=inf, use_trace_timestamps=False, max_concurrency=64, output_file='/results/points/64k_1_c64.json', output_details=True, print_requests=False, disable_tqdm=True, disable_stream=False, return_logprob=False, top_logprobs_num=0, token_ids_logprob=None, logprob_start_len=-1, return_routed_experts=False, cache_report=False, seed=20260904, disable_ignore_eos=False, temperature=0.0, top_p=1.0, extra_request_body=None, apply_chat_template=False, profile=False, plot_throughput=False, profile_activities=['CPU', 'GPU'], profile_start_step=None, profile_steps=None, profile_num_steps=None, profile_by_stage=False, profile_stages=None, profile_output_dir=None, profile_prefix=None, lora_name=None, lora_request_distribution='uniform', lora_zipf_alpha=1.5, prompt_suffix='', pd_separated=False, profile_prefill_url=None, profile_decode_url=None, flush_cache=True, flush_cache_timeout=60.0, warmup_requests=1, tokenize_prompt=True, gsp_num_groups=64, gsp_prompts_per_group=16, gsp_system_prompt_len=2048, gsp_question_len=128, gsp_output_len=256, gsp_range_ratio=1.0, gsp_fast_prepare=False, gsp_send_routing_key=False, gsp_num_turns=1, gsp_ordered=False, gsp_group_distribution='uniform', gsp_zipf_alpha=None, mooncake_slowdown_factor=1.0, mooncake_num_rounds=1, mooncake_workload='conversation', fake_prefill=False, tag=None, header=None)
+Waiting up to 60s for http://127.0.0.1:30000/v1/models to become ready...
+Server ready in 0.0s.
+
+WARNING It is recommended to use the `Chat` or `Instruct` model for benchmarking.
+Because when the tokenizer counts the output tokens, if there is gibberish, it might count incorrectly.
+
+Namespace(backend='sglang', base_url=None, host='127.0.0.1', port=30000, ready_check_timeout_sec=60, dataset_name='random-ids', dataset_path='', dataset_offset=0, agentic_max_turns=None, speed_bench_category=None, speed_bench_output_len=512, model='/models', served_model_name=None, tokenizer='/models', num_prompts=320, sharegpt_output_len=None, sharegpt_context_len=None, random_input_len=65536, random_output_len=1, random_range_ratio=1.0, image_count=1, image_resolution='1080p', random_image_count=False, image_format='jpeg', image_content='random', request_rate=inf, use_trace_timestamps=False, max_concurrency=64, output_file='/results/points/64k_1_c64.json', output_details=True, print_requests=False, disable_tqdm=True, disable_stream=False, return_logprob=False, top_logprobs_num=0, token_ids_logprob=None, logprob_start_len=-1, return_routed_experts=False, cache_report=False, seed=20260904, disable_ignore_eos=False, temperature=0.0, top_p=1.0, extra_request_body=None, apply_chat_template=False, profile=False, plot_throughput=False, profile_activities=['CPU', 'GPU'], profile_start_step=None, profile_steps=None, profile_num_steps=None, profile_by_stage=False, profile_stages=None, profile_output_dir=None, profile_prefix=None, lora_name=None, lora_request_distribution='uniform', lora_zipf_alpha=1.5, prompt_suffix='', pd_separated=False, profile_prefill_url=None, profile_decode_url=None, flush_cache=True, flush_cache_timeout=60.0, warmup_requests=1, tokenize_prompt=True, gsp_num_groups=64, gsp_prompts_per_group=16, gsp_system_prompt_len=2048, gsp_question_len=128, gsp_output_len=256, gsp_range_ratio=1.0, gsp_fast_prepare=False, gsp_send_routing_key=False, gsp_num_turns=1, gsp_ordered=False, gsp_group_distribution='uniform', gsp_zipf_alpha=None, mooncake_slowdown_factor=1.0, mooncake_num_rounds=1, mooncake_workload='conversation', fake_prefill=False, tag=None, header=None)
+
+#Input tokens: 20971520
+#Output tokens: 320
+Starting warmup with 1 sequences...
+Warmup completed with 1 sequences. Starting main benchmark run...
+
+============ Serving Benchmark Result ============
+Backend:                                 sglang    
+Traffic request rate:                    inf       
+Max request concurrency:                 64        
+Successful requests:                     320       
+Benchmark duration (s):                  319.61    
+Total input tokens:                      20971520  
+Total input text tokens:                 20971520  
+Total generated tokens:                  320       
+Total generated tokens (retokenized):    317       
+Request throughput (req/s):              1.00      
+Input token throughput (tok/s):          65616.93  
+Output token throughput (tok/s):         1.00      
+Peak output token throughput (tok/s):    5.00      
+Peak concurrent requests:                68        
+Total token throughput (tok/s):          65617.93  
+Concurrency:                             57.92     
+----------------End-to-End Latency----------------
+Mean E2E Latency (ms):                   57846.31  
+Median E2E Latency (ms):                 63517.85  
+P90 E2E Latency (ms):                    63880.28  
+P95 E2E Latency (ms):                    64084.94  
+P99 E2E Latency (ms):                    64357.08  
+---------------Time to First Token----------------
+Mean TTFT (ms):                          57407.25  
+Median TTFT (ms):                        63517.12  
+P90 TTFT (ms):                           63877.81  
+P95 TTFT (ms):                           64081.25  
+P99 TTFT (ms):                           64357.05  
+-----Time per Output Token (excl. 1st token)------
+Mean TPOT (ms):                          0.00      
+Median TPOT (ms):                        0.00      
+P90 TPOT (ms):                           0.00      
+P95 TPOT (ms):                           0.00      
+P99 TPOT (ms):                           0.00      
+---------------Inter-Token Latency----------------
+Mean ITL (ms):                           0.00      
+Median ITL (ms):                         0.00      
+P90 ITL (ms):                            0.00      
+P95 ITL (ms):                            0.00      
+P99 ITL (ms):                            0.00      
+Max ITL (ms):                            0.00      
+==================================================
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.final.log b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.final.log
index 101a6ee..3304880 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.final.log
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.final.log
@@ -1,642 +1 @@
-
-==========
-== CUDA ==
-==========
-
-CUDA Version 13.0.3
-
-Container image Copyright (c) 2016-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-
-This container image and its contents are governed by the NVIDIA Deep Learning Container License.
-By pulling and using the container, you accept the terms and conditions of this license:
-https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license
-
-A copy of this license is made available in this container at /NGC-DL-CONTAINER-LICENSE for your convenience.
-
-[2026-09-09 09:05:21] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:21] Hybrid SWA model detected. architectures=['DeepseekV4ForCausalLM']
-[2026-09-09 09:05:21] Breakable CUDA graph is incompatible with DeepSeek-V4 (heavy capture-pool memory pressure); disabling prefill CUDA graph.
-[2026-09-09 09:05:22] Use dsv4 attention backend for DeepseekV4ForCausalLM, setting page_size to 256.
-[2026-09-09 09:05:22] Setting swa_full_tokens_ratio to 0.1 for DeepseekV4ForCausalLM.
-[2026-09-09 09:05:22] Setting KV cache dtype to fp8_e4m3 for DeepseekV4ForCausalLM.
-[2026-09-09 09:05:22] Setting max_running_requests to 256 for DeepseekV4ForCausalLM.
-[2026-09-09 09:05:22] DP attention is enabled. chunked prefill size is adjusted from 16384 to 4096.
-[2026-09-09 09:05:22] deepep MoE is enabled. The expert parallel size is adjusted from 1 to the tensor parallel size [4].
-[2026-09-09 09:05:23] server_args=ServerArgs(model_path='/models', tokenizer_path='/models', tokenizer_mode='auto', tokenizer_backend='huggingface', tokenizer_worker_num=1, detokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=True, context_length=None, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', model_config_parser='auto', json_model_override_args='{}', dtype='auto', quantization=None, quantization_param_path=None, kv_cache_dtype='fp8_e4m3', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, rl_quant_profile=None, enable_tf32_matmul=False, mem_fraction_static=0.862, max_running_requests=256, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=4096, enable_dynamic_chunking=False, prefill_reserve_budget=0, max_prefill_tokens=16384, prefill_max_requests=None, schedule_policy='fcfs', enable_priority_scheduling=False, disable_priority_preemption=False, default_priority_value=None, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, retraction_policy='length', schedule_conservativeness=0.3, page_size=256, c128_page_size=16, swa_full_tokens_ratio=0.1, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', prefill_only_disable_kv_cache=False, disable_radix_cache=False, enable_page_major_kv_layout=False, enable_unified_memory=False, disable_chunked_prefix_cache=False, disable_overlap_schedule=False, num_continuous_decode_steps=1, scheduler_recv_interval=1, enable_mixed_chunk=False, nccl_port=None, dist_timeout=None, dist_init_addr=None, nnodes=1, node_rank=0, tp_size=4, dcp_size=1, pp_size=1, pp_max_micro_batch_size=None, pp_async_batch_depth=0, dp_size=4, load_balance_method='round_robin', prefix_affinity_fallback='total_tokens', prefix_affinity_max_load_skew=1.5, prefix_affinity_hash_tokens=4096, prefix_affinity_disable_token_fallback=False, attn_cp_size=1, moe_dp_size=1, dwdp_size=1, dcp_comm_backend='ag_rs', dcp_replicate_q_proj=None, enable_prefill_cp=False, cp_strategy=None, enable_dsa_cache_layer_split=False, enable_dsa_prefill_context_parallel=False, dsa_prefill_cp_mode='round-robin-split', enable_prefill_context_parallel=False, prefill_cp_mode='in-seq-split', enable_cp_decode_attn_tp=False, enable_dp_attention=True, enable_dp_attention_local_control_broadcast=False, enable_dp_lm_head=False, enable_tp_lm_head_all_to_all=False, enable_attn_tp_input_scattered=False, disable_attn_tp_gather=False, enable_p2p_check=False, device='cuda', base_gpu_id=0, gpu_id_step=1, random_seed=1071034650, mlx_enable_sampling=False, watchdog_timeout=300, soft_watchdog_timeout=None, sleep_on_idle=False, use_ray=False, custom_sigquit_handler=None, numa_node=None, gc_threshold=None, host='0.0.0.0', port=30000, fastapi_root_path='', smg_grpc_mode=False, grpc_mode=False, grpc_port=None, sidecar=None, sidecar_args=None, skip_server_warmup=False, warmups=None, enable_http2=False, http2_max_concurrent_streams=200, ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_keyfile_password=None, enable_ssl_refresh=False, api_key=None, admin_api_key=None, served_model_name='/models', weight_version='default', chat_template=None, hf_chat_template_name=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser='deepseek-v4', default_chat_template_kwargs=None, strip_thinking_cache=False, enable_strict_thinking=False, tool_call_parser='deepseekv4', tool_server=None, sampling_defaults='model', asr_max_buffer_seconds=60, asr_max_concurrent_sessions=32, preferred_sampling_params=None, allow_auto_truncate=False, stream_interval=1, batch_notify_size=16, stream_response_default_include_usage=False, incremental_streaming_output=False, enable_streaming_session=False, enable_session_radix_cache=False, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, log_requests_format='text', log_requests_target=None, uvicorn_access_log_exclude_prefixes=[], crash_dump_folder=None, show_time_cost=False, enable_metrics=False, smg_http_sidecar_port=None, enable_mfu_metrics=False, enable_metrics_for_all_schedulers=False, load_snapshot_publish_interval=15, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, extra_metric_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_forward_pass_metrics=False, forward_pass_metrics_worker_id='', forward_pass_metrics_ipc_name=None, enable_trace=False, trace_modules='request', otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, stat_loggers=None, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, attention_backend='dsv4', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', radix_cache_backend=None, mm_attention_backend=None, fp8_gemm_runner_backend='auto', fp4_gemm_runner_backend='auto', bf16_gemm_backend='auto', dsa_prefill_backend=None, dsa_decode_backend=None, dsa_paged_mqa_logits_backend='auto', dsa_topk_backend='sgl-kernel', disable_flashinfer_autotune=False, flashinfer_autotune_skip_ops=None, mamba_backend='triton', cuda_graph_config=CudaGraphConfig(decode=PhaseConfig(backend='full', max_bs=512, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], tc_compiler='eager', full_prefill_max_req=None, full_prefill_prefix_chunk_tokens=None), prefill=PhaseConfig(backend='disabled', max_bs=16384, bs=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], tc_compiler='eager', full_prefill_max_req=None, full_prefill_prefix_chunk_tokens=None)), cuda_graph_backend_decode=None, cuda_graph_backend_prefill=None, cuda_graph_max_bs_decode=None, cuda_graph_max_bs_prefill=None, cuda_graph_bs_decode=None, cuda_graph_bs_prefill=None, cuda_graph_tc_compiler=None, disable_prefill_cuda_graph=False, disable_decode_cuda_graph=False, disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, debug_cuda_graph=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, flashinfer_mla_disable_ragged=False, enable_fused_qk_norm_rope=False, enable_precise_embedding_interpolation=False, enable_fused_moe_sum_all_reduce=False, enable_deepseek_v4_fp4_indexer=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, enable_scattered_sconv=False, pre_warm_nccl=False, enable_quant_communications=False, enable_flashinfer_allreduce_fusion=False, enforce_disable_flashinfer_allreduce_fusion=False, flashinfer_allreduce_fusion_backend=None, enable_aiter_allreduce_fusion=False, enable_torch_compile=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_dflash_block_size=None, speculative_dspark_block_size=None, speculative_dspark_sps_table_path=None, speculative_dspark_confidence_sts_path=None, speculative_dspark_align_verify_tokens_to_graph_tier=False, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_use_rejection_sampling=False, speculative_token_map=None, speculative_attention_mode='prefill', speculative_draft_attention_backend=None, speculative_draft_kv_cache_dtype=None, speculative_draft_window_size=None, speculative_moe_runner_backend='auto', speculative_moe_a2a_backend=None, speculative_draft_model_quantization=None, _speculative_draft_quantization_explicitly_set=False, speculative_skip_dp_mlp_sync=False, enable_multi_layer_eagle=False, speculative_adaptive=False, speculative_adaptive_config=None, decoupled_spec_bind_endpoint=None, decoupled_spec_connect_endpoints=None, decoupled_spec_rank=None, decoupled_spec_role='null', spec_trace_dir=None, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_max_trie_depth=18, speculative_ngram_capacity=10000000, speculative_ngram_external_corpus_path=None, speculative_ngram_external_sam_budget=0, speculative_ngram_external_corpus_max_tokens=10000000, ep_size=4, moe_a2a_backend='deepep', moe_runner_backend='auto', flashinfer_mxfp4_moe_precision='default', deepep_mode='auto', fuseep_mode=2, deepep_dispatcher_output_dtype='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, expert_balancedness_report_mode='off', deepep_config='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}', moe_dense_tp_size=None, elastic_ep_backend=None, enable_elastic_expert_backup=False, mooncake_ib_device=None, enable_waterfill=False, ep_join_mode=None, ep_join_rank_offset=0, elastic_ep_initial_size=None, max_ep_size=None, elastic_ep_scale_timeout=600, elastic_ep_rejoin=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, disable_shared_experts_fusion=False, enforce_shared_experts_fusion=False, max_mamba_cache_size=None, mamba_ssm_dtype=None, mamba_max_states_per_path=-1, enable_mamba_cache_stochastic_rounding=False, mamba_cache_philox_rounds=0, mamba_full_memory_ratio=0.9, mamba_radix_cache_strategy='auto', uses_mamba_radix_cache=False, mamba_track_interval=256, enable_int8_mamba_checkpoint=False, int8_mamba_ckpt_size=None, linear_attn_backend='triton', linear_attn_decode_backend=None, linear_attn_prefill_backend=None, linear_attn_verify_backend=None, enable_linear_replayssm=False, linear_replayssm_cache_len=16, enable_linear_replayssm_spec=False, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='page_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='timeout', hicache_storage_backend_extra_config=None, enable_hisparse=False, hisparse_config=None, enable_broadcast_mm_inputs_process=False, enable_prefix_mm_cache=False, mm_enable_dp_encoder=False, mm_process_config={}, mm_processor_worker_num=0, mm_io_worker_num=0, allowed_media_domains=[], media_url_max_file_size_mb=64, mm_preprocess_cache_size_mb=None, trust_mm_content_hashes=False, limit_mm_data_per_request=None, enable_mm_global_cache=False, image_processor_backend='auto', mm_global_cache_backend='mooncake', disable_fast_image_processor=False, mm_feature_transport='cpu', keep_mm_feature_on_device=False, enable_lora=None, enable_lora_overlap_loading=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, experts_shared_outer_loras=None, lora_use_virtual_experts=False, lora_strict_loading=False, lora_drain_wait_threshold=0.0, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', enable_lmcache=False, lmcache_config_file=None, enable_flexkv=False, flexkv_config_file=None, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, dllm_fdfo=True, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_ib_device=None, disaggregation_decode_enable_radix_cache=False, disaggregation_decode_enable_offload_kvcache=False, disaggregation_decode_retraction_backup=None, num_reserved_decode_tokens=512, disaggregation_decode_extra_slots=None, disaggregation_decode_polling_interval=1, optimistic_prefill_attempts=0, encoder_only=False, language_only=False, language_model_only=False, encoder_transfer_backend='zmq_to_scheduler', encoder_urls=[], encoder_bootstrap_port=8997, encoder_register_urls=[], enable_adaptive_dispatch_to_encoder=False, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, startup_weight_load_mode='serial', custom_weight_loader=[], weight_loader_disable_mmap=False, weight_loader_prefetch_checkpoints=False, weight_loader_prefetch_num_threads=4, weight_loader_drop_cache_after_load=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, remote_instance_weight_loader_backend='nccl', remote_instance_weight_loader_start_seed_via_transfer_engine=False, engine_info_bootstrap_port=6789, modelexpress_config=None, download_dir=None, model_checksum=None, delete_ckpt_after_loading=False, decrypted_config_file=None, decrypted_draft_config_file=None, checkpoint_engine_wait_weights_before_ready=False, enable_prefill_delayer=False, prefill_delayer_max_delay_passes=30, prefill_delayer_token_usage_low_watermark=None, prefill_delayer_forward_passes_buckets=None, prefill_delayer_wait_seconds_buckets=None, prefill_delayer_queue_min_ratio=None, prefill_delayer_max_delay_ms=None, min_free_slots_delay=None, enable_deterministic_inference=False, rl_on_policy_target=None, kv_canary='none', kv_canary_real_data='none', kv_canary_sweep_interval=0, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, enable_custom_logit_processor=False, enable_return_hidden_states=False, return_hidden_states_mode=None, enable_return_routed_experts=False, enable_return_indexer_topk=False, disable_outlines_disk_cache=False, enable_mis=False, weight_cache_mode='off', weight_cache_socket=None, weight_cache_timeout=1800, forward_hooks=None, msprobe_dump_config=None)
-[2026-09-09 09:05:23] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:26] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:05:26] No HuggingFace chat template found
-[2026-09-09 09:05:26] No chat template found, defaulting to 'string' content format
-[2026-09-09 09:05:32] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command.
-[2026-09-09 09:05:32] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command.
-[2026-09-09 09:05:32] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command.
-[2026-09-09 09:05:32] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command.
-[2026-09-09 09:05:34] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:05:42 DP2 TP2 EP2] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:42 DP1 TP1 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:43 DP0 TP0 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:43 DP3 TP3 EP3] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:44 DP2 TP2 EP2] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:05:45 DP1 TP1 EP1] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:05:45 DP0 TP0 EP0] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:05:45 DP2 TP2 EP2] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:45 DP1 TP1 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:45 DP0 TP0 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:45 DP3 TP3 EP3] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:05:45 DP3 TP3 EP3] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-09 09:05:47 DP1 TP1 EP1] Init torch distributed begin.
-[2026-09-09 09:05:47 DP2 TP2 EP2] Init torch distributed begin.
-[2026-09-09 09:05:47 DP0 TP0 EP0] Init torch distributed begin.
-[2026-09-09 09:05:47 DP3 TP3 EP3] Init torch distributed begin.
-[W909 09:06:07.339724732 socket.cpp:207] [c10d] The hostname of the client socket cannot be retrieved. err=-3
-[W909 09:06:07.339722351 socket.cpp:207] [c10d] The hostname of the client socket cannot be retrieved. err=-3
-[W909 09:06:07.339926909 socket.cpp:207] [c10d] The hostname of the client socket cannot be retrieved. err=-3
-[W909 09:06:07.527388443 socket.cpp:207] [c10d] The hostname of the client socket cannot be retrieved. err=-3
-[2026-09-09 09:06:07 DP0 TP0 EP0] sglang is using nccl==2.29.7
-[2026-09-09 09:06:39 DP0 TP0 EP0] All Reduce config: symmetric_memory = 40.01 MB, local_buffer = 4.00 MB, multicast = False
-[2026-09-09 09:06:46 DP0 TP0 EP0] Init torch distributed ends. elapsed=59.65 s, mem usage=1.40 GB
-[2026-09-09 09:06:46 DP3 TP3 EP3] Init torch distributed ends. elapsed=59.46 s, mem usage=1.28 GB
-[2026-09-09 09:06:46 DP2 TP2 EP2] Init torch distributed ends. elapsed=59.67 s, mem usage=1.44 GB
-[2026-09-09 09:06:46 DP1 TP1 EP1] Init torch distributed ends. elapsed=59.68 s, mem usage=1.44 GB
-[2026-09-09 09:06:50 DP3 TP3 EP3] Ignore import error when loading sglang.srt.models.sarashina2_vision: cannot import name 'MultimodalDataItem' from 'sglang.srt.managers.mm_utils' (/sgl-workspace/sglang/python/sglang/srt/managers/mm_utils.py)
-[2026-09-09 09:06:50 DP1 TP1 EP1] Ignore import error when loading sglang.srt.models.sarashina2_vision: cannot import name 'MultimodalDataItem' from 'sglang.srt.managers.mm_utils' (/sgl-workspace/sglang/python/sglang/srt/managers/mm_utils.py)
-[2026-09-09 09:06:50 DP2 TP2 EP2] Ignore import error when loading sglang.srt.models.sarashina2_vision: cannot import name 'MultimodalDataItem' from 'sglang.srt.managers.mm_utils' (/sgl-workspace/sglang/python/sglang/srt/managers/mm_utils.py)
-[2026-09-09 09:06:50 DP0 TP0 EP0] Ignore import error when loading sglang.srt.models.sarashina2_vision: cannot import name 'MultimodalDataItem' from 'sglang.srt.managers.mm_utils' (/sgl-workspace/sglang/python/sglang/srt/managers/mm_utils.py)
-[2026-09-09 09:06:54 DP0 TP0 EP0] Load weight begin. avail mem=265.62 GB
-[2026-09-09 09:06:54 DP0 TP0 EP0] Detected fp8 checkpoint.
-[2026-09-09 09:06:54 DP0 TP0 EP0] Config does not support fused shared expert(s). Shared experts fusion optimization is disabled.
-[2026-09-09 09:06:54 DP2 TP2 EP2] Load weight begin. avail mem=265.58 GB
-[2026-09-09 09:06:54 DP1 TP1 EP1] Load weight begin. avail mem=265.58 GB
-[2026-09-09 09:06:54 DP3 TP3 EP3] Load weight begin. avail mem=265.74 GB
-[2026-09-09 09:07:01 DP3 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=auto, quant_method=Fp8MoEMethod).
-[2026-09-09 09:07:01 DP0 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=auto, quant_method=Fp8MoEMethod).
-[2026-09-09 09:07:01 DP1 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=auto, quant_method=Fp8MoEMethod).
-[2026-09-09 09:07:01 DP2 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=auto, quant_method=Fp8MoEMethod).
-
Multi-thread loading shards:   0% Completed | 0/48 [00:00<?, ?it/s]
Multi-thread loading shards:   4% Completed | 2/48 [00:08<03:05,  4.03s/it]
Multi-thread loading shards:   6% Completed | 3/48 [00:09<02:07,  2.82s/it]
Multi-thread loading shards:   8% Completed | 4/48 [00:09<01:22,  1.87s/it]
Multi-thread loading shards:  10% Completed | 5/48 [00:10<01:04,  1.49s/it]
Multi-thread loading shards:  12% Completed | 6/48 [00:10<00:49,  1.18s/it]
Multi-thread loading shards:  15% Completed | 7/48 [00:11<00:45,  1.11s/it]
Multi-thread loading shards:  17% Completed | 8/48 [00:12<00:43,  1.10s/it]
Multi-thread loading shards:  19% Completed | 9/48 [00:13<00:37,  1.04it/s]
Multi-thread loading shards:  21% Completed | 10/48 [00:14<00:38,  1.00s/it]
Multi-thread loading shards:  23% Completed | 11/48 [00:15<00:34,  1.06it/s]
Multi-thread loading shards:  25% Completed | 12/48 [00:22<01:41,  2.81s/it]
Multi-thread loading shards:  27% Completed | 13/48 [00:24<01:30,  2.59s/it]
Multi-thread loading shards:  29% Completed | 14/48 [00:25<01:10,  2.09s/it]
Multi-thread loading shards:  31% Completed | 15/48 [00:26<00:57,  1.74s/it]
Multi-thread loading shards:  33% Completed | 16/48 [00:27<00:45,  1.42s/it]
Multi-thread loading shards:  35% Completed | 17/48 [00:28<00:40,  1.30s/it]
Multi-thread loading shards:  38% Completed | 18/48 [00:28<00:33,  1.11s/it]
Multi-thread loading shards:  40% Completed | 19/48 [00:29<00:31,  1.09s/it]
Multi-thread loading shards:  42% Completed | 20/48 [00:35<01:07,  2.43s/it]
Multi-thread loading shards:  44% Completed | 21/48 [00:36<00:53,  1.98s/it]
Multi-thread loading shards:  46% Completed | 22/48 [00:39<01:01,  2.38s/it]
Multi-thread loading shards:  48% Completed | 23/48 [00:42<01:02,  2.49s/it]
Multi-thread loading shards:  50% Completed | 24/48 [00:43<00:52,  2.17s/it]
Multi-thread loading shards:  52% Completed | 25/48 [00:45<00:47,  2.07s/it]
Multi-thread loading shards:  54% Completed | 26/48 [00:47<00:41,  1.88s/it]
Multi-thread loading shards:  56% Completed | 27/48 [00:48<00:37,  1.81s/it]
Multi-thread loading shards:  58% Completed | 28/48 [00:50<00:34,  1.72s/it]
Multi-thread loading shards:  60% Completed | 29/48 [00:52<00:34,  1.84s/it]
Multi-thread loading shards:  62% Completed | 30/48 [00:53<00:31,  1.73s/it]
Multi-thread loading shards:  65% Completed | 31/48 [00:55<00:28,  1.69s/it]
Multi-thread loading shards:  67% Completed | 32/48 [00:57<00:29,  1.82s/it]
Multi-thread loading shards:  69% Completed | 33/48 [00:58<00:24,  1.61s/it]
Multi-thread loading shards:  71% Completed | 34/48 [00:59<00:20,  1.46s/it]
Multi-thread loading shards:  73% Completed | 35/48 [01:00<00:16,  1.29s/it]
Multi-thread loading shards:  75% Completed | 36/48 [01:01<00:13,  1.17s/it]
Multi-thread loading shards:  77% Completed | 37/48 [01:02<00:12,  1.12s/it]
Multi-thread loading shards:  79% Completed | 38/48 [01:02<00:08,  1.17it/s]
Multi-thread loading shards:  81% Completed | 39/48 [01:03<00:07,  1.25it/s]
Multi-thread loading shards:  83% Completed | 40/48 [01:04<00:05,  1.35it/s]
Multi-thread loading shards:  85% Completed | 41/48 [01:04<00:05,  1.24it/s]
Multi-thread loading shards:  88% Completed | 42/48 [01:05<00:04,  1.35it/s]
Multi-thread loading shards:  90% Completed | 43/48 [01:06<00:03,  1.28it/s]
Multi-thread loading shards:  92% Completed | 44/48 [01:07<00:03,  1.22it/s][2026-09-09 09:08:14 DP3 TP3 EP3] DeepSeek V4 MHC prenorm prewarm: 21 n_splits buckets
-[2026-09-09 09:08:15 DP2 TP2 EP2] DeepSeek V4 MHC prenorm prewarm: 21 n_splits buckets
-
Multi-thread loading shards:  96% Completed | 46/48 [01:11<00:02,  1.38s/it]
Multi-thread loading shards: 100% Completed | 48/48 [01:11<00:00,  1.49s/it]
-[2026-09-09 09:08:16 DP0 TP0 EP0] DeepSeek V4 MHC prenorm prewarm: 21 n_splits buckets
-[2026-09-09 09:08:18 DP1 TP1 EP1] DeepSeek V4 MHC prenorm prewarm: 21 n_splits buckets
-[2026-09-09 09:13:30 DP3 TP3 EP3] DeepSeek V4 MHC prewarm at load: compile 315.2s, rank sync +0.2s
-[2026-09-09 09:13:30 DP0 TP0 EP0] DeepSeek V4 MHC prewarm at load: compile 312.8s, rank sync +0.4s
-[2026-09-09 09:13:30 DP2 TP2 EP2] DeepSeek V4 MHC prewarm at load: compile 314.4s, rank sync +0.1s
-[2026-09-09 09:13:30 DP1 TP1 EP1] DeepSeek V4 MHC prewarm at load: compile 312.1s, rank sync +0.0s
-[2026-09-09 09:13:36 DP0 TP0 EP0] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results!
-[2026-09-09 09:13:36 DP3 TP3 EP3] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results!
-[2026-09-09 09:13:36 DP3 TP3 EP3] Load weight end. elapsed=402.72 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=222.77 GB, mem usage=42.97 GB.
-[2026-09-09 09:13:36 DP0 TP0 EP0] Load weight end. elapsed=402.88 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=222.65 GB, mem usage=42.97 GB.
-[2026-09-09 09:13:36 DP1 TP1 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results!
-[2026-09-09 09:13:36 DP1 TP1 EP1] Load weight end. elapsed=402.88 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=222.61 GB, mem usage=42.97 GB.
-[2026-09-09 09:13:37 DP2 TP2 EP2] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results!
-[2026-09-09 09:13:37 DP2 TP2 EP2] Load weight end. elapsed=403.02 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=222.61 GB, mem usage=42.97 GB.
-[2026-09-09 09:13:39 DP2 TP2 EP2] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:13:39 DP0 TP0 EP0] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:13:39 DP3 TP3 EP3] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:13:39 DP1 TP1 EP1] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-09 09:13:39 DP0 TP0 EP0] DSV4 memory calculation: bytes_per_full_token=7705.45, available_bytes=185.95 GB, c128_state_fixed=0.65 GB, full_token=25820928
-[2026-09-09 09:13:39 DP0 TP0 EP0] DSV4 pool sizes: full=25820928, swa=2582016, c4=6455232, c128=201726, c4_state=161376, c128_state=0
-[2026-09-09 09:13:39 DP3 TP3 EP3] DSV4 memory calculation: bytes_per_full_token=7705.45, available_bytes=185.95 GB, c128_state_fixed=0.65 GB, full_token=25820928
-[2026-09-09 09:13:39 DP3 TP3 EP3] DSV4 pool sizes: full=25820928, swa=2582016, c4=6455232, c128=201726, c4_state=161376, c128_state=0
-[2026-09-09 09:13:39 DP2 TP2 EP2] DSV4 memory calculation: bytes_per_full_token=7705.45, available_bytes=185.95 GB, c128_state_fixed=0.65 GB, full_token=25820928
-[2026-09-09 09:13:39 DP2 TP2 EP2] DSV4 pool sizes: full=25820928, swa=2582016, c4=6455232, c128=201726, c4_state=161376, c128_state=0
-[2026-09-09 09:13:39 DP0 TP0 EP0] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=64 swa_size=2582016 c4_size=6455232 c4_logical_size=6455232 c128_size=201726 c4_state_pool_size=161376 c128_state_pool_size=8320
-[2026-09-09 09:13:39 DP3 TP3 EP3] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=64 swa_size=2582016 c4_size=6455232 c4_logical_size=6455232 c128_size=201726 c4_state_pool_size=161376 c128_state_pool_size=8320
-[2026-09-09 09:13:39 DP1 TP1 EP1] DSV4 memory calculation: bytes_per_full_token=7705.45, available_bytes=185.95 GB, c128_state_fixed=0.65 GB, full_token=25820928
-[2026-09-09 09:13:39 DP1 TP1 EP1] DSV4 pool sizes: full=25820928, swa=2582016, c4=6455232, c128=201726, c4_state=161376, c128_state=0
-[2026-09-09 09:13:39 DP2 TP2 EP2] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=64 swa_size=2582016 c4_size=6455232 c4_logical_size=6455232 c128_size=201726 c4_state_pool_size=161376 c128_state_pool_size=8320
-[2026-09-09 09:13:39 DP1 TP1 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=64 swa_size=2582016 c4_size=6455232 c4_logical_size=6455232 c128_size=201726 c4_state_pool_size=161376 c128_state_pool_size=8320
-[2026-09-09 09:13:40 DP2 TP2 EP2] Memory pool end. avail mem=34.80 GB
-[2026-09-09 09:13:40 DP1 TP1 EP1] Memory pool end. avail mem=34.80 GB
-[2026-09-09 09:13:40 DP3 TP3 EP3] Memory pool end. avail mem=34.96 GB
-[2026-09-09 09:13:40 DP0 TP0 EP0] Memory pool end. avail mem=34.83 GB
-[2026-09-09 09:13:50 DP3 TP3 EP3] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA).
-[2026-09-09 09:13:50 DP0 TP0 EP0] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA).
-[2026-09-09 09:13:50 DP1 TP1 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA).
-[2026-09-09 09:13:50 DP2 TP2 EP2] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA).
-[2026-09-09 09:13:50 DP0 TP0 EP0] DeepGEMM masked layout budget: 8.40 GiB from 33.58 GiB free.
-[2026-09-09 09:13:50 DP0 TP0 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules).
-[2026-09-09 09:13:50 DP0 TP0 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=33.61 GB
-[2026-09-09 09:13:50 DP3 TP3 EP3] DeepGEMM masked layout budget: 8.40 GiB from 33.58 GiB free.
-[2026-09-09 09:13:50 DP3 TP3 EP3] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules).
-[2026-09-09 09:13:50 DP2 TP2 EP2] DeepGEMM masked layout budget: 8.40 GiB from 33.58 GiB free.
-[2026-09-09 09:13:50 DP2 TP2 EP2] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules).
-[2026-09-09 09:13:50 DP2 TP2 EP2] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=33.58 GB
-[2026-09-09 09:13:50 DP1 TP1 EP1] DeepGEMM masked layout budget: 8.40 GiB from 33.58 GiB free.
-[2026-09-09 09:13:50 DP3 TP3 EP3] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=33.74 GB
-[2026-09-09 09:13:50 DP1 TP1 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules).
-[2026-09-09 09:13:50 DP1 TP1 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=33.58 GB
-
  0%|          | 0/12 [00:00<?, ?it/s]
Capturing batches (bs=64 avail_mem=33.58 GB):   0%|          | 0/12 [00:00<?, ?it/s][2026-09-09 09:14:20 DP0 TP0 EP0] Disable CP decode attention TP
-[2026-09-09 09:14:20 DP3 TP3 EP3] Disable CP decode attention TP
-[2026-09-09 09:14:21 DP2 TP2 EP2] Disable CP decode attention TP
-[2026-09-09 09:14:21 DP1 TP1 EP1] Disable CP decode attention TP
-[2026-09-09 09:14:26 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:14:26 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=1536, K=4096, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:14:26 DP0 TP0 EP0] Required memory for warmup: 0.115234375GB, Available memory: 33.5611572265625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   0%|          | 1/16384 [00:02<10:51:28,  2.39s/it]
-
DeepGEMM warmup:   1%|          | 193/16384 [00:04<05:46, 46.76it/s] 
-
DeepGEMM warmup:   2%|▏         | 257/16384 [00:07<07:07, 37.71it/s]
-
DeepGEMM warmup:   2%|▏         | 385/16384 [00:09<06:07, 43.50it/s]
-
DeepGEMM warmup:   4%|▎         | 577/16384 [00:12<04:42, 55.98it/s]
-
DeepGEMM warmup:   4%|▍         | 641/16384 [00:14<05:43, 45.80it/s]
-
DeepGEMM warmup:   5%|▍         | 769/16384 [00:17<05:26, 47.85it/s]
-
DeepGEMM warmup:   6%|▌         | 961/16384 [00:19<04:29, 57.24it/s]
-
DeepGEMM warmup:   7%|▋         | 1153/16384 [00:22<04:00, 63.26it/s]
-
DeepGEMM warmup:   8%|▊         | 1345/16384 [00:24<03:42, 67.51it/s]
-
DeepGEMM warmup:   9%|▊         | 1409/16384 [00:27<04:31, 55.07it/s]
-
DeepGEMM warmup:   9%|▉         | 1537/16384 [00:29<04:32, 54.40it/s]
-
DeepGEMM warmup:  11%|█         | 1729/16384 [00:31<03:59, 61.15it/s]
-
DeepGEMM warmup:  11%|█         | 1761/16384 [00:34<05:14, 46.56it/s]
-
DeepGEMM warmup:  12%|█▏        | 1921/16384 [00:36<04:33, 52.82it/s]
-
DeepGEMM warmup:  13%|█▎        | 2113/16384 [00:39<03:56, 60.46it/s]
-
DeepGEMM warmup:  13%|█▎        | 2177/16384 [00:41<04:39, 50.89it/s]
-
DeepGEMM warmup:  14%|█▍        | 2305/16384 [00:43<04:32, 51.61it/s]
-
DeepGEMM warmup:  15%|█▌        | 2497/16384 [00:46<03:53, 59.36it/s]
-
DeepGEMM warmup:  16%|█▋        | 2689/16384 [00:48<03:29, 65.34it/s]
-
DeepGEMM warmup:  19%|█▉        | 3153/16384 [00:48<01:31, 144.66it/s]
-
DeepGEMM warmup:  25%|██▍       | 4061/16384 [00:49<00:34, 360.66it/s]
-
DeepGEMM warmup:  30%|███       | 4993/16384 [00:51<00:31, 366.40it/s]
-
DeepGEMM warmup:  33%|███▎      | 5464/16384 [00:51<00:22, 478.47it/s]
-
DeepGEMM warmup:  39%|███▉      | 6385/16384 [00:51<00:12, 788.85it/s]
-
DeepGEMM warmup:  48%|████▊     | 7849/16384 [00:51<00:05, 1458.22it/s]
-
DeepGEMM warmup:  57%|█████▋    | 9351/16384 [00:51<00:03, 2333.98it/s]
-
DeepGEMM warmup:  66%|██████▋   | 10859/16384 [00:52<00:01, 3410.52it/s]
-
DeepGEMM warmup:  76%|███████▌  | 12373/16384 [00:52<00:00, 4673.28it/s]
-
DeepGEMM warmup:  85%|████████▍ | 13895/16384 [00:52<00:00, 6078.81it/s]
-
DeepGEMM warmup:  94%|█████████▍| 15419/16384 [00:52<00:00, 7543.59it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:52<00:00, 312.43it/s] 
-[2026-09-09 09:15:19 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:15:19 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=32768, K=1024, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:15:19 DP0 TP0 EP0] Required memory for warmup: 1.046875GB, Available memory: 33.5592041015625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   0%|          | 1/16384 [00:02<10:56:15,  2.40s/it]
-
DeepGEMM warmup:   0%|          | 65/16384 [00:04<16:47, 16.19it/s]  
-
DeepGEMM warmup:   3%|▎         | 534/16384 [00:04<01:29, 177.54it/s]
-
DeepGEMM warmup:   9%|▉         | 1439/16384 [00:04<00:25, 594.53it/s]
-
DeepGEMM warmup:  17%|█▋        | 2781/16384 [00:05<00:09, 1401.39it/s]
-
DeepGEMM warmup:  22%|██▏       | 3680/16384 [00:05<00:06, 2027.14it/s]
-
DeepGEMM warmup:  28%|██▊       | 4511/16384 [00:05<00:04, 2662.21it/s]
-
DeepGEMM warmup:  32%|███▏      | 5318/16384 [00:05<00:03, 3270.05it/s]
-
DeepGEMM warmup:  37%|███▋      | 6079/16384 [00:05<00:02, 3766.41it/s]
-
DeepGEMM warmup:  41%|████▏     | 6787/16384 [00:05<00:02, 4114.86it/s]
-
DeepGEMM warmup:  45%|████▌     | 7443/16384 [00:05<00:02, 4313.80it/s]
-
DeepGEMM warmup:  49%|████▉     | 8050/16384 [00:05<00:01, 4388.85it/s]
-
DeepGEMM warmup:  53%|█████▎    | 8613/16384 [00:06<00:01, 4379.69it/s]
-
DeepGEMM warmup:  56%|█████▌    | 9137/16384 [00:06<00:01, 4313.24it/s]
-
DeepGEMM warmup:  59%|█████▉    | 9628/16384 [00:06<00:01, 4210.07it/s]
-
DeepGEMM warmup:  62%|██████▏   | 10090/16384 [00:06<00:01, 4090.92it/s]
-
DeepGEMM warmup:  64%|██████▍   | 10527/16384 [00:06<00:01, 3962.91it/s]
-
DeepGEMM warmup:  67%|██████▋   | 10942/16384 [00:06<00:01, 3834.32it/s]
-
DeepGEMM warmup:  69%|██████▉   | 11338/16384 [00:06<00:01, 3708.31it/s]
-
DeepGEMM warmup:  72%|███████▏  | 11717/16384 [00:06<00:01, 3589.20it/s]
-
DeepGEMM warmup:  74%|███████▎  | 12081/16384 [00:06<00:01, 3478.38it/s]
-
DeepGEMM warmup:  76%|███████▌  | 12432/16384 [00:07<00:01, 3375.25it/s]
-
DeepGEMM warmup:  78%|███████▊  | 12772/16384 [00:07<00:01, 3281.19it/s]
-
DeepGEMM warmup:  80%|███████▉  | 13101/16384 [00:07<00:01, 3192.80it/s]
-
DeepGEMM warmup:  82%|████████▏ | 13421/16384 [00:07<00:00, 3108.80it/s]
-
DeepGEMM warmup:  84%|████████▍ | 13732/16384 [00:07<00:00, 3034.33it/s]
-
DeepGEMM warmup:  86%|████████▌ | 14036/16384 [00:07<00:00, 2963.97it/s]
-
DeepGEMM warmup:  87%|████████▋ | 14333/16384 [00:07<00:00, 2898.48it/s]
-
DeepGEMM warmup:  89%|████████▉ | 14623/16384 [00:07<00:00, 2837.35it/s]
-
DeepGEMM warmup:  91%|█████████ | 14907/16384 [00:07<00:00, 2778.92it/s]
-
DeepGEMM warmup:  93%|█████████▎| 15185/16384 [00:08<00:00, 2726.33it/s]
-
DeepGEMM warmup:  94%|█████████▍| 15458/16384 [00:08<00:00, 2677.47it/s]
-
DeepGEMM warmup:  96%|█████████▌| 15726/16384 [00:08<00:00, 2629.80it/s]
-
DeepGEMM warmup:  98%|█████████▊| 15989/16384 [00:08<00:00, 2584.01it/s]
-
DeepGEMM warmup:  99%|█████████▉| 16248/16384 [00:08<00:00, 2543.69it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:08<00:00, 1919.87it/s]
-[2026-09-09 09:15:28 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:15:28 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=4096, K=8192, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:15:28 DP0 TP0 EP0] Required memory for warmup: 0.28125GB, Available memory: 33.5572509765625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   9%|▉         | 1490/16384 [00:00<00:01, 14891.88it/s]
-
DeepGEMM warmup:  18%|█▊        | 3010/16384 [00:00<00:00, 15072.65it/s]
-
DeepGEMM warmup:  28%|██▊       | 4518/16384 [00:00<00:00, 14076.61it/s]
-
DeepGEMM warmup:  36%|███▌      | 5933/16384 [00:00<00:00, 11900.60it/s]
-
DeepGEMM warmup:  44%|████▎     | 7165/16384 [00:00<00:00, 10253.70it/s]
-
DeepGEMM warmup:  50%|█████     | 8239/16384 [00:00<00:00, 9008.74it/s] 
-
DeepGEMM warmup:  56%|█████▌    | 9183/16384 [00:00<00:00, 8093.94it/s]
-
DeepGEMM warmup:  61%|██████    | 10025/16384 [00:01<00:00, 7342.90it/s]
-
DeepGEMM warmup:  66%|██████▌   | 10782/16384 [00:01<00:00, 6737.06it/s]
-
DeepGEMM warmup:  70%|███████   | 11470/16384 [00:01<00:00, 6260.09it/s]
-
DeepGEMM warmup:  74%|███████▍  | 12104/16384 [00:01<00:00, 5837.91it/s]
-
DeepGEMM warmup:  77%|███████▋  | 12691/16384 [00:01<00:00, 5487.44it/s]
-
DeepGEMM warmup:  81%|████████  | 13240/16384 [00:01<00:00, 5196.67it/s]
-
DeepGEMM warmup:  84%|████████▍ | 13758/16384 [00:01<00:00, 4947.85it/s]
-
DeepGEMM warmup:  87%|████████▋ | 14250/16384 [00:01<00:00, 4722.65it/s]
-
DeepGEMM warmup:  90%|████████▉ | 14719/16384 [00:02<00:00, 4534.45it/s]
-
DeepGEMM warmup:  93%|█████████▎| 15170/16384 [00:02<00:00, 4345.56it/s]
-
DeepGEMM warmup:  95%|█████████▌| 15602/16384 [00:02<00:00, 4210.16it/s]
-
DeepGEMM warmup:  98%|█████████▊| 16021/16384 [00:02<00:00, 4048.47it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:02<00:00, 6490.78it/s]
-[2026-09-09 09:15:30 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:15:30 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=4096, K=4096, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:15:30 DP0 TP0 EP0] Required memory for warmup: 0.203125GB, Available memory: 33.5240478515625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   9%|▉         | 1467/16384 [00:00<00:01, 14661.17it/s]
-
DeepGEMM warmup:  18%|█▊        | 2965/16384 [00:00<00:00, 14843.87it/s]
-
DeepGEMM warmup:  27%|██▋       | 4471/16384 [00:00<00:00, 14939.25it/s]
-
DeepGEMM warmup:  37%|███▋      | 5991/16384 [00:00<00:00, 15040.57it/s]
-
DeepGEMM warmup:  46%|████▌     | 7496/16384 [00:00<00:00, 14879.45it/s]
-
DeepGEMM warmup:  55%|█████▍    | 8985/16384 [00:00<00:00, 13508.36it/s]
-
DeepGEMM warmup:  63%|██████▎   | 10357/16384 [00:00<00:00, 12223.08it/s]
-
DeepGEMM warmup:  71%|███████   | 11610/16384 [00:00<00:00, 11137.52it/s]
-
DeepGEMM warmup:  78%|███████▊  | 12754/16384 [00:01<00:00, 10228.54it/s]
-
DeepGEMM warmup:  84%|████████▍ | 13803/16384 [00:01<00:00, 9486.95it/s] 
-
DeepGEMM warmup:  90%|█████████ | 14772/16384 [00:01<00:00, 8851.56it/s]
-
DeepGEMM warmup:  96%|█████████▌| 15671/16384 [00:01<00:00, 8299.38it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:01<00:00, 10623.87it/s]
-[2026-09-09 09:15:32 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:15:32 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=4096, K=2048, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:15:32 DP0 TP0 EP0] Required memory for warmup: 0.1640625GB, Available memory: 33.5240478515625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   9%|▉         | 1454/16384 [00:00<00:01, 14535.92it/s]
-
DeepGEMM warmup:  18%|█▊        | 2925/16384 [00:00<00:00, 14634.81it/s]
-
DeepGEMM warmup:  27%|██▋       | 4417/16384 [00:00<00:00, 14762.12it/s]
-
DeepGEMM warmup:  36%|███▌      | 5915/16384 [00:00<00:00, 14845.04it/s]
-
DeepGEMM warmup:  45%|████▌     | 7417/16384 [00:00<00:00, 14907.46it/s]
-
DeepGEMM warmup:  54%|█████▍    | 8928/16384 [00:00<00:00, 14975.87it/s]
-
DeepGEMM warmup:  64%|██████▍   | 10445/16384 [00:00<00:00, 15039.18it/s]
-
DeepGEMM warmup:  73%|███████▎  | 11970/16384 [00:00<00:00, 15105.96it/s]
-
DeepGEMM warmup:  82%|████████▏ | 13481/16384 [00:00<00:00, 14855.11it/s]
-
DeepGEMM warmup:  91%|█████████▏| 14968/16384 [00:01<00:00, 13977.75it/s]
-
DeepGEMM warmup: 100%|█████████▉| 16376/16384 [00:01<00:00, 13145.69it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:01<00:00, 14251.90it/s]
-[2026-09-09 09:15:33 DP0 TP0 EP0] Use DeepEP Config: {'normal_dispatch': {'num_sms': 96}, 'normal_combine': {'num_sms': 96}}
-/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA
-/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA
-/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA
-/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA
-[2026-09-09 09:15:51 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:15:51 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GROUPED_GEMM_NT_F8F8BF16_MASKED> N=4096, K=4096, num_groups=64 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:15:51 DP0 TP0 EP0] Required memory for warmup: 13.000000238418579GB, Available memory: 15.4010009765625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   0%|          | 1/16384 [00:02<11:35:48,  2.55s/it]
-
DeepGEMM warmup:   3%|▎         | 562/16384 [00:02<00:53, 298.16it/s]
-
DeepGEMM warmup:   7%|▋         | 1165/16384 [00:02<00:21, 702.15it/s]
-
DeepGEMM warmup:  10%|█         | 1667/16384 [00:02<00:13, 1096.97it/s]
-
DeepGEMM warmup:  13%|█▎        | 2169/16384 [00:02<00:09, 1550.02it/s]
-
DeepGEMM warmup:  16%|█▋        | 2671/16384 [00:03<00:06, 2041.14it/s]
-
DeepGEMM warmup:  19%|█▉        | 3173/16384 [00:03<00:05, 2541.61it/s]
-
DeepGEMM warmup:  22%|██▏       | 3675/16384 [00:03<00:04, 3020.90it/s]
-
DeepGEMM warmup:  25%|██▌       | 4177/16384 [00:03<00:03, 3452.72it/s]
-
DeepGEMM warmup:  29%|██▊       | 4678/16384 [00:03<00:03, 3820.68it/s]
-
DeepGEMM warmup:  32%|███▏      | 5180/16384 [00:03<00:02, 4121.66it/s]
-
DeepGEMM warmup:  35%|███▍      | 5682/16384 [00:03<00:02, 4357.36it/s]
-
DeepGEMM warmup:  38%|███▊      | 6183/16384 [00:03<00:02, 4536.30it/s]
-
DeepGEMM warmup:  41%|████      | 6684/16384 [00:03<00:02, 4669.54it/s]
-
DeepGEMM warmup:  44%|████▍     | 7186/16384 [00:03<00:01, 4767.44it/s]
-
DeepGEMM warmup:  47%|████▋     | 7688/16384 [00:04<00:01, 4838.11it/s]
-
DeepGEMM warmup:  50%|████▉     | 8190/16384 [00:04<00:01, 4889.00it/s]
-
DeepGEMM warmup:  53%|█████▎    | 8691/16384 [00:04<00:01, 4924.16it/s]
-
DeepGEMM warmup:  56%|█████▌    | 9193/16384 [00:04<00:01, 4949.95it/s]
-
DeepGEMM warmup:  59%|█████▉    | 9695/16384 [00:04<00:01, 4967.88it/s]
-
DeepGEMM warmup:  62%|██████▏   | 10197/16384 [00:04<00:01, 4980.56it/s]
-
DeepGEMM warmup:  65%|██████▌   | 10699/16384 [00:04<00:01, 4989.38it/s]
-
DeepGEMM warmup:  68%|██████▊   | 11200/16384 [00:04<00:01, 4995.41it/s]
-
DeepGEMM warmup:  71%|███████▏  | 11702/16384 [00:04<00:00, 4999.84it/s]
-
DeepGEMM warmup:  74%|███████▍  | 12203/16384 [00:04<00:00, 5002.46it/s]
-
DeepGEMM warmup:  78%|███████▊  | 12704/16384 [00:05<00:00, 5004.52it/s]
-
DeepGEMM warmup:  81%|████████  | 13205/16384 [00:05<00:00, 5006.05it/s]
-
DeepGEMM warmup:  84%|████████▎ | 13706/16384 [00:05<00:00, 5006.77it/s]
-
DeepGEMM warmup:  87%|████████▋ | 14207/16384 [00:05<00:00, 5007.34it/s]
-
DeepGEMM warmup:  90%|████████▉ | 14708/16384 [00:05<00:00, 5008.10it/s]
-
DeepGEMM warmup:  93%|█████████▎| 15209/16384 [00:05<00:00, 5008.35it/s]
-
DeepGEMM warmup:  96%|█████████▌| 15711/16384 [00:05<00:00, 5009.03it/s]
-
DeepGEMM warmup:  99%|█████████▉| 16212/16384 [00:05<00:00, 5008.93it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:05<00:00, 2831.54it/s]
-[2026-09-09 09:15:59 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:15:59 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GROUPED_GEMM_NT_F8F8BF16_MASKED> N=4096, K=2048, num_groups=64 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:15:59 DP0 TP0 EP0] Required memory for warmup: 10.500000238418579GB, Available memory: 14.9010009765625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   0%|          | 1/16384 [00:02<11:40:45,  2.57s/it]
-
DeepGEMM warmup:   4%|▎         | 582/16384 [00:02<00:51, 306.71it/s]
-
DeepGEMM warmup:  10%|█         | 1663/16384 [00:02<00:14, 1042.43it/s]
-
DeepGEMM warmup:  18%|█▊        | 2877/16384 [00:02<00:06, 2051.76it/s]
-
DeepGEMM warmup:  24%|██▍       | 3989/16384 [00:02<00:04, 3094.56it/s]
-
DeepGEMM warmup:  31%|███       | 5101/16384 [00:03<00:02, 4226.95it/s]
-
DeepGEMM warmup:  38%|███▊      | 6213/16384 [00:03<00:01, 5383.28it/s]
-
DeepGEMM warmup:  45%|████▍     | 7325/16384 [00:03<00:01, 6491.71it/s]
-
DeepGEMM warmup:  51%|█████▏    | 8437/16384 [00:03<00:01, 7493.02it/s]
-
DeepGEMM warmup:  58%|█████▊    | 9549/16384 [00:03<00:00, 8349.37it/s]
-
DeepGEMM warmup:  65%|██████▌   | 10661/16384 [00:03<00:00, 9047.93it/s]
-
DeepGEMM warmup:  72%|███████▏  | 11773/16384 [00:03<00:00, 9596.72it/s]
-
DeepGEMM warmup:  79%|███████▊  | 12885/16384 [00:03<00:00, 10013.67it/s]
-
DeepGEMM warmup:  85%|████████▌ | 13997/16384 [00:03<00:00, 10324.55it/s]
-
DeepGEMM warmup:  92%|█████████▏| 15110/16384 [00:03<00:00, 10554.12it/s]
-
DeepGEMM warmup:  99%|█████████▉| 16222/16384 [00:04<00:00, 10718.40it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:04<00:00, 4014.25it/s] 
-[2026-09-09 09:16:24 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:16:24 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=8192, K=1024, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:16:24 DP0 TP0 EP0] Required memory for warmup: 0.2734375GB, Available memory: 18.3365478515625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   9%|▊         | 1420/16384 [00:00<00:01, 14190.57it/s]
-
DeepGEMM warmup:  17%|█▋        | 2867/16384 [00:00<00:00, 14349.07it/s]
-
DeepGEMM warmup:  26%|██▋       | 4336/16384 [00:00<00:00, 14501.43it/s]
-
DeepGEMM warmup:  36%|███▌      | 5858/16384 [00:00<00:00, 14783.86it/s]
-
DeepGEMM warmup:  45%|████▌     | 7394/16384 [00:00<00:00, 14989.96it/s]
-
DeepGEMM warmup:  54%|█████▍    | 8899/16384 [00:00<00:00, 15008.77it/s]
-
DeepGEMM warmup:  63%|██████▎   | 10400/16384 [00:00<00:00, 13947.64it/s]
-
DeepGEMM warmup:  72%|███████▏  | 11809/16384 [00:00<00:00, 12899.83it/s]
-
DeepGEMM warmup:  80%|████████  | 13120/16384 [00:00<00:00, 11960.35it/s]
-
DeepGEMM warmup:  88%|████████▊ | 14338/16384 [00:01<00:00, 11137.14it/s]
-
DeepGEMM warmup:  94%|█████████▍| 15472/16384 [00:01<00:00, 10431.72it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:01<00:00, 12177.20it/s]
-
Capturing batches (bs=64 avail_mem=33.58 GB):   8%|▊         | 1/12 [03:17<36:17, 197.93s/it]
Capturing batches (bs=56 avail_mem=14.63 GB):   8%|▊         | 1/12 [03:17<36:17, 197.93s/it]
Capturing batches (bs=56 avail_mem=14.63 GB):  17%|█▋        | 2/12 [03:18<13:41, 82.12s/it] 
Capturing batches (bs=48 avail_mem=14.61 GB):  17%|█▋        | 2/12 [03:18<13:41, 82.12s/it]
Capturing batches (bs=48 avail_mem=14.61 GB):  25%|██▌       | 3/12 [03:19<06:44, 44.91s/it]
Capturing batches (bs=40 avail_mem=14.59 GB):  25%|██▌       | 3/12 [03:19<06:44, 44.91s/it]
Capturing batches (bs=40 avail_mem=14.59 GB):  33%|███▎      | 4/12 [03:20<03:39, 27.43s/it]
Capturing batches (bs=32 avail_mem=14.58 GB):  33%|███▎      | 4/12 [03:20<03:39, 27.43s/it]
Capturing batches (bs=32 avail_mem=14.58 GB):  42%|████▏     | 5/12 [03:28<02:23, 20.50s/it]
Capturing batches (bs=24 avail_mem=14.56 GB):  42%|████▏     | 5/12 [03:28<02:23, 20.50s/it]
Capturing batches (bs=24 avail_mem=14.56 GB):  50%|█████     | 6/12 [03:29<01:22, 13.74s/it]
Capturing batches (bs=16 avail_mem=14.54 GB):  50%|█████     | 6/12 [03:29<01:22, 13.74s/it]
Capturing batches (bs=16 avail_mem=14.54 GB):  58%|█████▊    | 7/12 [03:35<00:56, 11.31s/it]
Capturing batches (bs=12 avail_mem=14.52 GB):  58%|█████▊    | 7/12 [03:35<00:56, 11.31s/it]
Capturing batches (bs=12 avail_mem=14.52 GB):  67%|██████▋   | 8/12 [03:35<00:31,  7.90s/it]
Capturing batches (bs=8 avail_mem=14.50 GB):  67%|██████▋   | 8/12 [03:35<00:31,  7.90s/it] 
Capturing batches (bs=8 avail_mem=14.50 GB):  75%|███████▌  | 9/12 [03:36<00:16,  5.62s/it]
Capturing batches (bs=4 avail_mem=14.49 GB):  75%|███████▌  | 9/12 [03:36<00:16,  5.62s/it]
Capturing batches (bs=4 avail_mem=14.49 GB):  83%|████████▎ | 10/12 [03:49<00:15,  7.92s/it]
Capturing batches (bs=2 avail_mem=14.47 GB):  83%|████████▎ | 10/12 [03:49<00:15,  7.92s/it]
Capturing batches (bs=2 avail_mem=14.47 GB):  92%|█████████▏| 11/12 [03:50<00:05,  5.71s/it]
Capturing batches (bs=1 avail_mem=14.46 GB):  92%|█████████▏| 11/12 [03:50<00:05,  5.71s/it]
Capturing batches (bs=1 avail_mem=14.46 GB): 100%|██████████| 12/12 [03:51<00:00,  4.39s/it]
Capturing batches (bs=1 avail_mem=14.46 GB): 100%|██████████| 12/12 [03:51<00:00, 19.31s/it]
-[2026-09-09 09:17:42 DP1 TP1 EP1] Capture target decode CUDA graph end. elapsed=232.75 s, mem usage=19.33 GB, avail mem=14.25 GB.
-[2026-09-09 09:17:42 DP1 TP1 EP1] Init Unified Radix Cache. Components: (<ComponentType.FULL: 0>, <ComponentType.SWA: 1>). Tree Core: UnifiedTreeCore
-[2026-09-09 09:17:42 DP1 TP1 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hicache_attached=False streaming_wrapped=False
-[2026-09-09 09:17:42 DP0 TP0 EP0] Capture target decode CUDA graph end. elapsed=232.76 s, mem usage=19.18 GB, avail mem=14.44 GB.
-[2026-09-09 09:17:42 DP0 TP0 EP0] max_total_num_tokens=25820928, chunked_prefill_size=4096, max_prefill_tokens=16384, max_running_requests=64, context_len=1048576, available_gpu_mem=14.44 GB
-[2026-09-09 09:17:42 DP0 TP0 EP0] Init Unified Radix Cache. Components: (<ComponentType.FULL: 0>, <ComponentType.SWA: 1>). Tree Core: UnifiedTreeCore
-[2026-09-09 09:17:42 DP0 TP0 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hicache_attached=False streaming_wrapped=False
-[2026-09-09 09:17:42 DP2 TP2 EP2] Capture target decode CUDA graph end. elapsed=232.78 s, mem usage=19.33 GB, avail mem=14.25 GB.
-[2026-09-09 09:17:42 DP2 TP2 EP2] Init Unified Radix Cache. Components: (<ComponentType.FULL: 0>, <ComponentType.SWA: 1>). Tree Core: UnifiedTreeCore
-[2026-09-09 09:17:42 DP2 TP2 EP2] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hicache_attached=False streaming_wrapped=False
-[2026-09-09 09:17:42 DP3 TP3 EP3] Capture target decode CUDA graph end. elapsed=232.80 s, mem usage=18.55 GB, avail mem=15.19 GB.
-[2026-09-09 09:17:42 DP3 TP3 EP3] Init Unified Radix Cache. Components: (<ComponentType.FULL: 0>, <ComponentType.SWA: 1>). Tree Core: UnifiedTreeCore
-[2026-09-09 09:17:42 DP3 TP3 EP3] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hicache_attached=False streaming_wrapped=False
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-[2026-09-09 09:17:43] Engine startup timings (s): load_weight=403.02, kv_cache_allocation=0.82, scheduler_e2e=720.58, cuda_graph={prefill=0.00, decode=232.80, target_verify=0.00, draft_prefill=0.00, draft_decode=0.00, draft_extend=0.00}, tokenizer_e2e=739.50
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-[2026-09-09 09:17:43] INFO:     Started server process [1]
-[2026-09-09 09:17:43] INFO:     Waiting for application startup.
-[2026-09-09 09:17:43] Using default chat sampling params from model generation config: {'temperature': 1.0, 'top_p': 1.0}
-[2026-09-09 09:17:43] INFO:     Application startup complete.
-[2026-09-09 09:17:43] INFO:     Uvicorn running on http://0.0.0.0:30000 (Press CTRL+C to quit)
-[2026-09-09 09:17:44] INFO:     127.0.0.1:54148 - "GET /model_info HTTP/1.1" 200 OK
-[2026-09-09 09:17:48 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:17:48 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GROUPED_GEMM_NT_F8F8BF16_CONTIG> N=4096, K=4096, num_groups=64 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:17:48 DP0 TP0 EP0] Required memory for warmup: 1.18756103515625GB, Available memory: 14.3795166015625GB
-[2026-09-09 09:17:51] INFO:     172.17.0.1:40494 - "GET /health HTTP/1.1" 503 Service Unavailable
-
DeepGEMM warmup:   0%|          | 0/128 [00:00<?, ?it/s]
DeepGEMM warmup:   1%|          | 1/128 [00:02<05:18,  2.50s/it]
DeepGEMM warmup: 100%|██████████| 128/128 [00:02<00:00, 50.47it/s]
-[2026-09-09 09:17:58 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-09 09:17:58 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GROUPED_GEMM_NT_F8F8BF16_CONTIG> N=4096, K=2048, num_groups=64 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-09 09:17:58 DP0 TP0 EP0] Required memory for warmup: 0.65631103515625GB, Available memory: 14.3795166015625GB
-[2026-09-09 09:18:01] INFO:     172.17.0.1:55790 - "GET /health HTTP/1.1" 503 Service Unavailable
-
DeepGEMM warmup:   0%|          | 0/128 [00:00<?, ?it/s]
DeepGEMM warmup:   1%|          | 1/128 [00:02<05:23,  2.55s/it]
DeepGEMM warmup: 100%|██████████| 128/128 [00:02<00:00, 49.63it/s]
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-[2026-09-09 09:18:11] INFO:     172.17.0.1:50556 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:18:17 DP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 7.35
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-[2026-09-09 09:18:18 DP1 TP1 EP1] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 7.29
-[2026-09-09 09:18:18 DP2 TP2 EP2] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 7.29
-[2026-09-09 09:18:18 DP3 TP3 EP3] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 7.30
-[2026-09-09 09:18:18 DP0 TP0 EP0] Scheduler hit an exception: Traceback (most recent call last):
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 5331, in run_scheduler_process
-    scheduler.run_event_loop()
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 1716, in run_event_loop
-    dispatch_event_loop(self)
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 5180, in dispatch_event_loop
-    scheduler.event_loop_overlap()
-  File "/usr/local/lib/python3.12/dist-packages/torch/utils/_contextlib.py", line 124, in decorate_context
-    return func(*args, **kwargs)
-           ^^^^^^^^^^^^^^^^^^^^^
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 1825, in event_loop_overlap
-    pop_and_process()
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 1776, in pop_and_process
-    self.process_batch_result(tmp_batch, tmp_result)
-  File "/sgl-workspace/sglang/python/sglang/srt/utils/nvtx_utils.py", line 109, in wrapper
-    return func(*args, **kwargs)
-           ^^^^^^^^^^^^^^^^^^^^^
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler.py", line 4174, in process_batch_result
-    self.batch_result_processor.process_batch_result_decode(batch, result)
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/scheduler_components/batch_result_processor.py", line 907, in process_batch_result_decode
-    self.token_to_kv_pool_allocator.free_group_end()
-  File "/sgl-workspace/sglang/python/sglang/srt/mem_cache/allocator/swa.py", line 374, in free_group_end
-    super().free_group_end()
-  File "/sgl-workspace/sglang/python/sglang/srt/mem_cache/allocator/base.py", line 70, in free_group_end
-    self.free(torch.cat(self.free_group))
-  File "/sgl-workspace/sglang/python/sglang/srt/mem_cache/allocator/swa.py", line 326, in free
-    self.free_swa(free_index)
-  File "/sgl-workspace/sglang/python/sglang/srt/mem_cache/allocator/swa.py", line 359, in free_swa
-    self.clear_full_to_swa_mapping(mapping_indices)
-    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-AttributeError: 'SWATokenToKVPoolAllocator' object has no attribute 'clear_full_to_swa_mapping'. Did you mean: 'set_full_to_swa_mapping'?
-
-[2026-09-09 09:18:19] Subprocess scheduler_0 (pid=167) crashed with exit code -3. Triggering SIGQUIT for cleanup...
-[2026-09-09 09:18:19] SIGQUIT received. signum=None, frame=None. It usually means one child failed.
-[2026-09-09 09:18:19] Sleeping 5 seconds before crash diagnostics to let GPU activity settle.
-[2026-09-09 09:18:24] No live scheduler processes found; skipping py-spy and CUDA coredump.
-[2026-09-09 09:18:24] kill_process_tree called: parent_pid=1, include_parent=True, pid=1
-[2026-09-09 09:27:44] Initialization failed. warmup error: Traceback (most recent call last):
-  File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 534, in _make_request
-    response = conn.getresponse()
-               ^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/urllib3/connection.py", line 571, in getresponse
-    httplib_response = super().getresponse()
-                       ^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/lib/python3.12/http/client.py", line 1457, in getresponse
-    response.begin()
-  File "/usr/lib/python3.12/http/client.py", line 336, in begin
-    version, status, reason = self._read_status()
-                              ^^^^^^^^^^^^^^^^^^^
-  File "/usr/lib/python3.12/http/client.py", line 297, in _read_status
-    line = str(self.fp.readline(_MAXLINE + 1), "iso-8859-1")
-               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/lib/python3.12/socket.py", line 707, in readinto
-    return self._sock.recv_into(b)
-           ^^^^^^^^^^^^^^^^^^^^^^^
-TimeoutError: timed out
-
-The above exception was the direct cause of the following exception:
-
-Traceback (most recent call last):
-  File "/usr/local/lib/python3.12/dist-packages/requests/adapters.py", line 696, in send
-    resp = conn.urlopen(
-           ^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 842, in urlopen
-    retries = retries.increment(
-              ^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/urllib3/util/retry.py", line 498, in increment
-    raise reraise(type(error), error, _stacktrace)
-          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/urllib3/util/util.py", line 39, in reraise
-    raise value
-  File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 788, in urlopen
-    response = self._make_request(
-               ^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 536, in _make_request
-    self._raise_timeout(err=e, url=url, timeout_value=read_timeout)
-  File "/usr/local/lib/python3.12/dist-packages/urllib3/connectionpool.py", line 367, in _raise_timeout
-    raise ReadTimeoutError(
-urllib3.exceptions.ReadTimeoutError: HTTPConnectionPool(host='127.0.0.1', port=30000): Read timed out. (read timeout=600)
-
-During handling of the above exception, another exception occurred:
-
-Traceback (most recent call last):
-  File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/http_server.py", line 2303, in _execute_server_warmup
-    res = requests.post(
-          ^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/requests/api.py", line 134, in post
-    return request("post", url, data=data, json=json, **kwargs)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/requests/api.py", line 71, in request
-    return session.request(method=method, url=url, **kwargs)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/requests/sessions.py", line 651, in request
-    resp = self.send(prep, **send_kwargs)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/requests/sessions.py", line 784, in send
-    r = adapter.send(request, **kwargs)
-        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/requests/adapters.py", line 742, in send
-    raise ReadTimeout(e, request=request)
-requests.exceptions.ReadTimeout: HTTPConnectionPool(host='127.0.0.1', port=30000): Read timed out. (read timeout=600)
-
-[2026-09-09 09:27:44] kill_process_tree called: parent_pid=1, include_parent=True, pid=1
-[2026-09-09 09:27:44] ERROR:    Traceback (most recent call last):
-  File "/usr/lib/python3.12/asyncio/runners.py", line 194, in run
-    return runner.run(main)
-           ^^^^^^^^^^^^^^^^
-  File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
-    return self._loop.run_until_complete(task)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "uvloop/loop.pyx", line 1512, in uvloop.loop.Loop.run_until_complete
-  File "uvloop/loop.pyx", line 1505, in uvloop.loop.Loop.run_until_complete
-  File "uvloop/loop.pyx", line 1379, in uvloop.loop.Loop.run_forever
-  File "uvloop/loop.pyx", line 557, in uvloop.loop.Loop._run
-  File "uvloop/loop.pyx", line 476, in uvloop.loop.Loop._on_idle
-  File "uvloop/cbhandles.pyx", line 83, in uvloop.loop.Handle._run
-  File "uvloop/cbhandles.pyx", line 61, in uvloop.loop.Handle._run
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/tokenizer_manager.py", line 3665, in running_phase_sigquit_handler
-    kill_process_tree(os.getpid())
-  File "/sgl-workspace/sglang/python/sglang/srt/utils/common.py", line 2264, in kill_process_tree
-    sys.exit(0)
-SystemExit: 0
-
-During handling of the above exception, another exception occurred:
-
-Traceback (most recent call last):
-  File "/usr/local/lib/python3.12/dist-packages/starlette/routing.py", line 655, in lifespan
-    await receive()
-  File "/usr/local/lib/python3.12/dist-packages/uvicorn/lifespan/on.py", line 137, in receive
-    return await self.receive_queue.get()
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/lib/python3.12/asyncio/queues.py", line 158, in get
-    await getter
-asyncio.exceptions.CancelledError
-
-[2026-09-09 09:27:44] INFO:     172.17.0.1:41744 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:52350 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:52988 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] ERROR:    Exception in ASGI application
-Traceback (most recent call last):
-  File "/usr/lib/python3.12/asyncio/runners.py", line 194, in run
-    return runner.run(main)
-           ^^^^^^^^^^^^^^^^
-  File "/usr/lib/python3.12/asyncio/runners.py", line 118, in run
-    return self._loop.run_until_complete(task)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "uvloop/loop.pyx", line 1512, in uvloop.loop.Loop.run_until_complete
-  File "uvloop/loop.pyx", line 1505, in uvloop.loop.Loop.run_until_complete
-  File "uvloop/loop.pyx", line 1379, in uvloop.loop.Loop.run_forever
-  File "uvloop/loop.pyx", line 557, in uvloop.loop.Loop._run
-  File "uvloop/loop.pyx", line 476, in uvloop.loop.Loop._on_idle
-  File "uvloop/cbhandles.pyx", line 83, in uvloop.loop.Handle._run
-  File "uvloop/cbhandles.pyx", line 61, in uvloop.loop.Handle._run
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/tokenizer_manager.py", line 3665, in running_phase_sigquit_handler
-    kill_process_tree(os.getpid())
-  File "/sgl-workspace/sglang/python/sglang/srt/utils/common.py", line 2264, in kill_process_tree
-    sys.exit(0)
-SystemExit: 0
-
-During handling of the above exception, another exception occurred:
-
-Traceback (most recent call last):
-  File "/usr/local/lib/python3.12/dist-packages/uvicorn/protocols/http/h11_impl.py", line 416, in run_asgi
-    result = await app(  # type: ignore[func-returns-value]
-             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/uvicorn/middleware/proxy_headers.py", line 63, in __call__
-    return await self.app(scope, receive, send)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/fastapi/applications.py", line 1163, in __call__
-    await super().__call__(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/starlette/applications.py", line 96, in __call__
-    await self.middleware_stack(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/starlette/middleware/errors.py", line 164, in __call__
-    await self.app(scope, receive, _send)
-  File "/usr/local/lib/python3.12/dist-packages/starlette/middleware/cors.py", line 88, in __call__
-    await self.app(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/starlette/middleware/exceptions.py", line 63, in __call__
-    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/starlette/_exception_handler.py", line 42, in wrapped_app
-    await app(scope, receive, sender)
-  File "/usr/local/lib/python3.12/dist-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
-    await self.app(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/starlette/routing.py", line 670, in __call__
-    await self.middleware_stack(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/fastapi/routing.py", line 2734, in app
-    await route.handle(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/fastapi/routing.py", line 1281, in handle
-    await super().handle(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/starlette/routing.py", line 280, in handle
-    await self.app(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/fastapi/routing.py", line 158, in app
-    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
-  File "/usr/local/lib/python3.12/dist-packages/starlette/_exception_handler.py", line 42, in wrapped_app
-    await app(scope, receive, sender)
-  File "/usr/local/lib/python3.12/dist-packages/fastapi/routing.py", line 144, in app
-    response = await f(request)
-               ^^^^^^^^^^^^^^^^
-  File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/http_server.py", line 451, in custom_handler
-    return await original_handler(ORJSONRequest(request.scope, request.receive))
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/fastapi/routing.py", line 706, in app
-    raw_response = await run_endpoint_function(
-                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/fastapi/routing.py", line 352, in run_endpoint_function
-    return await dependant.call(**values)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/sgl-workspace/sglang/python/sglang/srt/entrypoints/http_server.py", line 933, in generate_request
-    ret = await _global_state.tokenizer_manager.generate_request(
-          ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/tokenizer_manager.py", line 820, in generate_request
-    async for response in self._handle_batch_request(obj, request):
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/tokenizer_manager.py", line 1949, in _handle_batch_request
-    outputs = await self._collect_batch_responses(generators)
-              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/tokenizer_manager.py", line 1958, in _collect_batch_responses
-    return await asyncio.gather(*tasks)
-           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/sgl-workspace/sglang/python/sglang/srt/managers/tokenizer_manager.py", line 1759, in _stream_one_response
-    and await request.is_disconnected()
-        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/starlette/requests.py", line 335, in is_disconnected
-    message = await self._receive()
-              ^^^^^^^^^^^^^^^^^^^^^
-  File "/usr/local/lib/python3.12/dist-packages/uvicorn/protocols/http/h11_impl.py", line 537, in receive
-    await self.message_event.wait()
-  File "/usr/lib/python3.12/asyncio/locks.py", line 212, in wait
-    await fut
-asyncio.exceptions.CancelledError
-[2026-09-09 09:27:44] INFO:     172.17.0.1:60166 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:42592 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:37430 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:38168 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:52036 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:47132 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:47038 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:43160 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:44436 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:39650 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:45938 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:47270 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] INFO:     172.17.0.1:46308 - "GET /health HTTP/1.1" 503 Service Unavailable
-[2026-09-09 09:27:44] kill_process_tree called: parent_pid=1, include_parent=False, pid=1
+Error response from daemon: can not get logs from container which is dead or marked for removal
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.log b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.log
index 2923e9f..3304880 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.log
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.log
@@ -1,390 +1 @@
-
-==========
-== CUDA ==
-==========
-
-CUDA Version 13.0.3
-
-Container image Copyright (c) 2016-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
-
-This container image and its contents are governed by the NVIDIA Deep Learning Container License.
-By pulling and using the container, you accept the terms and conditions of this license:
-https://developer.nvidia.com/ngc/nvidia-deep-learning-container-license
-
-A copy of this license is made available in this container at /NGC-DL-CONTAINER-LICENSE for your convenience.
-
-[2026-09-10 05:13:52] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:13:52] Hybrid SWA model detected. architectures=['DeepseekV4ForCausalLM']
-[2026-09-10 05:13:52] Breakable CUDA graph is incompatible with DeepSeek-V4 (heavy capture-pool memory pressure); disabling prefill CUDA graph.
-[2026-09-10 05:13:52] Use dsv4 attention backend for DeepseekV4ForCausalLM, setting page_size to 256.
-[2026-09-10 05:13:52] Setting swa_full_tokens_ratio to 0.1 for DeepseekV4ForCausalLM.
-[2026-09-10 05:13:52] Setting KV cache dtype to fp8_e4m3 for DeepseekV4ForCausalLM.
-[2026-09-10 05:13:52] Setting max_running_requests to 256 for DeepseekV4ForCausalLM.
-[2026-09-10 05:13:52] DP attention is enabled. chunked prefill size is adjusted from 16384 to 4096.
-[2026-09-10 05:13:52] deepep MoE is enabled. The expert parallel size is adjusted from 1 to the tensor parallel size [4].
-[2026-09-10 05:13:54] server_args=ServerArgs(model_path='/models', tokenizer_path='/models', tokenizer_mode='auto', tokenizer_backend='huggingface', tokenizer_worker_num=1, detokenizer_worker_num=1, skip_tokenizer_init=False, load_format='auto', model_loader_extra_config='{}', trust_remote_code=True, context_length=None, is_embedding=False, enable_multimodal=None, revision=None, model_impl='auto', model_config_parser='auto', json_model_override_args='{}', dtype='auto', quantization=None, quantization_param_path=None, kv_cache_dtype='fp8_e4m3', enable_fp32_lm_head=False, modelopt_quant=None, modelopt_checkpoint_restore_path=None, modelopt_checkpoint_save_path=None, modelopt_export_path=None, quantize_and_serve=False, rl_quant_profile=None, enable_tf32_matmul=False, mem_fraction_static=0.862, max_running_requests=256, max_queued_requests=None, max_total_tokens=None, chunked_prefill_size=4096, enable_dynamic_chunking=False, prefill_reserve_budget=0, max_prefill_tokens=16384, prefill_max_requests=None, schedule_policy='fcfs', enable_priority_scheduling=False, disable_priority_preemption=False, default_priority_value=None, abort_on_priority_when_disabled=False, schedule_low_priority_values_first=False, priority_scheduling_preemption_threshold=10, retraction_policy='length', schedule_conservativeness=0.3, page_size=256, c128_page_size=16, swa_full_tokens_ratio=0.1, disable_hybrid_swa_memory=False, radix_eviction_policy='lru', prefill_only_disable_kv_cache=False, disable_radix_cache=False, enable_page_major_kv_layout=False, enable_unified_memory=False, disable_chunked_prefix_cache=False, disable_overlap_schedule=False, num_continuous_decode_steps=1, scheduler_recv_interval=1, enable_mixed_chunk=False, nccl_port=None, dist_timeout=None, dist_init_addr=None, nnodes=1, node_rank=0, tp_size=4, dcp_size=1, pp_size=1, pp_max_micro_batch_size=None, pp_async_batch_depth=0, dp_size=4, load_balance_method='round_robin', prefix_affinity_fallback='total_tokens', prefix_affinity_max_load_skew=1.5, prefix_affinity_hash_tokens=4096, prefix_affinity_disable_token_fallback=False, attn_cp_size=1, moe_dp_size=1, dwdp_size=1, dcp_comm_backend='ag_rs', dcp_replicate_q_proj=None, enable_prefill_cp=False, cp_strategy=None, enable_dsa_cache_layer_split=False, enable_dsa_prefill_context_parallel=False, dsa_prefill_cp_mode='round-robin-split', enable_prefill_context_parallel=False, prefill_cp_mode='in-seq-split', enable_cp_decode_attn_tp=False, enable_dp_attention=True, enable_dp_attention_local_control_broadcast=False, enable_dp_lm_head=False, enable_tp_lm_head_all_to_all=False, enable_attn_tp_input_scattered=False, disable_attn_tp_gather=False, enable_p2p_check=False, device='cuda', base_gpu_id=0, gpu_id_step=1, random_seed=903930395, mlx_enable_sampling=False, watchdog_timeout=300, soft_watchdog_timeout=None, sleep_on_idle=False, use_ray=False, custom_sigquit_handler=None, numa_node=None, gc_threshold=None, host='0.0.0.0', port=30000, fastapi_root_path='', smg_grpc_mode=False, grpc_mode=False, grpc_port=None, sidecar=None, sidecar_args=None, skip_server_warmup=False, warmups=None, enable_http2=False, http2_max_concurrent_streams=200, ssl_keyfile=None, ssl_certfile=None, ssl_ca_certs=None, ssl_keyfile_password=None, enable_ssl_refresh=False, api_key=None, admin_api_key=None, served_model_name='/models', weight_version='default', chat_template=None, hf_chat_template_name=None, completion_template=None, file_storage_path='sglang_storage', enable_cache_report=False, reasoning_parser='deepseek-v4', default_chat_template_kwargs=None, strip_thinking_cache=False, enable_strict_thinking=False, tool_call_parser='deepseekv4', tool_server=None, sampling_defaults='model', asr_max_buffer_seconds=60, asr_max_concurrent_sessions=32, preferred_sampling_params=None, allow_auto_truncate=False, stream_interval=1, batch_notify_size=16, stream_response_default_include_usage=False, incremental_streaming_output=False, enable_streaming_session=False, enable_session_radix_cache=False, log_level='info', log_level_http=None, log_requests=False, log_requests_level=2, log_requests_format='text', log_requests_target=None, uvicorn_access_log_exclude_prefixes=[], crash_dump_folder=None, show_time_cost=False, enable_metrics=False, smg_http_sidecar_port=None, enable_mfu_metrics=False, enable_metrics_for_all_schedulers=False, load_snapshot_publish_interval=15, tokenizer_metrics_custom_labels_header='x-custom-labels', tokenizer_metrics_allowed_custom_labels=None, extra_metric_labels=None, bucket_time_to_first_token=None, bucket_inter_token_latency=None, bucket_e2e_request_latency=None, prompt_tokens_buckets=None, generation_tokens_buckets=None, gc_warning_threshold_secs=0.0, decode_log_interval=40, enable_request_time_stats_logging=False, kv_events_config=None, enable_forward_pass_metrics=False, forward_pass_metrics_worker_id='', forward_pass_metrics_ipc_name=None, enable_trace=False, trace_modules='request', otlp_traces_endpoint='localhost:4317', export_metrics_to_file=False, export_metrics_to_file_dir=None, stat_loggers=None, constrained_json_whitespace_pattern=None, constrained_json_disable_any_whitespace=False, attention_backend='dsv4', decode_attention_backend=None, prefill_attention_backend=None, sampling_backend='flashinfer', grammar_backend='xgrammar', radix_cache_backend=None, mm_attention_backend=None, fp8_gemm_runner_backend='auto', fp4_gemm_runner_backend='auto', bf16_gemm_backend='auto', dsa_prefill_backend=None, dsa_decode_backend=None, dsa_paged_mqa_logits_backend='auto', dsa_topk_backend='sgl-kernel', disable_flashinfer_autotune=False, flashinfer_autotune_skip_ops=None, mamba_backend='triton', cuda_graph_config=CudaGraphConfig(decode=PhaseConfig(backend='full', max_bs=512, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64, 72, 80, 88, 96, 104, 112, 120, 128, 136, 144, 152, 160, 168, 176, 184, 192, 200, 208, 216, 224, 232, 240, 248, 256, 272, 288, 304, 320, 336, 352, 368, 384, 400, 416, 432, 448, 464, 480, 496, 512], tc_compiler='eager', full_prefill_max_req=None, full_prefill_prefix_chunk_tokens=None), prefill=PhaseConfig(backend='disabled', max_bs=16384, bs=[4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192, 8704, 9216, 9728, 10240, 10752, 11264, 11776, 12288, 12800, 13312, 13824, 14336, 14848, 15360, 15872, 16384], tc_compiler='eager', full_prefill_max_req=None, full_prefill_prefix_chunk_tokens=None)), cuda_graph_backend_decode=None, cuda_graph_backend_prefill=None, cuda_graph_max_bs_decode=None, cuda_graph_max_bs_prefill=None, cuda_graph_bs_decode=None, cuda_graph_bs_prefill=None, cuda_graph_tc_compiler=None, disable_prefill_cuda_graph=False, disable_decode_cuda_graph=False, disable_cuda_graph=False, disable_cuda_graph_padding=False, enable_profile_cuda_graph=False, enable_cudagraph_gc=False, debug_cuda_graph=False, enable_layerwise_nvtx_marker=False, enable_nccl_nvls=False, enable_symm_mem=False, triton_attention_reduce_in_fp32=False, triton_attention_num_kv_splits=8, triton_attention_split_tile_size=None, flashinfer_mla_disable_ragged=False, enable_fused_qk_norm_rope=False, enable_precise_embedding_interpolation=False, enable_fused_moe_sum_all_reduce=False, enable_deepseek_v4_fp4_indexer=False, disable_custom_all_reduce=False, enable_mscclpp=False, enable_torch_symm_mem=False, enable_scattered_sconv=False, pre_warm_nccl=False, enable_quant_communications=False, enable_flashinfer_allreduce_fusion=False, enforce_disable_flashinfer_allreduce_fusion=False, flashinfer_allreduce_fusion_backend=None, enable_aiter_allreduce_fusion=False, enable_torch_compile=False, enable_torch_compile_debug_mode=False, torch_compile_max_bs=32, speculative_algorithm=None, speculative_draft_model_path=None, speculative_draft_model_revision=None, speculative_draft_load_format=None, speculative_num_steps=None, speculative_eagle_topk=None, speculative_num_draft_tokens=None, speculative_dflash_block_size=None, speculative_dspark_block_size=None, speculative_dspark_sps_table_path=None, speculative_dspark_confidence_sts_path=None, speculative_dspark_align_verify_tokens_to_graph_tier=False, speculative_accept_threshold_single=1.0, speculative_accept_threshold_acc=1.0, speculative_use_rejection_sampling=False, speculative_token_map=None, speculative_attention_mode='prefill', speculative_draft_attention_backend=None, speculative_draft_kv_cache_dtype=None, speculative_draft_window_size=None, speculative_moe_runner_backend='auto', speculative_moe_a2a_backend=None, speculative_draft_model_quantization=None, _speculative_draft_quantization_explicitly_set=False, speculative_skip_dp_mlp_sync=False, enable_multi_layer_eagle=False, speculative_adaptive=False, speculative_adaptive_config=None, decoupled_spec_bind_endpoint=None, decoupled_spec_connect_endpoints=None, decoupled_spec_rank=None, decoupled_spec_role='null', spec_trace_dir=None, speculative_ngram_min_bfs_breadth=1, speculative_ngram_max_bfs_breadth=10, speculative_ngram_match_type='BFS', speculative_ngram_max_trie_depth=18, speculative_ngram_capacity=10000000, speculative_ngram_external_corpus_path=None, speculative_ngram_external_sam_budget=0, speculative_ngram_external_corpus_max_tokens=10000000, ep_size=4, moe_a2a_backend='deepep', moe_runner_backend='auto', flashinfer_mxfp4_moe_precision='default', deepep_mode='auto', fuseep_mode=2, deepep_dispatcher_output_dtype='auto', ep_num_redundant_experts=0, ep_dispatch_algorithm=None, init_expert_location='trivial', enable_eplb=False, eplb_algorithm='auto', eplb_rebalance_num_iterations=1000, eplb_rebalance_layers_per_chunk=None, eplb_min_rebalancing_utilization_threshold=1.0, expert_distribution_recorder_mode=None, expert_distribution_recorder_buffer_size=1000, expert_balancedness_report_mode='off', deepep_config='{"normal_dispatch":{"num_sms":96},"normal_combine":{"num_sms":96}}', moe_dense_tp_size=None, elastic_ep_backend=None, enable_elastic_expert_backup=False, mooncake_ib_device=None, enable_waterfill=False, ep_join_mode=None, ep_join_rank_offset=0, elastic_ep_initial_size=None, max_ep_size=None, elastic_ep_scale_timeout=600, elastic_ep_rejoin=False, disable_flashinfer_cutlass_moe_fp4_allgather=False, disable_shared_experts_fusion=False, enforce_shared_experts_fusion=False, max_mamba_cache_size=None, mamba_ssm_dtype=None, mamba_max_states_per_path=-1, enable_mamba_cache_stochastic_rounding=False, mamba_cache_philox_rounds=0, mamba_full_memory_ratio=0.9, mamba_radix_cache_strategy='auto', uses_mamba_radix_cache=False, mamba_track_interval=256, enable_int8_mamba_checkpoint=False, int8_mamba_ckpt_size=None, linear_attn_backend='triton', linear_attn_decode_backend=None, linear_attn_prefill_backend=None, linear_attn_verify_backend=None, enable_linear_replayssm=False, linear_replayssm_cache_len=16, enable_linear_replayssm_spec=False, enable_hierarchical_cache=False, hicache_ratio=2.0, hicache_size=0, hicache_write_policy='write_through', hicache_io_backend='kernel', hicache_mem_layout='page_first', hicache_storage_backend=None, hicache_storage_prefetch_policy='timeout', hicache_storage_backend_extra_config=None, enable_hisparse=False, hisparse_config=None, enable_broadcast_mm_inputs_process=False, enable_prefix_mm_cache=False, mm_enable_dp_encoder=False, mm_process_config={}, mm_processor_worker_num=0, mm_io_worker_num=0, allowed_media_domains=[], media_url_max_file_size_mb=64, mm_preprocess_cache_size_mb=None, trust_mm_content_hashes=False, limit_mm_data_per_request=None, enable_mm_global_cache=False, image_processor_backend='auto', mm_global_cache_backend='mooncake', disable_fast_image_processor=False, mm_feature_transport='cpu', keep_mm_feature_on_device=False, enable_lora=None, enable_lora_overlap_loading=None, max_lora_rank=None, lora_target_modules=None, lora_paths=None, max_loaded_loras=None, max_loras_per_batch=8, lora_eviction_policy='lru', lora_backend='csgmv', max_lora_chunk_size=16, experts_shared_outer_loras=None, lora_use_virtual_experts=False, lora_strict_loading=False, lora_drain_wait_threshold=0.0, enable_two_batch_overlap=False, enable_single_batch_overlap=False, tbo_token_distribution_threshold=0.48, cpu_offload_gb=0, offload_group_size=-1, offload_num_in_group=1, offload_prefetch_step=1, offload_mode='cpu', enable_lmcache=False, lmcache_config_file=None, enable_flexkv=False, flexkv_config_file=None, kt_weight_path=None, kt_method='AMXINT4', kt_cpuinfer=None, kt_threadpool_count=2, kt_num_gpu_experts=None, kt_max_deferred_experts_per_token=None, dllm_algorithm=None, dllm_algorithm_config=None, dllm_fdfo=True, disaggregation_mode='null', disaggregation_transfer_backend='mooncake', disaggregation_bootstrap_port=8998, disaggregation_ib_device=None, disaggregation_decode_enable_radix_cache=False, disaggregation_decode_enable_offload_kvcache=False, disaggregation_decode_retraction_backup=None, num_reserved_decode_tokens=512, disaggregation_decode_extra_slots=None, disaggregation_decode_polling_interval=1, optimistic_prefill_attempts=0, encoder_only=False, language_only=False, language_model_only=False, encoder_transfer_backend='zmq_to_scheduler', encoder_urls=[], encoder_bootstrap_port=8997, encoder_register_urls=[], enable_adaptive_dispatch_to_encoder=False, enable_pdmux=False, pdmux_config_path=None, sm_group_num=8, startup_weight_load_mode='serial', custom_weight_loader=[], weight_loader_disable_mmap=False, weight_loader_prefetch_checkpoints=False, weight_loader_prefetch_num_threads=4, weight_loader_drop_cache_after_load=False, remote_instance_weight_loader_seed_instance_ip=None, remote_instance_weight_loader_seed_instance_service_port=None, remote_instance_weight_loader_send_weights_group_ports=None, remote_instance_weight_loader_backend='nccl', remote_instance_weight_loader_start_seed_via_transfer_engine=False, engine_info_bootstrap_port=6789, modelexpress_config=None, download_dir=None, model_checksum=None, delete_ckpt_after_loading=False, decrypted_config_file=None, decrypted_draft_config_file=None, checkpoint_engine_wait_weights_before_ready=False, enable_prefill_delayer=False, prefill_delayer_max_delay_passes=30, prefill_delayer_token_usage_low_watermark=None, prefill_delayer_forward_passes_buckets=None, prefill_delayer_wait_seconds_buckets=None, prefill_delayer_queue_min_ratio=None, prefill_delayer_max_delay_ms=None, min_free_slots_delay=None, enable_deterministic_inference=False, rl_on_policy_target=None, kv_canary='none', kv_canary_real_data='none', kv_canary_sweep_interval=0, enable_dynamic_batch_tokenizer=False, dynamic_batch_tokenizer_batch_size=32, dynamic_batch_tokenizer_batch_timeout=0.002, enable_tokenizer_batch_encode=False, disable_tokenizer_batch_decode=False, debug_tensor_dump_output_folder=None, debug_tensor_dump_layers=None, debug_tensor_dump_input_file=None, enable_memory_saver=False, enable_weights_cpu_backup=False, enable_draft_weights_cpu_backup=False, enable_custom_logit_processor=False, enable_return_hidden_states=False, return_hidden_states_mode=None, enable_return_routed_experts=False, enable_return_indexer_topk=False, disable_outlines_disk_cache=False, enable_mis=False, weight_cache_mode='off', weight_cache_socket=None, weight_cache_timeout=1800, forward_hooks=None, msprobe_dump_config=None)
-[2026-09-10 05:13:54] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:13:56] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:13:57] No HuggingFace chat template found
-[2026-09-10 05:13:57] No chat template found, defaulting to 'string' content format
-[2026-09-10 05:14:03] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command.
-[2026-09-10 05:14:03] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command.
-[2026-09-10 05:14:03] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command.
-[2026-09-10 05:14:03] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command.
-[2026-09-10 05:14:04] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:14:13 DP1 TP1 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:14:13 DP3 TP3 EP3] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:14:13 DP2 TP2 EP2] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:14:13 DP0 TP0 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:14:15 DP1 TP1 EP1] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:14:15 DP3 TP3 EP3] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:14:15 DP2 TP2 EP2] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:14:15 DP0 TP0 EP0] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:14:15 DP1 TP1 EP1] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:14:15 DP3 TP3 EP3] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:14:15 DP2 TP2 EP2] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:14:15 DP0 TP0 EP0] Auto-detected DSV4 routed-expert layout: is_fp4_experts=True
-[2026-09-10 05:14:17 DP1 TP1 EP1] Init torch distributed begin.
-[2026-09-10 05:14:17 DP0 TP0 EP0] Init torch distributed begin.
-[2026-09-10 05:14:17 DP3 TP3 EP3] Init torch distributed begin.
-[2026-09-10 05:14:17 DP2 TP2 EP2] Init torch distributed begin.
-[W910 05:14:37.039630932 socket.cpp:207] [c10d] The hostname of the client socket cannot be retrieved. err=-3
-[W910 05:14:37.043647589 socket.cpp:207] [c10d] The hostname of the client socket cannot be retrieved. err=-3
-[W910 05:14:37.056494789 socket.cpp:207] [c10d] The hostname of the client socket cannot be retrieved. err=-3
-[W910 05:14:37.064698745 socket.cpp:207] [c10d] The hostname of the client socket cannot be retrieved. err=-3
-[2026-09-10 05:14:38 DP0 TP0 EP0] sglang is using nccl==2.29.7
-[2026-09-10 05:15:02 DP0 TP0 EP0] All Reduce config: symmetric_memory = 40.01 MB, local_buffer = 4.00 MB, multicast = False
-[2026-09-10 05:15:02 DP0 TP0 EP0] Init torch distributed ends. elapsed=44.96 s, mem usage=1.40 GB
-[2026-09-10 05:15:02 DP3 TP3 EP3] Init torch distributed ends. elapsed=44.95 s, mem usage=1.28 GB
-[2026-09-10 05:15:02 DP2 TP2 EP2] Init torch distributed ends. elapsed=44.94 s, mem usage=1.44 GB
-[2026-09-10 05:15:02 DP1 TP1 EP1] Init torch distributed ends. elapsed=44.99 s, mem usage=1.44 GB
-[2026-09-10 05:15:05 DP0 TP0 EP0] Ignore import error when loading sglang.srt.models.sarashina2_vision: cannot import name 'MultimodalDataItem' from 'sglang.srt.managers.mm_utils' (/sgl-workspace/sglang/python/sglang/srt/managers/mm_utils.py)
-[2026-09-10 05:15:05 DP3 TP3 EP3] Ignore import error when loading sglang.srt.models.sarashina2_vision: cannot import name 'MultimodalDataItem' from 'sglang.srt.managers.mm_utils' (/sgl-workspace/sglang/python/sglang/srt/managers/mm_utils.py)
-[2026-09-10 05:15:05 DP1 TP1 EP1] Ignore import error when loading sglang.srt.models.sarashina2_vision: cannot import name 'MultimodalDataItem' from 'sglang.srt.managers.mm_utils' (/sgl-workspace/sglang/python/sglang/srt/managers/mm_utils.py)
-[2026-09-10 05:15:05 DP2 TP2 EP2] Ignore import error when loading sglang.srt.models.sarashina2_vision: cannot import name 'MultimodalDataItem' from 'sglang.srt.managers.mm_utils' (/sgl-workspace/sglang/python/sglang/srt/managers/mm_utils.py)
-[2026-09-10 05:15:09 DP2 TP2 EP2] Load weight begin. avail mem=265.58 GB
-[2026-09-10 05:15:09 DP1 TP1 EP1] Load weight begin. avail mem=265.58 GB
-[2026-09-10 05:15:09 DP0 TP0 EP0] Load weight begin. avail mem=265.62 GB
-[2026-09-10 05:15:09 DP0 TP0 EP0] Detected fp8 checkpoint.
-[2026-09-10 05:15:09 DP0 TP0 EP0] Config does not support fused shared expert(s). Shared experts fusion optimization is disabled.
-[2026-09-10 05:15:09 DP3 TP3 EP3] Load weight begin. avail mem=265.74 GB
-[2026-09-10 05:15:15 DP2 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=auto, quant_method=Fp8MoEMethod).
-[2026-09-10 05:15:15 DP1 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=auto, quant_method=Fp8MoEMethod).
-[2026-09-10 05:15:16 DP3 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=auto, quant_method=Fp8MoEMethod).
-[2026-09-10 05:15:16 DP0 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=auto, quant_method=Fp8MoEMethod).
-
Multi-thread loading shards:   0% Completed | 0/48 [00:00<?, ?it/s]
Multi-thread loading shards:   4% Completed | 2/48 [00:07<02:57,  3.87s/it]
Multi-thread loading shards:   6% Completed | 3/48 [00:08<02:01,  2.70s/it]
Multi-thread loading shards:   8% Completed | 4/48 [00:09<01:31,  2.07s/it]
Multi-thread loading shards:  10% Completed | 5/48 [00:10<01:12,  1.69s/it]
Multi-thread loading shards:  12% Completed | 6/48 [00:11<00:59,  1.42s/it]
Multi-thread loading shards:  15% Completed | 7/48 [00:12<00:54,  1.34s/it]
Multi-thread loading shards:  17% Completed | 8/48 [00:13<00:39,  1.00it/s]
Multi-thread loading shards:  19% Completed | 9/48 [00:13<00:35,  1.11it/s]
Multi-thread loading shards:  21% Completed | 10/48 [00:14<00:31,  1.21it/s]
Multi-thread loading shards:  23% Completed | 11/48 [00:15<00:36,  1.01it/s]
Multi-thread loading shards:  25% Completed | 12/48 [00:16<00:33,  1.09it/s]
Multi-thread loading shards:  27% Completed | 13/48 [00:17<00:29,  1.19it/s]
Multi-thread loading shards:  29% Completed | 14/48 [00:18<00:29,  1.17it/s]
Multi-thread loading shards:  31% Completed | 15/48 [00:18<00:28,  1.15it/s]
Multi-thread loading shards:  33% Completed | 16/48 [00:20<00:29,  1.08it/s]
Multi-thread loading shards:  35% Completed | 17/48 [00:29<01:44,  3.36s/it]
Multi-thread loading shards:  38% Completed | 18/48 [00:29<01:12,  2.43s/it]
Multi-thread loading shards:  40% Completed | 19/48 [00:30<00:55,  1.91s/it]
Multi-thread loading shards:  42% Completed | 20/48 [00:30<00:45,  1.61s/it]
Multi-thread loading shards:  44% Completed | 21/48 [00:32<00:40,  1.50s/it]
Multi-thread loading shards:  46% Completed | 22/48 [00:32<00:29,  1.13s/it]
Multi-thread loading shards:  48% Completed | 23/48 [00:33<00:26,  1.07s/it]
Multi-thread loading shards:  50% Completed | 24/48 [00:34<00:23,  1.02it/s]
Multi-thread loading shards:  52% Completed | 25/48 [00:35<00:23,  1.01s/it]
Multi-thread loading shards:  54% Completed | 26/48 [00:41<00:57,  2.63s/it]
Multi-thread loading shards:  56% Completed | 27/48 [00:45<01:00,  2.89s/it]
Multi-thread loading shards:  58% Completed | 28/48 [00:46<00:48,  2.42s/it]
Multi-thread loading shards:  60% Completed | 29/48 [00:47<00:39,  2.07s/it]
Multi-thread loading shards:  62% Completed | 30/48 [00:48<00:31,  1.76s/it]
Multi-thread loading shards:  65% Completed | 31/48 [00:49<00:26,  1.57s/it]
Multi-thread loading shards:  67% Completed | 32/48 [00:50<00:21,  1.34s/it]
Multi-thread loading shards:  69% Completed | 33/48 [00:51<00:19,  1.33s/it]
Multi-thread loading shards:  71% Completed | 34/48 [00:52<00:17,  1.22s/it]
Multi-thread loading shards:  73% Completed | 35/48 [00:53<00:14,  1.12s/it]
Multi-thread loading shards:  75% Completed | 36/48 [00:54<00:10,  1.18it/s]
Multi-thread loading shards:  77% Completed | 37/48 [00:54<00:07,  1.41it/s]
Multi-thread loading shards:  79% Completed | 38/48 [00:55<00:07,  1.41it/s]
Multi-thread loading shards:  81% Completed | 39/48 [00:55<00:06,  1.47it/s]
Multi-thread loading shards:  83% Completed | 40/48 [00:56<00:05,  1.46it/s]
Multi-thread loading shards:  85% Completed | 41/48 [00:57<00:04,  1.42it/s]
Multi-thread loading shards:  88% Completed | 42/48 [00:58<00:04,  1.30it/s]
Multi-thread loading shards:  90% Completed | 43/48 [00:58<00:03,  1.37it/s]
Multi-thread loading shards:  92% Completed | 44/48 [01:04<00:08,  2.25s/it]
Multi-thread loading shards: 100% Completed | 48/48 [01:04<00:00,  1.35s/it]
-[2026-09-10 05:16:25 DP3 TP3 EP3] DeepSeek V4 MHC prenorm prewarm: 21 n_splits buckets
-[2026-09-10 05:16:26 DP0 TP0 EP0] DeepSeek V4 MHC prenorm prewarm: 21 n_splits buckets
-[2026-09-10 05:16:26 DP2 TP2 EP2] DeepSeek V4 MHC prenorm prewarm: 21 n_splits buckets
-[2026-09-10 05:16:27 DP1 TP1 EP1] DeepSeek V4 MHC prenorm prewarm: 21 n_splits buckets
-[2026-09-10 05:20:53 DP2 TP2 EP2] DeepSeek V4 MHC prewarm at load: compile 266.2s, rank sync +0.4s
-[2026-09-10 05:20:53 DP0 TP0 EP0] DeepSeek V4 MHC prewarm at load: compile 266.7s, rank sync +0.2s
-[2026-09-10 05:20:53 DP3 TP3 EP3] DeepSeek V4 MHC prewarm at load: compile 267.2s, rank sync +0.3s
-[2026-09-10 05:20:53 DP1 TP1 EP1] DeepSeek V4 MHC prewarm at load: compile 265.5s, rank sync +0.0s
-[2026-09-10 05:20:57 DP1 TP1 EP1] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results!
-[2026-09-10 05:20:58 DP1 TP1 EP1] Load weight end. elapsed=348.44 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=222.61 GB, mem usage=42.97 GB.
-[2026-09-10 05:20:58 DP0 TP0 EP0] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results!
-[2026-09-10 05:20:58 DP0 TP0 EP0] Load weight end. elapsed=348.49 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=222.65 GB, mem usage=42.97 GB.
-[2026-09-10 05:20:58 DP2 TP2 EP2] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results!
-[2026-09-10 05:20:58 DP2 TP2 EP2] Load weight end. elapsed=348.84 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=222.61 GB, mem usage=42.97 GB.
-[2026-09-10 05:20:58 DP3 TP3 EP3] Using FP8 KV cache but no scaling factors provided. Defaulting to scaling factors of 1.0. This may lead to less accurate results!
-[2026-09-10 05:20:58 DP3 TP3 EP3] Load weight end. elapsed=348.59 s, type=DeepseekV4ForCausalLM, quant=fp8, fmt=e4m3, avail mem=222.77 GB, mem usage=42.97 GB.
-[2026-09-10 05:21:00 DP2 TP2 EP2] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:21:00 DP3 TP3 EP3] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:21:00 DP1 TP1 EP1] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:21:00 DP0 TP0 EP0] Tokenizer for /models is still TokenizersBackend after retries with --trust-remote-code. Model-specific tokenizer attributes may be missing.
-[2026-09-10 05:21:01 DP0 TP0 EP0] DSV4 memory calculation: bytes_per_full_token=7705.45, available_bytes=185.95 GB, c128_state_fixed=0.65 GB, full_token=25820928
-[2026-09-10 05:21:01 DP0 TP0 EP0] DSV4 pool sizes: full=25820928, swa=2582016, c4=6455232, c128=201726, c4_state=161376, c128_state=0
-[2026-09-10 05:21:01 DP3 TP3 EP3] DSV4 memory calculation: bytes_per_full_token=7705.45, available_bytes=185.95 GB, c128_state_fixed=0.65 GB, full_token=25820928
-[2026-09-10 05:21:01 DP3 TP3 EP3] DSV4 pool sizes: full=25820928, swa=2582016, c4=6455232, c128=201726, c4_state=161376, c128_state=0
-[2026-09-10 05:21:01 DP2 TP2 EP2] DSV4 memory calculation: bytes_per_full_token=7705.45, available_bytes=185.95 GB, c128_state_fixed=0.65 GB, full_token=25820928
-[2026-09-10 05:21:01 DP2 TP2 EP2] DSV4 pool sizes: full=25820928, swa=2582016, c4=6455232, c128=201726, c4_state=161376, c128_state=0
-[2026-09-10 05:21:01 DP0 TP0 EP0] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=64 swa_size=2582016 c4_size=6455232 c4_logical_size=6455232 c128_size=201726 c4_state_pool_size=161376 c128_state_pool_size=8320
-[2026-09-10 05:21:01 DP1 TP1 EP1] DSV4 memory calculation: bytes_per_full_token=7705.45, available_bytes=185.95 GB, c128_state_fixed=0.65 GB, full_token=25820928
-[2026-09-10 05:21:01 DP3 TP3 EP3] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=64 swa_size=2582016 c4_size=6455232 c4_logical_size=6455232 c128_size=201726 c4_state_pool_size=161376 c128_state_pool_size=8320
-[2026-09-10 05:21:01 DP1 TP1 EP1] DSV4 pool sizes: full=25820928, swa=2582016, c4=6455232, c128=201726, c4_state=161376, c128_state=0
-[2026-09-10 05:21:01 DP2 TP2 EP2] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=64 swa_size=2582016 c4_size=6455232 c4_logical_size=6455232 c128_size=201726 c4_state_pool_size=161376 c128_state_pool_size=8320
-[2026-09-10 05:21:01 DP1 TP1 EP1] Initialize DeepSeekV4TokenToKVPool with max_num_reqs=64 swa_size=2582016 c4_size=6455232 c4_logical_size=6455232 c128_size=201726 c4_state_pool_size=161376 c128_state_pool_size=8320
-[2026-09-10 05:21:01 DP1 TP1 EP1] Memory pool end. avail mem=34.80 GB
-[2026-09-10 05:21:01 DP2 TP2 EP2] Memory pool end. avail mem=34.80 GB
-[2026-09-10 05:21:01 DP0 TP0 EP0] Memory pool end. avail mem=34.83 GB
-[2026-09-10 05:21:01 DP3 TP3 EP3] Memory pool end. avail mem=34.96 GB
-[2026-09-10 05:21:11 DP1 TP1 EP1] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA).
-[2026-09-10 05:21:11 DP0 TP0 EP0] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA).
-[2026-09-10 05:21:11 DP3 TP3 EP3] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA).
-[2026-09-10 05:21:11 DP2 TP2 EP2] Using DeepseekV4AttnBackend for dsv4 attention backend (CUDA).
-[2026-09-10 05:21:11 DP0 TP0 EP0] DeepGEMM masked layout budget: 8.40 GiB from 33.58 GiB free.
-[2026-09-10 05:21:11 DP0 TP0 EP0] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules).
-[2026-09-10 05:21:11 DP3 TP3 EP3] DeepGEMM masked layout budget: 8.40 GiB from 33.58 GiB free.
-[2026-09-10 05:21:11 DP3 TP3 EP3] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules).
-[2026-09-10 05:21:11 DP3 TP3 EP3] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=33.74 GB
-[2026-09-10 05:21:11 DP2 TP2 EP2] DeepGEMM masked layout budget: 8.40 GiB from 33.58 GiB free.
-[2026-09-10 05:21:11 DP2 TP2 EP2] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules).
-[2026-09-10 05:21:11 DP0 TP0 EP0] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=33.61 GB
-[2026-09-10 05:21:11 DP2 TP2 EP2] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=33.58 GB
-[2026-09-10 05:21:11 DP1 TP1 EP1] DeepGEMM masked layout budget: 8.40 GiB from 33.58 GiB free.
-[2026-09-10 05:21:11 DP1 TP1 EP1] Disable prefill CUDA graph because cuda_graph_config resolved prefill.backend='disabled' (e.g. via --cuda-graph-backend-prefill=disabled or auto-disable rules).
-[2026-09-10 05:21:11 DP1 TP1 EP1] Capture target decode CUDA graph begin. backend=full, num_tokens_per_req=1, bs=[1, 2, 4, 8, 12, 16, 24, 32, 40, 48, 56, 64], avail mem=33.58 GB
-
  0%|          | 0/12 [00:00<?, ?it/s]
Capturing batches (bs=64 avail_mem=33.58 GB):   0%|          | 0/12 [00:00<?, ?it/s][2026-09-10 05:21:15 DP1 TP1 EP1] Disable CP decode attention TP
-[2026-09-10 05:21:15 DP3 TP3 EP3] Disable CP decode attention TP
-[2026-09-10 05:21:15 DP2 TP2 EP2] Disable CP decode attention TP
-[2026-09-10 05:21:15 DP0 TP0 EP0] Disable CP decode attention TP
-[2026-09-10 05:21:15 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:21:15 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=1536, K=4096, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:21:15 DP0 TP0 EP0] Required memory for warmup: 0.115234375GB, Available memory: 33.5611572265625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   8%|▊         | 1297/16384 [00:00<00:01, 12964.05it/s]
-
DeepGEMM warmup:  16%|█▋        | 2692/16384 [00:00<00:01, 13542.97it/s]
-
DeepGEMM warmup:  26%|██▌       | 4191/16384 [00:00<00:00, 14199.81it/s]
-
DeepGEMM warmup:  35%|███▍      | 5681/16384 [00:00<00:00, 14474.48it/s]
-
DeepGEMM warmup:  44%|████▍     | 7185/16384 [00:00<00:00, 14676.98it/s]
-
DeepGEMM warmup:  53%|█████▎    | 8693/16384 [00:00<00:00, 14812.42it/s]
-
DeepGEMM warmup:  62%|██████▏   | 10205/16384 [00:00<00:00, 14911.31it/s]
-
DeepGEMM warmup:  72%|███████▏  | 11717/16384 [00:00<00:00, 14975.66it/s]
-
DeepGEMM warmup:  81%|████████  | 13235/16384 [00:00<00:00, 15038.76it/s]
-
DeepGEMM warmup:  90%|█████████ | 14753/16384 [00:01<00:00, 15080.08it/s]
-
DeepGEMM warmup:  99%|█████████▉| 16269/16384 [00:01<00:00, 15101.63it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:01<00:00, 14783.46it/s]
-[2026-09-10 05:21:16 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:21:16 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=32768, K=1024, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:21:16 DP0 TP0 EP0] Required memory for warmup: 1.046875GB, Available memory: 33.5592041015625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   0%|          | 1/16384 [00:02<11:17:18,  2.48s/it]
-
DeepGEMM warmup:   3%|▎         | 458/16384 [00:02<01:03, 249.23it/s]
-
DeepGEMM warmup:   8%|▊         | 1362/16384 [00:02<00:16, 883.70it/s]
-
DeepGEMM warmup:  17%|█▋        | 2722/16384 [00:02<00:06, 2072.27it/s]
-
DeepGEMM warmup:  22%|██▏       | 3632/16384 [00:02<00:04, 2899.71it/s]
-
DeepGEMM warmup:  28%|██▊       | 4520/16384 [00:02<00:03, 3672.82it/s]
-
DeepGEMM warmup:  33%|███▎      | 5365/16384 [00:03<00:02, 4269.67it/s]
-
DeepGEMM warmup:  38%|███▊      | 6149/16384 [00:03<00:02, 4642.18it/s]
-
DeepGEMM warmup:  42%|████▏     | 6870/16384 [00:03<00:01, 4817.08it/s]
-
DeepGEMM warmup:  46%|████▌     | 7533/16384 [00:03<00:01, 4840.30it/s]
-
DeepGEMM warmup:  50%|████▉     | 8143/16384 [00:03<00:01, 4765.07it/s]
-
DeepGEMM warmup:  53%|█████▎    | 8706/16384 [00:03<00:01, 4639.51it/s]
-
DeepGEMM warmup:  56%|█████▋    | 9229/16384 [00:03<00:01, 4490.49it/s]
-
DeepGEMM warmup:  59%|█████▉    | 9718/16384 [00:04<00:01, 4324.61it/s]
-
DeepGEMM warmup:  62%|██████▏   | 10177/16384 [00:04<00:01, 4164.20it/s]
-
DeepGEMM warmup:  65%|██████▍   | 10611/16384 [00:04<00:01, 4005.14it/s]
-
DeepGEMM warmup:  67%|██████▋   | 11023/16384 [00:04<00:01, 3858.48it/s]
-
DeepGEMM warmup:  70%|██████▉   | 11416/16384 [00:04<00:01, 3719.39it/s]
-
DeepGEMM warmup:  72%|███████▏  | 11793/16384 [00:04<00:01, 3590.19it/s]
-
DeepGEMM warmup:  74%|███████▍  | 12155/16384 [00:04<00:01, 3475.27it/s]
-
DeepGEMM warmup:  76%|███████▋  | 12504/16384 [00:04<00:01, 3367.40it/s]
-
DeepGEMM warmup:  78%|███████▊  | 12842/16384 [00:04<00:01, 3271.93it/s]
-
DeepGEMM warmup:  80%|████████  | 13170/16384 [00:05<00:01, 3180.59it/s]
-
DeepGEMM warmup:  82%|████████▏ | 13488/16384 [00:05<00:00, 3097.02it/s]
-
DeepGEMM warmup:  84%|████████▍ | 13798/16384 [00:05<00:00, 3022.90it/s]
-
DeepGEMM warmup:  86%|████████▌ | 14100/16384 [00:05<00:00, 2952.58it/s]
-
DeepGEMM warmup:  88%|████████▊ | 14395/16384 [00:05<00:00, 2885.77it/s]
-
DeepGEMM warmup:  90%|████████▉ | 14684/16384 [00:05<00:00, 2825.40it/s]
-
DeepGEMM warmup:  91%|█████████▏| 14967/16384 [00:05<00:00, 2767.95it/s]
-
DeepGEMM warmup:  93%|█████████▎| 15244/16384 [00:05<00:00, 2716.79it/s]
-
DeepGEMM warmup:  95%|█████████▍| 15516/16384 [00:05<00:00, 2667.84it/s]
-
DeepGEMM warmup:  96%|█████████▋| 15783/16384 [00:06<00:00, 2620.04it/s]
-
DeepGEMM warmup:  98%|█████████▊| 16045/16384 [00:06<00:00, 2574.49it/s]
-
DeepGEMM warmup: 100%|█████████▉| 16303/16384 [00:06<00:00, 2533.75it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:06<00:00, 2599.03it/s]
-[2026-09-10 05:21:23 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:21:23 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=4096, K=8192, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:21:23 DP0 TP0 EP0] Required memory for warmup: 0.28125GB, Available memory: 33.5572509765625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   9%|▉         | 1486/16384 [00:00<00:01, 14855.37it/s]
-
DeepGEMM warmup:  18%|█▊        | 3007/16384 [00:00<00:00, 15062.88it/s]
-
DeepGEMM warmup:  28%|██▊       | 4514/16384 [00:00<00:00, 14076.07it/s]
-
DeepGEMM warmup:  36%|███▌      | 5929/16384 [00:00<00:00, 11924.31it/s]
-
DeepGEMM warmup:  44%|████▎     | 7162/16384 [00:00<00:00, 10248.82it/s]
-
DeepGEMM warmup:  50%|█████     | 8236/16384 [00:00<00:00, 9005.50it/s] 
-
DeepGEMM warmup:  56%|█████▌    | 9180/16384 [00:00<00:00, 8091.20it/s]
-
DeepGEMM warmup:  61%|██████    | 10021/16384 [00:01<00:00, 7343.98it/s]
-
DeepGEMM warmup:  66%|██████▌   | 10778/16384 [00:01<00:00, 6737.52it/s]
-
DeepGEMM warmup:  70%|██████▉   | 11466/16384 [00:01<00:00, 6260.56it/s]
-
DeepGEMM warmup:  74%|███████▍  | 12100/16384 [00:01<00:00, 5837.64it/s]
-
DeepGEMM warmup:  77%|███████▋  | 12687/16384 [00:01<00:00, 5487.44it/s]
-
DeepGEMM warmup:  81%|████████  | 13236/16384 [00:01<00:00, 5196.62it/s]
-
DeepGEMM warmup:  84%|████████▍ | 13754/16384 [00:01<00:00, 4949.03it/s]
-
DeepGEMM warmup:  87%|████████▋ | 14246/16384 [00:01<00:00, 4723.32it/s]
-
DeepGEMM warmup:  90%|████████▉ | 14716/16384 [00:02<00:00, 4535.07it/s]
-
DeepGEMM warmup:  93%|█████████▎| 15167/16384 [00:02<00:00, 4346.54it/s]
-
DeepGEMM warmup:  95%|█████████▌| 15599/16384 [00:02<00:00, 4211.03it/s]
-
DeepGEMM warmup:  98%|█████████▊| 16018/16384 [00:02<00:00, 4049.65it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:02<00:00, 6489.53it/s]
-[2026-09-10 05:21:25 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:21:25 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=4096, K=4096, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:21:25 DP0 TP0 EP0] Required memory for warmup: 0.203125GB, Available memory: 33.5240478515625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   9%|▉         | 1454/16384 [00:00<00:01, 14530.87it/s]
-
DeepGEMM warmup:  18%|█▊        | 2932/16384 [00:00<00:00, 14675.18it/s]
-
DeepGEMM warmup:  27%|██▋       | 4417/16384 [00:00<00:00, 14753.44it/s]
-
DeepGEMM warmup:  36%|███▌      | 5915/16384 [00:00<00:00, 14842.47it/s]
-
DeepGEMM warmup:  45%|████▌     | 7402/16384 [00:00<00:00, 14850.00it/s]
-
DeepGEMM warmup:  54%|█████▍    | 8888/16384 [00:00<00:00, 13514.91it/s]
-
DeepGEMM warmup:  63%|██████▎   | 10260/16384 [00:00<00:00, 12257.98it/s]
-
DeepGEMM warmup:  70%|███████   | 11515/16384 [00:00<00:00, 11187.36it/s]
-
DeepGEMM warmup:  77%|███████▋  | 12664/16384 [00:01<00:00, 10272.88it/s]
-
DeepGEMM warmup:  84%|████████▎ | 13718/16384 [00:01<00:00, 9529.04it/s] 
-
DeepGEMM warmup:  90%|████████▉ | 14691/16384 [00:01<00:00, 8892.17it/s]
-
DeepGEMM warmup:  95%|█████████▌| 15595/16384 [00:01<00:00, 8331.99it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:01<00:00, 10591.25it/s]
-[2026-09-10 05:21:27 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:21:27 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=4096, K=2048, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:21:27 DP0 TP0 EP0] Required memory for warmup: 0.1640625GB, Available memory: 33.5240478515625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   9%|▉         | 1477/16384 [00:00<00:01, 14763.61it/s]
-
DeepGEMM warmup:  18%|█▊        | 2996/16384 [00:00<00:00, 15012.73it/s]
-
DeepGEMM warmup:  28%|██▊       | 4604/16384 [00:00<00:00, 15497.79it/s]
-
DeepGEMM warmup:  38%|███▊      | 6154/16384 [00:00<00:00, 15235.04it/s]
-
DeepGEMM warmup:  47%|████▋     | 7679/16384 [00:00<00:00, 15124.89it/s]
-
DeepGEMM warmup:  56%|█████▌    | 9192/16384 [00:00<00:00, 15061.76it/s]
-
DeepGEMM warmup:  65%|██████▌   | 10709/16384 [00:00<00:00, 15093.96it/s]
-
DeepGEMM warmup:  75%|███████▍  | 12226/16384 [00:00<00:00, 15117.13it/s]
-
DeepGEMM warmup:  84%|████████▍ | 13738/16384 [00:00<00:00, 14745.56it/s]
-
DeepGEMM warmup:  93%|█████████▎| 15215/16384 [00:01<00:00, 13858.29it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:01<00:00, 14373.58it/s]
-[2026-09-10 05:21:30 DP0 TP0 EP0] Use DeepEP Config: {'normal_dispatch': {'num_sms': 96}, 'normal_combine': {'num_sms': 96}}
-/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA
-/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA
-/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA
-/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA
-[2026-09-10 05:21:48 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:21:48 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GROUPED_GEMM_NT_F8F8BF16_MASKED> N=4096, K=4096, num_groups=64 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:21:48 DP0 TP0 EP0] Required memory for warmup: 13.000000238418579GB, Available memory: 15.4010009765625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   0%|          | 1/16384 [00:02<11:15:04,  2.47s/it]
-
DeepGEMM warmup:   3%|▎         | 554/16384 [00:02<00:52, 302.45it/s]
-
DeepGEMM warmup:   7%|▋         | 1164/16384 [00:02<00:21, 722.08it/s]
-
DeepGEMM warmup:  10%|█         | 1665/16384 [00:02<00:13, 1123.88it/s]
-
DeepGEMM warmup:  13%|█▎        | 2166/16384 [00:02<00:08, 1582.55it/s]
-
DeepGEMM warmup:  16%|█▋        | 2667/16384 [00:02<00:06, 2076.91it/s]
-
DeepGEMM warmup:  19%|█▉        | 3168/16384 [00:03<00:05, 2577.76it/s]
-
DeepGEMM warmup:  22%|██▏       | 3669/16384 [00:03<00:04, 3054.29it/s]
-
DeepGEMM warmup:  25%|██▌       | 4170/16384 [00:03<00:03, 3481.72it/s]
-
DeepGEMM warmup:  29%|██▊       | 4671/16384 [00:03<00:03, 3844.79it/s]
-
DeepGEMM warmup:  32%|███▏      | 5172/16384 [00:03<00:02, 4139.67it/s]
-
DeepGEMM warmup:  35%|███▍      | 5673/16384 [00:03<00:02, 4370.39it/s]
-
DeepGEMM warmup:  38%|███▊      | 6174/16384 [00:03<00:02, 4545.72it/s]
-
DeepGEMM warmup:  41%|████      | 6675/16384 [00:03<00:02, 4675.99it/s]
-
DeepGEMM warmup:  44%|████▍     | 7176/16384 [00:03<00:01, 4770.55it/s]
-
DeepGEMM warmup:  47%|████▋     | 7677/16384 [00:03<00:01, 4839.04it/s]
-
DeepGEMM warmup:  50%|████▉     | 8178/16384 [00:04<00:01, 4887.60it/s]
-
DeepGEMM warmup:  53%|█████▎    | 8679/16384 [00:04<00:01, 4922.57it/s]
-
DeepGEMM warmup:  56%|█████▌    | 9180/16384 [00:04<00:01, 4947.20it/s]
-
DeepGEMM warmup:  59%|█████▉    | 9681/16384 [00:04<00:01, 4964.71it/s]
-
DeepGEMM warmup:  62%|██████▏   | 10182/16384 [00:04<00:01, 4977.09it/s]
-
DeepGEMM warmup:  65%|██████▌   | 10683/16384 [00:04<00:01, 4985.67it/s]
-
DeepGEMM warmup:  68%|██████▊   | 11184/16384 [00:04<00:01, 4991.72it/s]
-
DeepGEMM warmup:  71%|███████▏  | 11685/16384 [00:04<00:00, 4995.63it/s]
-
DeepGEMM warmup:  74%|███████▍  | 12186/16384 [00:04<00:00, 4998.67it/s]
-
DeepGEMM warmup:  77%|███████▋  | 12687/16384 [00:04<00:00, 5000.42it/s]
-
DeepGEMM warmup:  80%|████████  | 13188/16384 [00:05<00:00, 5001.98it/s]
-
DeepGEMM warmup:  84%|████████▎ | 13689/16384 [00:05<00:00, 5003.14it/s]
-
DeepGEMM warmup:  87%|████████▋ | 14190/16384 [00:05<00:00, 5004.09it/s]
-
DeepGEMM warmup:  90%|████████▉ | 14691/16384 [00:05<00:00, 5004.11it/s]
-
DeepGEMM warmup:  93%|█████████▎| 15192/16384 [00:05<00:00, 5004.39it/s]
-
DeepGEMM warmup:  96%|█████████▌| 15693/16384 [00:05<00:00, 5004.41it/s]
-
DeepGEMM warmup:  99%|█████████▉| 16194/16384 [00:05<00:00, 5004.96it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:05<00:00, 2867.77it/s]
-[2026-09-10 05:21:56 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:21:56 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GROUPED_GEMM_NT_F8F8BF16_MASKED> N=4096, K=2048, num_groups=64 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:21:56 DP0 TP0 EP0] Required memory for warmup: 10.500000238418579GB, Available memory: 14.9010009765625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   0%|          | 1/16384 [00:02<11:52:25,  2.61s/it]
-
DeepGEMM warmup:   3%|▎         | 561/16384 [00:02<00:54, 291.05it/s]
-
DeepGEMM warmup:  10%|▉         | 1612/16384 [00:02<00:14, 996.48it/s]
-
DeepGEMM warmup:  17%|█▋        | 2835/16384 [00:02<00:06, 2004.81it/s]
-
DeepGEMM warmup:  24%|██▍       | 3947/16384 [00:03<00:04, 3040.54it/s]
-
DeepGEMM warmup:  31%|███       | 5058/16384 [00:03<00:02, 4167.54it/s]
-
DeepGEMM warmup:  38%|███▊      | 6169/16384 [00:03<00:01, 5322.13it/s]
-
DeepGEMM warmup:  44%|████▍     | 7280/16384 [00:03<00:01, 6432.93it/s]
-
DeepGEMM warmup:  51%|█████     | 8391/16384 [00:03<00:01, 7439.27it/s]
-
DeepGEMM warmup:  58%|█████▊    | 9502/16384 [00:03<00:00, 8301.36it/s]
-
DeepGEMM warmup:  65%|██████▍   | 10613/16384 [00:03<00:00, 9006.57it/s]
-
DeepGEMM warmup:  72%|███████▏  | 11724/16384 [00:03<00:00, 9561.25it/s]
-
DeepGEMM warmup:  78%|███████▊  | 12835/16384 [00:03<00:00, 9984.85it/s]
-
DeepGEMM warmup:  85%|████████▌ | 13946/16384 [00:03<00:00, 10300.75it/s]
-
DeepGEMM warmup:  92%|█████████▏| 15057/16384 [00:04<00:00, 10531.93it/s]
-
DeepGEMM warmup:  99%|█████████▊| 16168/16384 [00:04<00:00, 10699.11it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:04<00:00, 3967.77it/s] 
-[2026-09-10 05:22:09 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:22:09 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GEMM_NT_F8F8BF16> N=8192, K=1024, num_groups=1 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:22:09 DP0 TP0 EP0] Required memory for warmup: 0.2734375GB, Available memory: 18.3365478515625GB
-
-
DeepGEMM warmup:   0%|          | 0/16384 [00:00<?, ?it/s]
-
DeepGEMM warmup:   9%|▊         | 1433/16384 [00:00<00:01, 14327.55it/s]
-
DeepGEMM warmup:  18%|█▊        | 2892/16384 [00:00<00:00, 14478.26it/s]
-
DeepGEMM warmup:  27%|██▋       | 4356/16384 [00:00<00:00, 14549.86it/s]
-
DeepGEMM warmup:  36%|███▌      | 5836/16384 [00:00<00:00, 14647.96it/s]
-
DeepGEMM warmup:  45%|████▍     | 7325/16384 [00:00<00:00, 14734.75it/s]
-
DeepGEMM warmup:  54%|█████▍    | 8818/16384 [00:00<00:00, 14800.87it/s]
-
DeepGEMM warmup:  63%|██████▎   | 10299/16384 [00:00<00:00, 13990.31it/s]
-
DeepGEMM warmup:  71%|███████▏  | 11707/16384 [00:00<00:00, 12950.24it/s]
-
DeepGEMM warmup:  79%|███████▉  | 13020/16384 [00:00<00:00, 12004.36it/s]
-
DeepGEMM warmup:  87%|████████▋ | 14241/16384 [00:01<00:00, 11176.50it/s]
-
DeepGEMM warmup:  94%|█████████▍| 15378/16384 [00:01<00:00, 10470.10it/s]
DeepGEMM warmup: 100%|██████████| 16384/16384 [00:01<00:00, 12148.19it/s]
-
Capturing batches (bs=64 avail_mem=33.58 GB):   8%|▊         | 1/12 [01:08<12:33, 68.48s/it]
Capturing batches (bs=56 avail_mem=14.63 GB):   8%|▊         | 1/12 [01:08<12:33, 68.48s/it]
Capturing batches (bs=56 avail_mem=14.63 GB):  17%|█▋        | 2/12 [01:09<04:45, 28.56s/it]
Capturing batches (bs=48 avail_mem=14.61 GB):  17%|█▋        | 2/12 [01:09<04:45, 28.56s/it]
Capturing batches (bs=48 avail_mem=14.61 GB):  25%|██▌       | 3/12 [01:09<02:22, 15.80s/it]
Capturing batches (bs=40 avail_mem=14.59 GB):  25%|██▌       | 3/12 [01:09<02:22, 15.80s/it]
Capturing batches (bs=40 avail_mem=14.59 GB):  33%|███▎      | 4/12 [01:10<01:18,  9.81s/it]
Capturing batches (bs=32 avail_mem=14.58 GB):  33%|███▎      | 4/12 [01:10<01:18,  9.81s/it]
Capturing batches (bs=32 avail_mem=14.58 GB):  42%|████▏     | 5/12 [01:18<01:04,  9.26s/it]
Capturing batches (bs=24 avail_mem=14.56 GB):  42%|████▏     | 5/12 [01:18<01:04,  9.26s/it]
Capturing batches (bs=24 avail_mem=14.56 GB):  50%|█████     | 6/12 [01:19<00:37,  6.31s/it]
Capturing batches (bs=16 avail_mem=14.54 GB):  50%|█████     | 6/12 [01:19<00:37,  6.31s/it]
Capturing batches (bs=16 avail_mem=14.54 GB):  58%|█████▊    | 7/12 [01:20<00:22,  4.52s/it]
Capturing batches (bs=12 avail_mem=14.52 GB):  58%|█████▊    | 7/12 [01:20<00:22,  4.52s/it]
Capturing batches (bs=12 avail_mem=14.52 GB):  67%|██████▋   | 8/12 [01:20<00:13,  3.27s/it]
Capturing batches (bs=8 avail_mem=14.50 GB):  67%|██████▋   | 8/12 [01:20<00:13,  3.27s/it] 
Capturing batches (bs=8 avail_mem=14.50 GB):  75%|███████▌  | 9/12 [01:21<00:07,  2.44s/it]
Capturing batches (bs=4 avail_mem=14.49 GB):  75%|███████▌  | 9/12 [01:21<00:07,  2.44s/it]
Capturing batches (bs=4 avail_mem=14.49 GB):  83%|████████▎ | 10/12 [01:26<00:06,  3.44s/it]
Capturing batches (bs=2 avail_mem=14.47 GB):  83%|████████▎ | 10/12 [01:26<00:06,  3.44s/it]
Capturing batches (bs=2 avail_mem=14.47 GB):  92%|█████████▏| 11/12 [01:27<00:02,  2.56s/it]
Capturing batches (bs=1 avail_mem=14.46 GB):  92%|█████████▏| 11/12 [01:27<00:02,  2.56s/it]
Capturing batches (bs=1 avail_mem=14.46 GB): 100%|██████████| 12/12 [01:28<00:00,  2.10s/it]
Capturing batches (bs=1 avail_mem=14.46 GB): 100%|██████████| 12/12 [01:28<00:00,  7.38s/it]
-[2026-09-10 05:22:41 DP0 TP0 EP0] Capture target decode CUDA graph end. elapsed=89.42 s, mem usage=19.18 GB, avail mem=14.44 GB.
-[2026-09-10 05:22:41 DP0 TP0 EP0] max_total_num_tokens=25820928, chunked_prefill_size=4096, max_prefill_tokens=16384, max_running_requests=64, context_len=1048576, available_gpu_mem=14.44 GB
-[2026-09-10 05:22:41 DP0 TP0 EP0] Init Unified Radix Cache. Components: (<ComponentType.FULL: 0>, <ComponentType.SWA: 1>). Tree Core: UnifiedTreeCore
-[2026-09-10 05:22:41 DP0 TP0 EP0] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hicache_attached=False streaming_wrapped=False
-[2026-09-10 05:22:41 DP2 TP2 EP2] Capture target decode CUDA graph end. elapsed=89.43 s, mem usage=19.33 GB, avail mem=14.25 GB.
-[2026-09-10 05:22:41 DP2 TP2 EP2] Init Unified Radix Cache. Components: (<ComponentType.FULL: 0>, <ComponentType.SWA: 1>). Tree Core: UnifiedTreeCore
-[2026-09-10 05:22:41 DP2 TP2 EP2] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hicache_attached=False streaming_wrapped=False
-[2026-09-10 05:22:41 DP1 TP1 EP1] Capture target decode CUDA graph end. elapsed=89.44 s, mem usage=19.33 GB, avail mem=14.25 GB.
-[2026-09-10 05:22:41 DP3 TP3 EP3] Capture target decode CUDA graph end. elapsed=89.44 s, mem usage=18.55 GB, avail mem=15.19 GB.
-[2026-09-10 05:22:41 DP1 TP1 EP1] Init Unified Radix Cache. Components: (<ComponentType.FULL: 0>, <ComponentType.SWA: 1>). Tree Core: UnifiedTreeCore
-[2026-09-10 05:22:41 DP1 TP1 EP1] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hicache_attached=False streaming_wrapped=False
-[2026-09-10 05:22:41 DP3 TP3 EP3] Init Unified Radix Cache. Components: (<ComponentType.FULL: 0>, <ComponentType.SWA: 1>). Tree Core: UnifiedTreeCore
-[2026-09-10 05:22:41 DP3 TP3 EP3] Tree cache initialized: source=default impl=UnifiedRadixCache hybrid_swa=True hybrid_ssm=False hicache_attached=False streaming_wrapped=False
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-[2026-09-10 05:22:41] Engine startup timings (s): load_weight=348.84, kv_cache_allocation=0.90, scheduler_e2e=508.14, cuda_graph={prefill=0.00, decode=89.44, target_verify=0.00, draft_prefill=0.00, draft_decode=0.00, draft_extend=0.00}, tokenizer_e2e=526.95
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-[2026-09-10 05:22:41] INFO:     Started server process [1]
-[2026-09-10 05:22:41] INFO:     Waiting for application startup.
-[2026-09-10 05:22:41] Using default chat sampling params from model generation config: {'temperature': 1.0, 'top_p': 1.0}
-[2026-09-10 05:22:41] INFO:     Application startup complete.
-[2026-09-10 05:22:41] INFO:     Uvicorn running on http://0.0.0.0:30000 (Press CTRL+C to quit)
-[2026-09-10 05:22:42] INFO:     127.0.0.1:49410 - "GET /model_info HTTP/1.1" 200 OK
-[2026-09-10 05:22:44 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:22:44 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GROUPED_GEMM_NT_F8F8BF16_CONTIG> N=4096, K=4096, num_groups=64 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:22:44 DP0 TP0 EP0] Required memory for warmup: 1.18756103515625GB, Available memory: 14.3795166015625GB
-
DeepGEMM warmup:   0%|          | 0/128 [00:00<?, ?it/s]
DeepGEMM warmup:   1%|          | 1/128 [00:02<05:23,  2.55s/it]
DeepGEMM warmup: 100%|██████████| 128/128 [00:02<00:00, 49.61it/s]
-[2026-09-10 05:22:47 DP0 TP0 EP0] Entering DeepGEMM JIT Pre-Compile session. It may take a long time (typically 10-20 mins) if you have not run `sglang.compile_deep_gemm`. It is recommended to run `sglang.compile_deep_gemm` with same args as `sglang.launch_server` for pre-compilation to reduce the overhead if you have not run it before. For example: `python3 -m sglang.compile_deep_gemm --model deepseek-ai/DeepSeek-V3 --tp 8 --trust-remote-code`
-[2026-09-10 05:22:47 DP0 TP0 EP0] Try DeepGEMM JIT Compiling for <GROUPED_GEMM_NT_F8F8BF16_CONTIG> N=4096, K=2048, num_groups=64 with all Ms. It only takes a little time (typically 1 sec) if you have run `python3 -m sglang.compile_deep_gemm`. 
-[2026-09-10 05:22:47 DP0 TP0 EP0] Required memory for warmup: 0.65631103515625GB, Available memory: 14.3795166015625GB
-[2026-09-10 05:22:49] INFO:     172.17.0.1:60716 - "GET /health HTTP/1.1" 503 Service Unavailable
-
DeepGEMM warmup:   0%|          | 0/128 [00:00<?, ?it/s]
DeepGEMM warmup:   1%|          | 1/128 [00:02<05:22,  2.54s/it]
DeepGEMM warmup: 100%|██████████| 128/128 [00:02<00:00, 49.70it/s]
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-[2026-09-10 05:22:57 DP3 TP3 EP3] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 15.69
-[2026-09-10 05:22:57 DP1 TP1 EP1] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 15.67
-[2026-09-10 05:22:57 DP2 TP2 EP2] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 15.64
-[2026-09-10 05:22:57 DP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 15.62
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-[2026-09-10 05:22:57] INFO:     127.0.0.1:49426 - "POST /generate HTTP/1.1" 200 OK
-[2026-09-10 05:22:57 DP0 TP0 EP0] Freezing GC in Scheduler process. gen0: 54->0, gen1: 1430->0, gen2: 1147859->0
-[2026-09-10 05:22:57 DP2 TP2 EP2] Freezing GC in Scheduler process. gen0: 54->0, gen1: 1378->0, gen2: 1147916->0
-[2026-09-10 05:22:57 DP3 TP3 EP3] Freezing GC in Scheduler process. gen0: 54->0, gen1: 1160->0, gen2: 1148118->0
-[2026-09-10 05:22:57 DP1 TP1 EP1] Freezing GC in Scheduler process. gen0: 54->0, gen1: 1431->0, gen2: 1147879->0
-[2026-09-10 05:22:57] INFO:     127.0.0.1:50866 - "POST /freeze_gc HTTP/1.1" 200 OK
-[2026-09-10 05:22:57] Freezing GC in Tokenizer Manager process. gen0: 320->0, gen1: 1997->0, gen2: 1031011->0
-[2026-09-10 05:22:57] The server is fired up and ready to roll!
-[2026-09-10 05:22:57] Freezing GC in Detokenizer Manager process. gen0: 351->0, gen1: 2->0, gen2: 851786->0
-[2026-09-10 05:22:57] Freezing GC in Detokenizer Manager process. gen0: 1->0, gen1: 0->0, gen2: 0->0
-[2026-09-10 05:22:57] Freezing GC in Detokenizer Manager process. gen0: 1->0, gen1: 0->0, gen2: 0->0
-[2026-09-10 05:22:57] Freezing GC in Detokenizer Manager process. gen0: 1->0, gen1: 0->0, gen2: 0->0
-/usr/local/lib/python3.12/dist-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead.
-  return func(*args, **kwargs)
-[2026-09-10 05:23:00 DP0 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 256, #cached-token: 0, full token usage: 0.00, swa token usage: 0.00, #running-req: 0, #queue-req: 0, #pending-token: 0, cuda graph: False, input throughput (token/s): 94.08
-[2026-09-10 05:23:00] INFO:     172.17.0.1:54214 - "GET /health HTTP/1.1" 200 OK
+Error response from daemon: can not get logs from container which is dead or marked for removal
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server_info.json b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server_info.json
index 9b77150..e69de29 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server_info.json
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server_info.json
@@ -1 +0,0 @@
-{"model_path":"/models","tokenizer_path":"/models","tokenizer_mode":"auto","tokenizer_backend":"huggingface","tokenizer_worker_num":1,"detokenizer_worker_num":1,"skip_tokenizer_init":false,"load_format":"auto","model_loader_extra_config":"{}","trust_remote_code":true,"context_length":null,"is_embedding":false,"enable_multimodal":null,"revision":null,"model_impl":"auto","model_config_parser":"auto","json_model_override_args":"{}","dtype":"auto","quantization":null,"quantization_param_path":null,"kv_cache_dtype":"fp8_e4m3","enable_fp32_lm_head":false,"modelopt_quant":null,"modelopt_checkpoint_restore_path":null,"modelopt_checkpoint_save_path":null,"modelopt_export_path":null,"quantize_and_serve":false,"rl_quant_profile":null,"enable_tf32_matmul":false,"mem_fraction_static":0.862,"max_running_requests":256,"max_queued_requests":null,"max_total_tokens":null,"chunked_prefill_size":4096,"enable_dynamic_chunking":false,"prefill_reserve_budget":0,"max_prefill_tokens":16384,"prefill_max_requests":null,"schedule_policy":"fcfs","enable_priority_scheduling":false,"disable_priority_preemption":false,"default_priority_value":null,"abort_on_priority_when_disabled":false,"schedule_low_priority_values_first":false,"priority_scheduling_preemption_threshold":10,"retraction_policy":"length","schedule_conservativeness":0.3,"page_size":256,"c128_page_size":16,"swa_full_tokens_ratio":0.1,"disable_hybrid_swa_memory":false,"radix_eviction_policy":"lru","prefill_only_disable_kv_cache":false,"disable_radix_cache":false,"enable_page_major_kv_layout":false,"enable_unified_memory":false,"disable_chunked_prefix_cache":false,"disable_overlap_schedule":false,"num_continuous_decode_steps":1,"scheduler_recv_interval":1,"enable_mixed_chunk":false,"nccl_port":null,"dist_timeout":null,"dist_init_addr":null,"nnodes":1,"node_rank":0,"tp_size":4,"dcp_size":1,"pp_size":1,"pp_max_micro_batch_size":null,"pp_async_batch_depth":0,"dp_size":4,"load_balance_method":"round_robin","prefix_affinity_fallback":"total_tokens","prefix_affinity_max_load_skew":1.5,"prefix_affinity_hash_tokens":4096,"prefix_affinity_disable_token_fallback":false,"attn_cp_size":1,"moe_dp_size":1,"dwdp_size":1,"dcp_comm_backend":"ag_rs","dcp_replicate_q_proj":null,"enable_prefill_cp":false,"cp_strategy":null,"enable_dsa_cache_layer_split":false,"enable_dsa_prefill_context_parallel":false,"dsa_prefill_cp_mode":"round-robin-split","enable_prefill_context_parallel":false,"prefill_cp_mode":"in-seq-split","enable_cp_decode_attn_tp":false,"enable_dp_attention":true,"enable_dp_attention_local_control_broadcast":false,"enable_dp_lm_head":false,"enable_tp_lm_head_all_to_all":false,"enable_attn_tp_input_scattered":false,"disable_attn_tp_gather":false,"enable_p2p_check":false,"device":"cuda","base_gpu_id":0,"gpu_id_step":1,"random_seed":903930395,"mlx_enable_sampling":false,"watchdog_timeout":300,"soft_watchdog_timeout":null,"sleep_on_idle":false,"use_ray":false,"custom_sigquit_handler":null,"numa_node":null,"gc_threshold":null,"host":"0.0.0.0","port":30000,"fastapi_root_path":"","smg_grpc_mode":false,"grpc_mode":false,"grpc_port":null,"sidecar":null,"sidecar_args":null,"skip_server_warmup":false,"warmups":null,"enable_http2":false,"http2_max_concurrent_streams":200,"ssl_keyfile":null,"ssl_certfile":null,"ssl_ca_certs":null,"ssl_keyfile_password":null,"enable_ssl_refresh":false,"api_key":null,"admin_api_key":null,"served_model_name":"/models","weight_version":"default","chat_template":null,"hf_chat_template_name":null,"completion_template":null,"file_storage_path":"sglang_storage","enable_cache_report":false,"reasoning_parser":"deepseek-v4","default_chat_template_kwargs":null,"strip_thinking_cache":false,"enable_strict_thinking":false,"tool_call_parser":"deepseekv4","tool_server":null,"sampling_defaults":"model","asr_max_buffer_seconds":60,"asr_max_concurrent_sessions":32,"preferred_sampling_params":null,"allow_auto_truncate":false,"stream_interval":1,"batch_notify_size":16,"stream_response_default_include_usage":false,"incremental_streaming_output":false,"enable_streaming_session":false,"enable_session_radix_cache":false,"log_level":"info","log_level_http":null,"log_requests":false,"log_requests_level":2,"log_requests_format":"text","log_requests_target":null,"uvicorn_access_log_exclude_prefixes":[],"crash_dump_folder":null,"show_time_cost":false,"enable_metrics":false,"smg_http_sidecar_port":null,"enable_mfu_metrics":false,"enable_metrics_for_all_schedulers":false,"load_snapshot_publish_interval":15,"tokenizer_metrics_custom_labels_header":"x-custom-labels","tokenizer_metrics_allowed_custom_labels":null,"extra_metric_labels":null,"bucket_time_to_first_token":null,"bucket_inter_token_latency":null,"bucket_e2e_request_latency":null,"prompt_tokens_buckets":null,"generation_tokens_buckets":null,"gc_warning_threshold_secs":0.0,"decode_log_interval":40,"enable_request_time_stats_logging":false,"kv_events_config":null,"enable_forward_pass_metrics":false,"forward_pass_metrics_worker_id":"","forward_pass_metrics_ipc_name":null,"enable_trace":false,"trace_modules":"request","otlp_traces_endpoint":"localhost:4317","export_metrics_to_file":false,"export_metrics_to_file_dir":null,"stat_loggers":null,"constrained_json_whitespace_pattern":null,"constrained_json_disable_any_whitespace":false,"attention_backend":"dsv4","decode_attention_backend":null,"prefill_attention_backend":null,"sampling_backend":"flashinfer","grammar_backend":"xgrammar","radix_cache_backend":null,"mm_attention_backend":null,"fp8_gemm_runner_backend":"auto","fp4_gemm_runner_backend":"auto","bf16_gemm_backend":"auto","dsa_prefill_backend":null,"dsa_decode_backend":null,"dsa_paged_mqa_logits_backend":"auto","dsa_topk_backend":"sgl-kernel","disable_flashinfer_autotune":false,"flashinfer_autotune_skip_ops":null,"mamba_backend":"triton","cuda_graph_config":{"decode":{"backend":"full","max_bs":512,"bs":[1,2,4,8,12,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,272,288,304,320,336,352,368,384,400,416,432,448,464,480,496,512],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null},"prefill":{"backend":"disabled","max_bs":16384,"bs":[4,8,12,16,20,24,28,32,48,64,80,96,112,128,144,160,176,192,208,224,240,256,288,320,352,384,416,448,480,512,576,640,704,768,832,896,960,1024,1280,1536,1792,2048,2304,2560,2816,3072,3328,3584,3840,4096,4608,5120,5632,6144,6656,7168,7680,8192,8704,9216,9728,10240,10752,11264,11776,12288,12800,13312,13824,14336,14848,15360,15872,16384],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null}},"cuda_graph_backend_decode":null,"cuda_graph_backend_prefill":null,"cuda_graph_max_bs_decode":null,"cuda_graph_max_bs_prefill":null,"cuda_graph_bs_decode":null,"cuda_graph_bs_prefill":null,"cuda_graph_tc_compiler":null,"disable_prefill_cuda_graph":false,"disable_decode_cuda_graph":false,"disable_cuda_graph":false,"disable_cuda_graph_padding":false,"enable_profile_cuda_graph":false,"enable_cudagraph_gc":false,"debug_cuda_graph":false,"enable_layerwise_nvtx_marker":false,"enable_nccl_nvls":false,"enable_symm_mem":false,"triton_attention_reduce_in_fp32":false,"triton_attention_num_kv_splits":8,"triton_attention_split_tile_size":null,"flashinfer_mla_disable_ragged":false,"enable_fused_qk_norm_rope":false,"enable_precise_embedding_interpolation":false,"enable_fused_moe_sum_all_reduce":false,"enable_deepseek_v4_fp4_indexer":false,"disable_custom_all_reduce":false,"enable_mscclpp":false,"enable_torch_symm_mem":false,"enable_scattered_sconv":false,"pre_warm_nccl":false,"enable_quant_communications":false,"enable_flashinfer_allreduce_fusion":false,"enforce_disable_flashinfer_allreduce_fusion":false,"flashinfer_allreduce_fusion_backend":null,"enable_aiter_allreduce_fusion":false,"enable_torch_compile":false,"enable_torch_compile_debug_mode":false,"torch_compile_max_bs":32,"speculative_algorithm":null,"speculative_draft_model_path":null,"speculative_draft_model_revision":null,"speculative_draft_load_format":null,"speculative_num_steps":null,"speculative_eagle_topk":null,"speculative_num_draft_tokens":null,"speculative_dflash_block_size":null,"speculative_dspark_block_size":null,"speculative_dspark_sps_table_path":null,"speculative_dspark_confidence_sts_path":null,"speculative_dspark_align_verify_tokens_to_graph_tier":false,"speculative_accept_threshold_single":1.0,"speculative_accept_threshold_acc":1.0,"speculative_use_rejection_sampling":false,"speculative_token_map":null,"speculative_attention_mode":"prefill","speculative_draft_attention_backend":null,"speculative_draft_kv_cache_dtype":null,"speculative_draft_window_size":null,"speculative_moe_runner_backend":"auto","speculative_moe_a2a_backend":null,"speculative_draft_model_quantization":null,"_speculative_draft_quantization_explicitly_set":false,"speculative_skip_dp_mlp_sync":false,"enable_multi_layer_eagle":false,"speculative_adaptive":false,"speculative_adaptive_config":null,"decoupled_spec_bind_endpoint":null,"decoupled_spec_connect_endpoints":null,"decoupled_spec_rank":null,"decoupled_spec_role":"null","spec_trace_dir":null,"speculative_ngram_min_bfs_breadth":1,"speculative_ngram_max_bfs_breadth":10,"speculative_ngram_match_type":"BFS","speculative_ngram_max_trie_depth":18,"speculative_ngram_capacity":10000000,"speculative_ngram_external_corpus_path":null,"speculative_ngram_external_sam_budget":0,"speculative_ngram_external_corpus_max_tokens":10000000,"ep_size":4,"moe_a2a_backend":"deepep","moe_runner_backend":"auto","flashinfer_mxfp4_moe_precision":"default","deepep_mode":"auto","fuseep_mode":2,"deepep_dispatcher_output_dtype":"auto","ep_num_redundant_experts":0,"ep_dispatch_algorithm":null,"init_expert_location":"trivial","enable_eplb":false,"eplb_algorithm":"auto","eplb_rebalance_num_iterations":1000,"eplb_rebalance_layers_per_chunk":null,"eplb_min_rebalancing_utilization_threshold":1.0,"expert_distribution_recorder_mode":null,"expert_distribution_recorder_buffer_size":1000,"expert_balancedness_report_mode":"off","deepep_config":"{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}","moe_dense_tp_size":null,"elastic_ep_backend":null,"enable_elastic_expert_backup":false,"mooncake_ib_device":null,"enable_waterfill":false,"ep_join_mode":null,"ep_join_rank_offset":0,"elastic_ep_initial_size":null,"max_ep_size":null,"elastic_ep_scale_timeout":600,"elastic_ep_rejoin":false,"disable_flashinfer_cutlass_moe_fp4_allgather":false,"disable_shared_experts_fusion":false,"enforce_shared_experts_fusion":false,"max_mamba_cache_size":null,"mamba_ssm_dtype":null,"mamba_max_states_per_path":-1,"enable_mamba_cache_stochastic_rounding":false,"mamba_cache_philox_rounds":0,"mamba_full_memory_ratio":0.9,"mamba_radix_cache_strategy":"auto","uses_mamba_radix_cache":false,"mamba_track_interval":256,"enable_int8_mamba_checkpoint":false,"int8_mamba_ckpt_size":null,"linear_attn_backend":"triton","linear_attn_decode_backend":null,"linear_attn_prefill_backend":null,"linear_attn_verify_backend":null,"enable_linear_replayssm":false,"linear_replayssm_cache_len":16,"enable_linear_replayssm_spec":false,"enable_hierarchical_cache":false,"hicache_ratio":2.0,"hicache_size":0,"hicache_write_policy":"write_through","hicache_io_backend":"kernel","hicache_mem_layout":"page_first","hicache_storage_backend":null,"hicache_storage_prefetch_policy":"timeout","hicache_storage_backend_extra_config":null,"enable_hisparse":false,"hisparse_config":null,"enable_broadcast_mm_inputs_process":false,"enable_prefix_mm_cache":false,"mm_enable_dp_encoder":false,"mm_process_config":{},"mm_processor_worker_num":0,"mm_io_worker_num":0,"allowed_media_domains":[],"media_url_max_file_size_mb":64,"mm_preprocess_cache_size_mb":null,"trust_mm_content_hashes":false,"limit_mm_data_per_request":null,"enable_mm_global_cache":false,"image_processor_backend":"auto","mm_global_cache_backend":"mooncake","disable_fast_image_processor":false,"mm_feature_transport":"cpu","keep_mm_feature_on_device":false,"enable_lora":null,"enable_lora_overlap_loading":null,"max_lora_rank":null,"lora_target_modules":null,"lora_paths":null,"max_loaded_loras":null,"max_loras_per_batch":8,"lora_eviction_policy":"lru","lora_backend":"csgmv","max_lora_chunk_size":16,"experts_shared_outer_loras":null,"lora_use_virtual_experts":false,"lora_strict_loading":false,"lora_drain_wait_threshold":0.0,"enable_two_batch_overlap":false,"enable_single_batch_overlap":false,"tbo_token_distribution_threshold":0.48,"cpu_offload_gb":0,"offload_group_size":-1,"offload_num_in_group":1,"offload_prefetch_step":1,"offload_mode":"cpu","enable_lmcache":false,"lmcache_config_file":null,"enable_flexkv":false,"flexkv_config_file":null,"kt_weight_path":null,"kt_method":"AMXINT4","kt_cpuinfer":null,"kt_threadpool_count":2,"kt_num_gpu_experts":null,"kt_max_deferred_experts_per_token":null,"dllm_algorithm":null,"dllm_algorithm_config":null,"dllm_fdfo":true,"disaggregation_mode":"null","disaggregation_transfer_backend":"mooncake","disaggregation_bootstrap_port":8998,"disaggregation_ib_device":null,"disaggregation_decode_enable_radix_cache":false,"disaggregation_decode_enable_offload_kvcache":false,"disaggregation_decode_retraction_backup":null,"num_reserved_decode_tokens":512,"disaggregation_decode_extra_slots":null,"disaggregation_decode_polling_interval":1,"optimistic_prefill_attempts":0,"encoder_only":false,"language_only":false,"language_model_only":false,"encoder_transfer_backend":"zmq_to_scheduler","encoder_urls":[],"encoder_bootstrap_port":8997,"encoder_register_urls":[],"enable_adaptive_dispatch_to_encoder":false,"enable_pdmux":false,"pdmux_config_path":null,"sm_group_num":8,"startup_weight_load_mode":"serial","custom_weight_loader":[],"weight_loader_disable_mmap":false,"weight_loader_prefetch_checkpoints":false,"weight_loader_prefetch_num_threads":4,"weight_loader_drop_cache_after_load":false,"remote_instance_weight_loader_seed_instance_ip":null,"remote_instance_weight_loader_seed_instance_service_port":null,"remote_instance_weight_loader_send_weights_group_ports":null,"remote_instance_weight_loader_backend":"nccl","remote_instance_weight_loader_start_seed_via_transfer_engine":false,"engine_info_bootstrap_port":6789,"modelexpress_config":null,"download_dir":null,"model_checksum":null,"delete_ckpt_after_loading":false,"decrypted_config_file":null,"decrypted_draft_config_file":null,"checkpoint_engine_wait_weights_before_ready":false,"enable_prefill_delayer":false,"prefill_delayer_max_delay_passes":30,"prefill_delayer_token_usage_low_watermark":null,"prefill_delayer_forward_passes_buckets":null,"prefill_delayer_wait_seconds_buckets":null,"prefill_delayer_queue_min_ratio":null,"prefill_delayer_max_delay_ms":null,"min_free_slots_delay":null,"enable_deterministic_inference":false,"rl_on_policy_target":null,"kv_canary":"none","kv_canary_real_data":"none","kv_canary_sweep_interval":0,"enable_dynamic_batch_tokenizer":false,"dynamic_batch_tokenizer_batch_size":32,"dynamic_batch_tokenizer_batch_timeout":0.002,"enable_tokenizer_batch_encode":false,"disable_tokenizer_batch_decode":false,"debug_tensor_dump_output_folder":null,"debug_tensor_dump_layers":null,"debug_tensor_dump_input_file":null,"enable_memory_saver":false,"enable_weights_cpu_backup":false,"enable_draft_weights_cpu_backup":false,"enable_custom_logit_processor":false,"enable_return_hidden_states":false,"return_hidden_states_mode":null,"enable_return_routed_experts":false,"enable_return_indexer_topk":false,"disable_outlines_disk_cache":false,"enable_mis":false,"weight_cache_mode":"off","weight_cache_socket":null,"weight_cache_timeout":1800,"forward_hooks":null,"msprobe_dump_config":null,"status":"ready","max_total_num_tokens":25820928,"max_req_input_len":1048570,"startup_time":{"load_weight":348.83619413909037,"kv_cache_allocation":0.8970330059528351,"scheduler_e2e":508.1397862319136,"cuda_graph":{"prefill":0.0,"decode":89.43910579301883,"target_verify":0.0,"draft_prefill":0.0,"draft_decode":0.0,"draft_extend":0.0},"tokenizer_e2e":526.9549640880432},"scheduler_pids":[440,441,442,443],"internal_states":[{"model_path":"/models","tokenizer_path":"/models","tokenizer_mode":"auto","tokenizer_backend":"huggingface","tokenizer_worker_num":1,"detokenizer_worker_num":1,"skip_tokenizer_init":false,"load_format":"auto","model_loader_extra_config":"{}","trust_remote_code":true,"context_length":null,"is_embedding":false,"enable_multimodal":null,"revision":null,"model_impl":"auto","model_config_parser":"auto","json_model_override_args":"{}","dtype":"auto","quantization":null,"quantization_param_path":null,"kv_cache_dtype":"fp8_e4m3","enable_fp32_lm_head":false,"modelopt_quant":null,"modelopt_checkpoint_restore_path":null,"modelopt_checkpoint_save_path":null,"modelopt_export_path":null,"quantize_and_serve":false,"rl_quant_profile":null,"enable_tf32_matmul":false,"mem_fraction_static":0.862,"max_running_requests":256,"max_queued_requests":null,"max_total_tokens":null,"chunked_prefill_size":4096,"enable_dynamic_chunking":false,"prefill_reserve_budget":0,"max_prefill_tokens":16384,"prefill_max_requests":null,"schedule_policy":"fcfs","enable_priority_scheduling":false,"disable_priority_preemption":false,"default_priority_value":null,"abort_on_priority_when_disabled":false,"schedule_low_priority_values_first":false,"priority_scheduling_preemption_threshold":10,"retraction_policy":"length","schedule_conservativeness":0.3,"page_size":256,"c128_page_size":16,"swa_full_tokens_ratio":0.1,"disable_hybrid_swa_memory":false,"radix_eviction_policy":"lru","prefill_only_disable_kv_cache":false,"disable_radix_cache":false,"enable_page_major_kv_layout":false,"enable_unified_memory":false,"disable_chunked_prefix_cache":true,"disable_overlap_schedule":false,"num_continuous_decode_steps":1,"scheduler_recv_interval":1,"enable_mixed_chunk":false,"nccl_port":null,"dist_timeout":null,"dist_init_addr":null,"nnodes":1,"node_rank":0,"tp_size":4,"dcp_size":1,"pp_size":1,"pp_max_micro_batch_size":64,"pp_async_batch_depth":0,"dp_size":4,"load_balance_method":"round_robin","prefix_affinity_fallback":"total_tokens","prefix_affinity_max_load_skew":1.5,"prefix_affinity_hash_tokens":4096,"prefix_affinity_disable_token_fallback":false,"attn_cp_size":1,"moe_dp_size":1,"dwdp_size":1,"dcp_comm_backend":"ag_rs","dcp_replicate_q_proj":null,"enable_prefill_cp":false,"cp_strategy":null,"enable_dsa_cache_layer_split":false,"enable_dsa_prefill_context_parallel":false,"dsa_prefill_cp_mode":"round-robin-split","enable_prefill_context_parallel":false,"prefill_cp_mode":"in-seq-split","enable_cp_decode_attn_tp":false,"enable_dp_attention":true,"enable_dp_attention_local_control_broadcast":false,"enable_dp_lm_head":false,"enable_tp_lm_head_all_to_all":false,"enable_attn_tp_input_scattered":false,"disable_attn_tp_gather":false,"enable_p2p_check":false,"device":"cuda","base_gpu_id":0,"gpu_id_step":1,"random_seed":903930395,"mlx_enable_sampling":false,"watchdog_timeout":300,"soft_watchdog_timeout":null,"sleep_on_idle":false,"use_ray":false,"numa_node":null,"gc_threshold":null,"host":"0.0.0.0","port":30000,"fastapi_root_path":"","smg_grpc_mode":false,"grpc_mode":false,"grpc_port":null,"sidecar":null,"sidecar_args":null,"skip_server_warmup":false,"warmups":null,"enable_http2":false,"http2_max_concurrent_streams":200,"ssl_keyfile":null,"ssl_certfile":null,"ssl_ca_certs":null,"ssl_keyfile_password":null,"enable_ssl_refresh":false,"api_key":null,"admin_api_key":null,"served_model_name":"/models","weight_version":"default","chat_template":null,"hf_chat_template_name":null,"completion_template":null,"file_storage_path":"sglang_storage","enable_cache_report":false,"reasoning_parser":"deepseek-v4","default_chat_template_kwargs":null,"strip_thinking_cache":false,"enable_strict_thinking":false,"tool_call_parser":"deepseekv4","tool_server":null,"sampling_defaults":"model","asr_max_buffer_seconds":60,"asr_max_concurrent_sessions":32,"preferred_sampling_params":null,"allow_auto_truncate":false,"stream_interval":1,"batch_notify_size":16,"stream_response_default_include_usage":false,"incremental_streaming_output":false,"enable_streaming_session":false,"enable_session_radix_cache":false,"log_level":"info","log_level_http":null,"log_requests":false,"log_requests_level":2,"log_requests_format":"text","log_requests_target":null,"uvicorn_access_log_exclude_prefixes":[],"crash_dump_folder":null,"show_time_cost":false,"enable_metrics":false,"smg_http_sidecar_port":null,"enable_mfu_metrics":false,"enable_metrics_for_all_schedulers":false,"load_snapshot_publish_interval":15,"tokenizer_metrics_custom_labels_header":"x-custom-labels","tokenizer_metrics_allowed_custom_labels":null,"extra_metric_labels":null,"bucket_time_to_first_token":null,"bucket_inter_token_latency":null,"bucket_e2e_request_latency":null,"prompt_tokens_buckets":null,"generation_tokens_buckets":null,"gc_warning_threshold_secs":0.0,"decode_log_interval":40,"enable_request_time_stats_logging":false,"kv_events_config":null,"enable_forward_pass_metrics":false,"forward_pass_metrics_worker_id":"","forward_pass_metrics_ipc_name":null,"enable_trace":false,"trace_modules":"request","otlp_traces_endpoint":"localhost:4317","export_metrics_to_file":false,"export_metrics_to_file_dir":null,"stat_loggers":null,"constrained_json_whitespace_pattern":null,"constrained_json_disable_any_whitespace":false,"attention_backend":"dsv4","decode_attention_backend":null,"prefill_attention_backend":null,"sampling_backend":"flashinfer","grammar_backend":"xgrammar","radix_cache_backend":null,"mm_attention_backend":null,"fp8_gemm_runner_backend":"auto","fp4_gemm_runner_backend":"auto","bf16_gemm_backend":"auto","dsa_prefill_backend":null,"dsa_decode_backend":null,"dsa_paged_mqa_logits_backend":"auto","dsa_topk_backend":"sgl-kernel","disable_flashinfer_autotune":false,"flashinfer_autotune_skip_ops":null,"mamba_backend":"triton","cuda_graph_config":{"decode":{"backend":"full","max_bs":512,"bs":[1,2,4,8,12,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,272,288,304,320,336,352,368,384,400,416,432,448,464,480,496,512],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null},"prefill":{"backend":"disabled","max_bs":16384,"bs":[4,8,12,16,20,24,28,32,48,64,80,96,112,128,144,160,176,192,208,224,240,256,288,320,352,384,416,448,480,512,576,640,704,768,832,896,960,1024,1280,1536,1792,2048,2304,2560,2816,3072,3328,3584,3840,4096,4608,5120,5632,6144,6656,7168,7680,8192,8704,9216,9728,10240,10752,11264,11776,12288,12800,13312,13824,14336,14848,15360,15872,16384],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null}},"cuda_graph_backend_decode":null,"cuda_graph_backend_prefill":null,"cuda_graph_max_bs_decode":null,"cuda_graph_max_bs_prefill":null,"cuda_graph_bs_decode":null,"cuda_graph_bs_prefill":null,"cuda_graph_tc_compiler":null,"disable_prefill_cuda_graph":false,"disable_decode_cuda_graph":false,"disable_cuda_graph":false,"disable_cuda_graph_padding":false,"enable_profile_cuda_graph":false,"enable_cudagraph_gc":false,"debug_cuda_graph":false,"enable_layerwise_nvtx_marker":false,"enable_nccl_nvls":false,"enable_symm_mem":false,"triton_attention_reduce_in_fp32":false,"triton_attention_num_kv_splits":8,"triton_attention_split_tile_size":null,"flashinfer_mla_disable_ragged":false,"enable_fused_qk_norm_rope":false,"enable_precise_embedding_interpolation":false,"enable_fused_moe_sum_all_reduce":false,"enable_deepseek_v4_fp4_indexer":false,"disable_custom_all_reduce":false,"enable_mscclpp":false,"enable_torch_symm_mem":false,"enable_scattered_sconv":false,"pre_warm_nccl":false,"enable_quant_communications":false,"enable_flashinfer_allreduce_fusion":false,"enforce_disable_flashinfer_allreduce_fusion":false,"flashinfer_allreduce_fusion_backend":null,"enable_aiter_allreduce_fusion":false,"enable_torch_compile":false,"enable_torch_compile_debug_mode":false,"torch_compile_max_bs":32,"speculative_algorithm":null,"speculative_draft_model_path":null,"speculative_draft_model_revision":null,"speculative_draft_load_format":null,"speculative_num_steps":null,"speculative_eagle_topk":null,"speculative_num_draft_tokens":null,"speculative_dflash_block_size":null,"speculative_dspark_block_size":null,"speculative_dspark_sps_table_path":null,"speculative_dspark_confidence_sts_path":null,"speculative_dspark_align_verify_tokens_to_graph_tier":false,"speculative_accept_threshold_single":1.0,"speculative_accept_threshold_acc":1.0,"speculative_use_rejection_sampling":false,"speculative_token_map":null,"speculative_attention_mode":"prefill","speculative_draft_attention_backend":null,"speculative_draft_kv_cache_dtype":null,"speculative_draft_window_size":null,"speculative_moe_runner_backend":"auto","speculative_moe_a2a_backend":null,"speculative_draft_model_quantization":null,"_speculative_draft_quantization_explicitly_set":false,"speculative_skip_dp_mlp_sync":false,"enable_multi_layer_eagle":false,"speculative_adaptive":false,"speculative_adaptive_config":null,"decoupled_spec_bind_endpoint":null,"decoupled_spec_connect_endpoints":null,"decoupled_spec_rank":null,"decoupled_spec_role":"null","spec_trace_dir":null,"speculative_ngram_min_bfs_breadth":1,"speculative_ngram_max_bfs_breadth":10,"speculative_ngram_match_type":"BFS","speculative_ngram_max_trie_depth":18,"speculative_ngram_capacity":10000000,"speculative_ngram_external_corpus_path":null,"speculative_ngram_external_sam_budget":0,"speculative_ngram_external_corpus_max_tokens":10000000,"ep_size":4,"moe_a2a_backend":"deepep","moe_runner_backend":"auto","flashinfer_mxfp4_moe_precision":"default","deepep_mode":"auto","fuseep_mode":2,"deepep_dispatcher_output_dtype":"auto","ep_num_redundant_experts":0,"ep_dispatch_algorithm":null,"init_expert_location":"trivial","enable_eplb":false,"eplb_algorithm":"auto","eplb_rebalance_num_iterations":1000,"eplb_rebalance_layers_per_chunk":null,"eplb_min_rebalancing_utilization_threshold":1.0,"expert_distribution_recorder_mode":null,"expert_distribution_recorder_buffer_size":1000,"expert_balancedness_report_mode":"off","deepep_config":"{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}","moe_dense_tp_size":null,"elastic_ep_backend":null,"enable_elastic_expert_backup":false,"mooncake_ib_device":null,"enable_waterfill":false,"ep_join_mode":null,"ep_join_rank_offset":0,"elastic_ep_initial_size":null,"max_ep_size":null,"elastic_ep_scale_timeout":600,"elastic_ep_rejoin":false,"disable_flashinfer_cutlass_moe_fp4_allgather":false,"disable_shared_experts_fusion":false,"enforce_shared_experts_fusion":false,"max_mamba_cache_size":null,"mamba_ssm_dtype":null,"mamba_max_states_per_path":-1,"enable_mamba_cache_stochastic_rounding":false,"mamba_cache_philox_rounds":0,"mamba_full_memory_ratio":0.9,"mamba_radix_cache_strategy":"auto","uses_mamba_radix_cache":false,"mamba_track_interval":256,"enable_int8_mamba_checkpoint":false,"int8_mamba_ckpt_size":null,"linear_attn_backend":"triton","linear_attn_decode_backend":null,"linear_attn_prefill_backend":null,"linear_attn_verify_backend":null,"enable_linear_replayssm":false,"linear_replayssm_cache_len":16,"enable_linear_replayssm_spec":false,"enable_hierarchical_cache":false,"hicache_ratio":2.0,"hicache_size":0,"hicache_write_policy":"write_through","hicache_io_backend":"kernel","hicache_mem_layout":"page_first","hicache_storage_backend":null,"hicache_storage_prefetch_policy":"timeout","hicache_storage_backend_extra_config":null,"enable_hisparse":false,"hisparse_config":null,"enable_broadcast_mm_inputs_process":false,"enable_prefix_mm_cache":false,"mm_enable_dp_encoder":false,"mm_process_config":{},"mm_processor_worker_num":0,"mm_io_worker_num":0,"allowed_media_domains":[],"media_url_max_file_size_mb":64,"mm_preprocess_cache_size_mb":null,"trust_mm_content_hashes":false,"limit_mm_data_per_request":null,"enable_mm_global_cache":false,"image_processor_backend":"auto","mm_global_cache_backend":"mooncake","disable_fast_image_processor":false,"mm_feature_transport":"cpu","keep_mm_feature_on_device":false,"enable_lora":null,"enable_lora_overlap_loading":null,"max_lora_rank":null,"lora_target_modules":null,"lora_paths":null,"max_loaded_loras":null,"max_loras_per_batch":8,"lora_eviction_policy":"lru","lora_backend":"csgmv","max_lora_chunk_size":16,"experts_shared_outer_loras":null,"lora_use_virtual_experts":false,"lora_strict_loading":false,"lora_drain_wait_threshold":0.0,"enable_two_batch_overlap":false,"enable_single_batch_overlap":false,"tbo_token_distribution_threshold":0.48,"cpu_offload_gb":0,"offload_group_size":-1,"offload_num_in_group":1,"offload_prefetch_step":1,"offload_mode":"cpu","enable_lmcache":false,"lmcache_config_file":null,"enable_flexkv":false,"flexkv_config_file":null,"kt_weight_path":null,"kt_method":"AMXINT4","kt_cpuinfer":null,"kt_threadpool_count":2,"kt_num_gpu_experts":null,"kt_max_deferred_experts_per_token":null,"dllm_algorithm":null,"dllm_algorithm_config":null,"dllm_fdfo":true,"disaggregation_mode":"null","disaggregation_transfer_backend":"mooncake","disaggregation_bootstrap_port":8998,"disaggregation_ib_device":null,"disaggregation_decode_enable_radix_cache":false,"disaggregation_decode_enable_offload_kvcache":false,"disaggregation_decode_retraction_backup":"cpu_tensor","num_reserved_decode_tokens":512,"disaggregation_decode_extra_slots":null,"disaggregation_decode_polling_interval":1,"optimistic_prefill_attempts":0,"encoder_only":false,"language_only":false,"language_model_only":false,"encoder_transfer_backend":"zmq_to_scheduler","encoder_urls":[],"encoder_bootstrap_port":8997,"encoder_register_urls":[],"enable_adaptive_dispatch_to_encoder":false,"enable_pdmux":false,"pdmux_config_path":null,"sm_group_num":8,"startup_weight_load_mode":"serial","custom_weight_loader":[],"weight_loader_disable_mmap":false,"weight_loader_prefetch_checkpoints":false,"weight_loader_prefetch_num_threads":4,"weight_loader_drop_cache_after_load":false,"remote_instance_weight_loader_seed_instance_ip":null,"remote_instance_weight_loader_seed_instance_service_port":null,"remote_instance_weight_loader_send_weights_group_ports":null,"remote_instance_weight_loader_backend":"nccl","remote_instance_weight_loader_start_seed_via_transfer_engine":false,"engine_info_bootstrap_port":6789,"modelexpress_config":null,"download_dir":null,"model_checksum":null,"delete_ckpt_after_loading":false,"decrypted_config_file":null,"decrypted_draft_config_file":null,"checkpoint_engine_wait_weights_before_ready":false,"enable_prefill_delayer":false,"prefill_delayer_max_delay_passes":30,"prefill_delayer_token_usage_low_watermark":null,"prefill_delayer_forward_passes_buckets":null,"prefill_delayer_wait_seconds_buckets":null,"prefill_delayer_queue_min_ratio":null,"prefill_delayer_max_delay_ms":null,"min_free_slots_delay":null,"enable_deterministic_inference":false,"rl_on_policy_target":null,"kv_canary":"none","kv_canary_real_data":"none","kv_canary_sweep_interval":0,"enable_dynamic_batch_tokenizer":false,"dynamic_batch_tokenizer_batch_size":32,"dynamic_batch_tokenizer_batch_timeout":0.002,"enable_tokenizer_batch_encode":false,"disable_tokenizer_batch_decode":false,"debug_tensor_dump_output_folder":null,"debug_tensor_dump_layers":null,"debug_tensor_dump_input_file":null,"enable_memory_saver":false,"enable_weights_cpu_backup":false,"enable_draft_weights_cpu_backup":false,"enable_custom_logit_processor":false,"enable_return_hidden_states":false,"return_hidden_states_mode":null,"enable_return_routed_experts":false,"enable_return_indexer_topk":false,"disable_outlines_disk_cache":false,"enable_mis":false,"weight_cache_mode":"off","weight_cache_socket":null,"weight_cache_timeout":1800,"forward_hooks":null,"msprobe_dump_config":null,"_resolved_overrides":[["_deepseek_v4_overrides",{"attention_backend":"dsv4","page_size":256,"swa_full_tokens_ratio":0.1}],["_deepseek_v4_kv_cache_dtype",{"kv_cache_dtype":"fp8_e4m3"}],["_sampling_backend_default",{"sampling_backend":"flashinfer"}],["_tp_lm_head_all_to_all_default",{"enable_tp_lm_head_all_to_all":false}],["_a2a_ep_size",{"ep_size":4}],["_speculative_moe_runner_default",{"speculative_moe_runner_backend":"auto"}]],"grpc_worker_threads":4,"_quantization_explicitly_unset":false,"_cuda_graph_config_locked":[],"_declarations_materialized":true,"_mx_config_cache":{},"max_speculative_num_draft_tokens":null,"last_gen_throughput":0.0,"memory_usage":{"weight":42.971,"kvcache":0,"startup_available":14.438,"token_capacity":25820928,"token_capacity_swa":2582016,"graph":{"prefill":0.0,"decode":19.176,"target_verify":0.0,"draft_prefill":0.0,"draft_decode":0.0,"draft_extend":0.0}},"startup_time":{"load_weight":348.49362135794945,"kv_cache_allocation":0.8837988879531622,"scheduler_e2e":507.92600555799436,"cuda_graph":{"prefill":0.0,"decode":89.42433235095814,"target_verify":0.0,"draft_prefill":0.0,"draft_decode":0.0,"draft_extend":0.0}},"effective_max_running_requests_per_dp":64},{"model_path":"/models","tokenizer_path":"/models","tokenizer_mode":"auto","tokenizer_backend":"huggingface","tokenizer_worker_num":1,"detokenizer_worker_num":1,"skip_tokenizer_init":false,"load_format":"auto","model_loader_extra_config":"{}","trust_remote_code":true,"context_length":null,"is_embedding":false,"enable_multimodal":null,"revision":null,"model_impl":"auto","model_config_parser":"auto","json_model_override_args":"{}","dtype":"auto","quantization":null,"quantization_param_path":null,"kv_cache_dtype":"fp8_e4m3","enable_fp32_lm_head":false,"modelopt_quant":null,"modelopt_checkpoint_restore_path":null,"modelopt_checkpoint_save_path":null,"modelopt_export_path":null,"quantize_and_serve":false,"rl_quant_profile":null,"enable_tf32_matmul":false,"mem_fraction_static":0.862,"max_running_requests":256,"max_queued_requests":null,"max_total_tokens":null,"chunked_prefill_size":4096,"enable_dynamic_chunking":false,"prefill_reserve_budget":0,"max_prefill_tokens":16384,"prefill_max_requests":null,"schedule_policy":"fcfs","enable_priority_scheduling":false,"disable_priority_preemption":false,"default_priority_value":null,"abort_on_priority_when_disabled":false,"schedule_low_priority_values_first":false,"priority_scheduling_preemption_threshold":10,"retraction_policy":"length","schedule_conservativeness":0.3,"page_size":256,"c128_page_size":16,"swa_full_tokens_ratio":0.1,"disable_hybrid_swa_memory":false,"radix_eviction_policy":"lru","prefill_only_disable_kv_cache":false,"disable_radix_cache":false,"enable_page_major_kv_layout":false,"enable_unified_memory":false,"disable_chunked_prefix_cache":true,"disable_overlap_schedule":false,"num_continuous_decode_steps":1,"scheduler_recv_interval":1,"enable_mixed_chunk":false,"nccl_port":null,"dist_timeout":null,"dist_init_addr":null,"nnodes":1,"node_rank":0,"tp_size":4,"dcp_size":1,"pp_size":1,"pp_max_micro_batch_size":64,"pp_async_batch_depth":0,"dp_size":4,"load_balance_method":"round_robin","prefix_affinity_fallback":"total_tokens","prefix_affinity_max_load_skew":1.5,"prefix_affinity_hash_tokens":4096,"prefix_affinity_disable_token_fallback":false,"attn_cp_size":1,"moe_dp_size":1,"dwdp_size":1,"dcp_comm_backend":"ag_rs","dcp_replicate_q_proj":null,"enable_prefill_cp":false,"cp_strategy":null,"enable_dsa_cache_layer_split":false,"enable_dsa_prefill_context_parallel":false,"dsa_prefill_cp_mode":"round-robin-split","enable_prefill_context_parallel":false,"prefill_cp_mode":"in-seq-split","enable_cp_decode_attn_tp":false,"enable_dp_attention":true,"enable_dp_attention_local_control_broadcast":false,"enable_dp_lm_head":false,"enable_tp_lm_head_all_to_all":false,"enable_attn_tp_input_scattered":false,"disable_attn_tp_gather":false,"enable_p2p_check":false,"device":"cuda","base_gpu_id":0,"gpu_id_step":1,"random_seed":903930395,"mlx_enable_sampling":false,"watchdog_timeout":300,"soft_watchdog_timeout":null,"sleep_on_idle":false,"use_ray":false,"numa_node":null,"gc_threshold":null,"host":"0.0.0.0","port":30000,"fastapi_root_path":"","smg_grpc_mode":false,"grpc_mode":false,"grpc_port":null,"sidecar":null,"sidecar_args":null,"skip_server_warmup":false,"warmups":null,"enable_http2":false,"http2_max_concurrent_streams":200,"ssl_keyfile":null,"ssl_certfile":null,"ssl_ca_certs":null,"ssl_keyfile_password":null,"enable_ssl_refresh":false,"api_key":null,"admin_api_key":null,"served_model_name":"/models","weight_version":"default","chat_template":null,"hf_chat_template_name":null,"completion_template":null,"file_storage_path":"sglang_storage","enable_cache_report":false,"reasoning_parser":"deepseek-v4","default_chat_template_kwargs":null,"strip_thinking_cache":false,"enable_strict_thinking":false,"tool_call_parser":"deepseekv4","tool_server":null,"sampling_defaults":"model","asr_max_buffer_seconds":60,"asr_max_concurrent_sessions":32,"preferred_sampling_params":null,"allow_auto_truncate":false,"stream_interval":1,"batch_notify_size":16,"stream_response_default_include_usage":false,"incremental_streaming_output":false,"enable_streaming_session":false,"enable_session_radix_cache":false,"log_level":"info","log_level_http":null,"log_requests":false,"log_requests_level":2,"log_requests_format":"text","log_requests_target":null,"uvicorn_access_log_exclude_prefixes":[],"crash_dump_folder":null,"show_time_cost":false,"enable_metrics":false,"smg_http_sidecar_port":null,"enable_mfu_metrics":false,"enable_metrics_for_all_schedulers":false,"load_snapshot_publish_interval":15,"tokenizer_metrics_custom_labels_header":"x-custom-labels","tokenizer_metrics_allowed_custom_labels":null,"extra_metric_labels":null,"bucket_time_to_first_token":null,"bucket_inter_token_latency":null,"bucket_e2e_request_latency":null,"prompt_tokens_buckets":null,"generation_tokens_buckets":null,"gc_warning_threshold_secs":0.0,"decode_log_interval":40,"enable_request_time_stats_logging":false,"kv_events_config":null,"enable_forward_pass_metrics":false,"forward_pass_metrics_worker_id":"","forward_pass_metrics_ipc_name":null,"enable_trace":false,"trace_modules":"request","otlp_traces_endpoint":"localhost:4317","export_metrics_to_file":false,"export_metrics_to_file_dir":null,"stat_loggers":null,"constrained_json_whitespace_pattern":null,"constrained_json_disable_any_whitespace":false,"attention_backend":"dsv4","decode_attention_backend":null,"prefill_attention_backend":null,"sampling_backend":"flashinfer","grammar_backend":"xgrammar","radix_cache_backend":null,"mm_attention_backend":null,"fp8_gemm_runner_backend":"auto","fp4_gemm_runner_backend":"auto","bf16_gemm_backend":"auto","dsa_prefill_backend":null,"dsa_decode_backend":null,"dsa_paged_mqa_logits_backend":"auto","dsa_topk_backend":"sgl-kernel","disable_flashinfer_autotune":false,"flashinfer_autotune_skip_ops":null,"mamba_backend":"triton","cuda_graph_config":{"decode":{"backend":"full","max_bs":512,"bs":[1,2,4,8,12,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,272,288,304,320,336,352,368,384,400,416,432,448,464,480,496,512],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null},"prefill":{"backend":"disabled","max_bs":16384,"bs":[4,8,12,16,20,24,28,32,48,64,80,96,112,128,144,160,176,192,208,224,240,256,288,320,352,384,416,448,480,512,576,640,704,768,832,896,960,1024,1280,1536,1792,2048,2304,2560,2816,3072,3328,3584,3840,4096,4608,5120,5632,6144,6656,7168,7680,8192,8704,9216,9728,10240,10752,11264,11776,12288,12800,13312,13824,14336,14848,15360,15872,16384],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null}},"cuda_graph_backend_decode":null,"cuda_graph_backend_prefill":null,"cuda_graph_max_bs_decode":null,"cuda_graph_max_bs_prefill":null,"cuda_graph_bs_decode":null,"cuda_graph_bs_prefill":null,"cuda_graph_tc_compiler":null,"disable_prefill_cuda_graph":false,"disable_decode_cuda_graph":false,"disable_cuda_graph":false,"disable_cuda_graph_padding":false,"enable_profile_cuda_graph":false,"enable_cudagraph_gc":false,"debug_cuda_graph":false,"enable_layerwise_nvtx_marker":false,"enable_nccl_nvls":false,"enable_symm_mem":false,"triton_attention_reduce_in_fp32":false,"triton_attention_num_kv_splits":8,"triton_attention_split_tile_size":null,"flashinfer_mla_disable_ragged":false,"enable_fused_qk_norm_rope":false,"enable_precise_embedding_interpolation":false,"enable_fused_moe_sum_all_reduce":false,"enable_deepseek_v4_fp4_indexer":false,"disable_custom_all_reduce":false,"enable_mscclpp":false,"enable_torch_symm_mem":false,"enable_scattered_sconv":false,"pre_warm_nccl":false,"enable_quant_communications":false,"enable_flashinfer_allreduce_fusion":false,"enforce_disable_flashinfer_allreduce_fusion":false,"flashinfer_allreduce_fusion_backend":null,"enable_aiter_allreduce_fusion":false,"enable_torch_compile":false,"enable_torch_compile_debug_mode":false,"torch_compile_max_bs":32,"speculative_algorithm":null,"speculative_draft_model_path":null,"speculative_draft_model_revision":null,"speculative_draft_load_format":null,"speculative_num_steps":null,"speculative_eagle_topk":null,"speculative_num_draft_tokens":null,"speculative_dflash_block_size":null,"speculative_dspark_block_size":null,"speculative_dspark_sps_table_path":null,"speculative_dspark_confidence_sts_path":null,"speculative_dspark_align_verify_tokens_to_graph_tier":false,"speculative_accept_threshold_single":1.0,"speculative_accept_threshold_acc":1.0,"speculative_use_rejection_sampling":false,"speculative_token_map":null,"speculative_attention_mode":"prefill","speculative_draft_attention_backend":null,"speculative_draft_kv_cache_dtype":null,"speculative_draft_window_size":null,"speculative_moe_runner_backend":"auto","speculative_moe_a2a_backend":null,"speculative_draft_model_quantization":null,"_speculative_draft_quantization_explicitly_set":false,"speculative_skip_dp_mlp_sync":false,"enable_multi_layer_eagle":false,"speculative_adaptive":false,"speculative_adaptive_config":null,"decoupled_spec_bind_endpoint":null,"decoupled_spec_connect_endpoints":null,"decoupled_spec_rank":null,"decoupled_spec_role":"null","spec_trace_dir":null,"speculative_ngram_min_bfs_breadth":1,"speculative_ngram_max_bfs_breadth":10,"speculative_ngram_match_type":"BFS","speculative_ngram_max_trie_depth":18,"speculative_ngram_capacity":10000000,"speculative_ngram_external_corpus_path":null,"speculative_ngram_external_sam_budget":0,"speculative_ngram_external_corpus_max_tokens":10000000,"ep_size":4,"moe_a2a_backend":"deepep","moe_runner_backend":"auto","flashinfer_mxfp4_moe_precision":"default","deepep_mode":"auto","fuseep_mode":2,"deepep_dispatcher_output_dtype":"auto","ep_num_redundant_experts":0,"ep_dispatch_algorithm":null,"init_expert_location":"trivial","enable_eplb":false,"eplb_algorithm":"auto","eplb_rebalance_num_iterations":1000,"eplb_rebalance_layers_per_chunk":null,"eplb_min_rebalancing_utilization_threshold":1.0,"expert_distribution_recorder_mode":null,"expert_distribution_recorder_buffer_size":1000,"expert_balancedness_report_mode":"off","deepep_config":"{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}","moe_dense_tp_size":null,"elastic_ep_backend":null,"enable_elastic_expert_backup":false,"mooncake_ib_device":null,"enable_waterfill":false,"ep_join_mode":null,"ep_join_rank_offset":0,"elastic_ep_initial_size":null,"max_ep_size":null,"elastic_ep_scale_timeout":600,"elastic_ep_rejoin":false,"disable_flashinfer_cutlass_moe_fp4_allgather":false,"disable_shared_experts_fusion":false,"enforce_shared_experts_fusion":false,"max_mamba_cache_size":null,"mamba_ssm_dtype":null,"mamba_max_states_per_path":-1,"enable_mamba_cache_stochastic_rounding":false,"mamba_cache_philox_rounds":0,"mamba_full_memory_ratio":0.9,"mamba_radix_cache_strategy":"auto","uses_mamba_radix_cache":false,"mamba_track_interval":256,"enable_int8_mamba_checkpoint":false,"int8_mamba_ckpt_size":null,"linear_attn_backend":"triton","linear_attn_decode_backend":null,"linear_attn_prefill_backend":null,"linear_attn_verify_backend":null,"enable_linear_replayssm":false,"linear_replayssm_cache_len":16,"enable_linear_replayssm_spec":false,"enable_hierarchical_cache":false,"hicache_ratio":2.0,"hicache_size":0,"hicache_write_policy":"write_through","hicache_io_backend":"kernel","hicache_mem_layout":"page_first","hicache_storage_backend":null,"hicache_storage_prefetch_policy":"timeout","hicache_storage_backend_extra_config":null,"enable_hisparse":false,"hisparse_config":null,"enable_broadcast_mm_inputs_process":false,"enable_prefix_mm_cache":false,"mm_enable_dp_encoder":false,"mm_process_config":{},"mm_processor_worker_num":0,"mm_io_worker_num":0,"allowed_media_domains":[],"media_url_max_file_size_mb":64,"mm_preprocess_cache_size_mb":null,"trust_mm_content_hashes":false,"limit_mm_data_per_request":null,"enable_mm_global_cache":false,"image_processor_backend":"auto","mm_global_cache_backend":"mooncake","disable_fast_image_processor":false,"mm_feature_transport":"cpu","keep_mm_feature_on_device":false,"enable_lora":null,"enable_lora_overlap_loading":null,"max_lora_rank":null,"lora_target_modules":null,"lora_paths":null,"max_loaded_loras":null,"max_loras_per_batch":8,"lora_eviction_policy":"lru","lora_backend":"csgmv","max_lora_chunk_size":16,"experts_shared_outer_loras":null,"lora_use_virtual_experts":false,"lora_strict_loading":false,"lora_drain_wait_threshold":0.0,"enable_two_batch_overlap":false,"enable_single_batch_overlap":false,"tbo_token_distribution_threshold":0.48,"cpu_offload_gb":0,"offload_group_size":-1,"offload_num_in_group":1,"offload_prefetch_step":1,"offload_mode":"cpu","enable_lmcache":false,"lmcache_config_file":null,"enable_flexkv":false,"flexkv_config_file":null,"kt_weight_path":null,"kt_method":"AMXINT4","kt_cpuinfer":null,"kt_threadpool_count":2,"kt_num_gpu_experts":null,"kt_max_deferred_experts_per_token":null,"dllm_algorithm":null,"dllm_algorithm_config":null,"dllm_fdfo":true,"disaggregation_mode":"null","disaggregation_transfer_backend":"mooncake","disaggregation_bootstrap_port":8998,"disaggregation_ib_device":null,"disaggregation_decode_enable_radix_cache":false,"disaggregation_decode_enable_offload_kvcache":false,"disaggregation_decode_retraction_backup":"cpu_tensor","num_reserved_decode_tokens":512,"disaggregation_decode_extra_slots":null,"disaggregation_decode_polling_interval":1,"optimistic_prefill_attempts":0,"encoder_only":false,"language_only":false,"language_model_only":false,"encoder_transfer_backend":"zmq_to_scheduler","encoder_urls":[],"encoder_bootstrap_port":8997,"encoder_register_urls":[],"enable_adaptive_dispatch_to_encoder":false,"enable_pdmux":false,"pdmux_config_path":null,"sm_group_num":8,"startup_weight_load_mode":"serial","custom_weight_loader":[],"weight_loader_disable_mmap":false,"weight_loader_prefetch_checkpoints":false,"weight_loader_prefetch_num_threads":4,"weight_loader_drop_cache_after_load":false,"remote_instance_weight_loader_seed_instance_ip":null,"remote_instance_weight_loader_seed_instance_service_port":null,"remote_instance_weight_loader_send_weights_group_ports":null,"remote_instance_weight_loader_backend":"nccl","remote_instance_weight_loader_start_seed_via_transfer_engine":false,"engine_info_bootstrap_port":6789,"modelexpress_config":null,"download_dir":null,"model_checksum":null,"delete_ckpt_after_loading":false,"decrypted_config_file":null,"decrypted_draft_config_file":null,"checkpoint_engine_wait_weights_before_ready":false,"enable_prefill_delayer":false,"prefill_delayer_max_delay_passes":30,"prefill_delayer_token_usage_low_watermark":null,"prefill_delayer_forward_passes_buckets":null,"prefill_delayer_wait_seconds_buckets":null,"prefill_delayer_queue_min_ratio":null,"prefill_delayer_max_delay_ms":null,"min_free_slots_delay":null,"enable_deterministic_inference":false,"rl_on_policy_target":null,"kv_canary":"none","kv_canary_real_data":"none","kv_canary_sweep_interval":0,"enable_dynamic_batch_tokenizer":false,"dynamic_batch_tokenizer_batch_size":32,"dynamic_batch_tokenizer_batch_timeout":0.002,"enable_tokenizer_batch_encode":false,"disable_tokenizer_batch_decode":false,"debug_tensor_dump_output_folder":null,"debug_tensor_dump_layers":null,"debug_tensor_dump_input_file":null,"enable_memory_saver":false,"enable_weights_cpu_backup":false,"enable_draft_weights_cpu_backup":false,"enable_custom_logit_processor":false,"enable_return_hidden_states":false,"return_hidden_states_mode":null,"enable_return_routed_experts":false,"enable_return_indexer_topk":false,"disable_outlines_disk_cache":false,"enable_mis":false,"weight_cache_mode":"off","weight_cache_socket":null,"weight_cache_timeout":1800,"forward_hooks":null,"msprobe_dump_config":null,"_resolved_overrides":[["_deepseek_v4_overrides",{"attention_backend":"dsv4","page_size":256,"swa_full_tokens_ratio":0.1}],["_deepseek_v4_kv_cache_dtype",{"kv_cache_dtype":"fp8_e4m3"}],["_sampling_backend_default",{"sampling_backend":"flashinfer"}],["_tp_lm_head_all_to_all_default",{"enable_tp_lm_head_all_to_all":false}],["_a2a_ep_size",{"ep_size":4}],["_speculative_moe_runner_default",{"speculative_moe_runner_backend":"auto"}]],"grpc_worker_threads":4,"_quantization_explicitly_unset":false,"_cuda_graph_config_locked":[],"_declarations_materialized":true,"_mx_config_cache":{},"max_speculative_num_draft_tokens":null,"last_gen_throughput":0.0,"memory_usage":{"weight":42.971,"kvcache":0,"startup_available":14.253,"token_capacity":25820928,"token_capacity_swa":2582016,"graph":{"prefill":0.0,"decode":19.33,"target_verify":0.0,"draft_prefill":0.0,"draft_decode":0.0,"draft_extend":0.0}},"startup_time":{"load_weight":348.83619413909037,"kv_cache_allocation":0.8781842190073803,"scheduler_e2e":507.96765940997284,"cuda_graph":{"prefill":0.0,"decode":89.42852122697514,"target_verify":0.0,"draft_prefill":0.0,"draft_decode":0.0,"draft_extend":0.0}},"effective_max_running_requests_per_dp":64},{"model_path":"/models","tokenizer_path":"/models","tokenizer_mode":"auto","tokenizer_backend":"huggingface","tokenizer_worker_num":1,"detokenizer_worker_num":1,"skip_tokenizer_init":false,"load_format":"auto","model_loader_extra_config":"{}","trust_remote_code":true,"context_length":null,"is_embedding":false,"enable_multimodal":null,"revision":null,"model_impl":"auto","model_config_parser":"auto","json_model_override_args":"{}","dtype":"auto","quantization":null,"quantization_param_path":null,"kv_cache_dtype":"fp8_e4m3","enable_fp32_lm_head":false,"modelopt_quant":null,"modelopt_checkpoint_restore_path":null,"modelopt_checkpoint_save_path":null,"modelopt_export_path":null,"quantize_and_serve":false,"rl_quant_profile":null,"enable_tf32_matmul":false,"mem_fraction_static":0.862,"max_running_requests":256,"max_queued_requests":null,"max_total_tokens":null,"chunked_prefill_size":4096,"enable_dynamic_chunking":false,"prefill_reserve_budget":0,"max_prefill_tokens":16384,"prefill_max_requests":null,"schedule_policy":"fcfs","enable_priority_scheduling":false,"disable_priority_preemption":false,"default_priority_value":null,"abort_on_priority_when_disabled":false,"schedule_low_priority_values_first":false,"priority_scheduling_preemption_threshold":10,"retraction_policy":"length","schedule_conservativeness":0.3,"page_size":256,"c128_page_size":16,"swa_full_tokens_ratio":0.1,"disable_hybrid_swa_memory":false,"radix_eviction_policy":"lru","prefill_only_disable_kv_cache":false,"disable_radix_cache":false,"enable_page_major_kv_layout":false,"enable_unified_memory":false,"disable_chunked_prefix_cache":true,"disable_overlap_schedule":false,"num_continuous_decode_steps":1,"scheduler_recv_interval":1,"enable_mixed_chunk":false,"nccl_port":null,"dist_timeout":null,"dist_init_addr":null,"nnodes":1,"node_rank":0,"tp_size":4,"dcp_size":1,"pp_size":1,"pp_max_micro_batch_size":64,"pp_async_batch_depth":0,"dp_size":4,"load_balance_method":"round_robin","prefix_affinity_fallback":"total_tokens","prefix_affinity_max_load_skew":1.5,"prefix_affinity_hash_tokens":4096,"prefix_affinity_disable_token_fallback":false,"attn_cp_size":1,"moe_dp_size":1,"dwdp_size":1,"dcp_comm_backend":"ag_rs","dcp_replicate_q_proj":null,"enable_prefill_cp":false,"cp_strategy":null,"enable_dsa_cache_layer_split":false,"enable_dsa_prefill_context_parallel":false,"dsa_prefill_cp_mode":"round-robin-split","enable_prefill_context_parallel":false,"prefill_cp_mode":"in-seq-split","enable_cp_decode_attn_tp":false,"enable_dp_attention":true,"enable_dp_attention_local_control_broadcast":false,"enable_dp_lm_head":false,"enable_tp_lm_head_all_to_all":false,"enable_attn_tp_input_scattered":false,"disable_attn_tp_gather":false,"enable_p2p_check":false,"device":"cuda","base_gpu_id":0,"gpu_id_step":1,"random_seed":903930395,"mlx_enable_sampling":false,"watchdog_timeout":300,"soft_watchdog_timeout":null,"sleep_on_idle":false,"use_ray":false,"numa_node":null,"gc_threshold":null,"host":"0.0.0.0","port":30000,"fastapi_root_path":"","smg_grpc_mode":false,"grpc_mode":false,"grpc_port":null,"sidecar":null,"sidecar_args":null,"skip_server_warmup":false,"warmups":null,"enable_http2":false,"http2_max_concurrent_streams":200,"ssl_keyfile":null,"ssl_certfile":null,"ssl_ca_certs":null,"ssl_keyfile_password":null,"enable_ssl_refresh":false,"api_key":null,"admin_api_key":null,"served_model_name":"/models","weight_version":"default","chat_template":null,"hf_chat_template_name":null,"completion_template":null,"file_storage_path":"sglang_storage","enable_cache_report":false,"reasoning_parser":"deepseek-v4","default_chat_template_kwargs":null,"strip_thinking_cache":false,"enable_strict_thinking":false,"tool_call_parser":"deepseekv4","tool_server":null,"sampling_defaults":"model","asr_max_buffer_seconds":60,"asr_max_concurrent_sessions":32,"preferred_sampling_params":null,"allow_auto_truncate":false,"stream_interval":1,"batch_notify_size":16,"stream_response_default_include_usage":false,"incremental_streaming_output":false,"enable_streaming_session":false,"enable_session_radix_cache":false,"log_level":"info","log_level_http":null,"log_requests":false,"log_requests_level":2,"log_requests_format":"text","log_requests_target":null,"uvicorn_access_log_exclude_prefixes":[],"crash_dump_folder":null,"show_time_cost":false,"enable_metrics":false,"smg_http_sidecar_port":null,"enable_mfu_metrics":false,"enable_metrics_for_all_schedulers":false,"load_snapshot_publish_interval":15,"tokenizer_metrics_custom_labels_header":"x-custom-labels","tokenizer_metrics_allowed_custom_labels":null,"extra_metric_labels":null,"bucket_time_to_first_token":null,"bucket_inter_token_latency":null,"bucket_e2e_request_latency":null,"prompt_tokens_buckets":null,"generation_tokens_buckets":null,"gc_warning_threshold_secs":0.0,"decode_log_interval":40,"enable_request_time_stats_logging":false,"kv_events_config":null,"enable_forward_pass_metrics":false,"forward_pass_metrics_worker_id":"","forward_pass_metrics_ipc_name":null,"enable_trace":false,"trace_modules":"request","otlp_traces_endpoint":"localhost:4317","export_metrics_to_file":false,"export_metrics_to_file_dir":null,"stat_loggers":null,"constrained_json_whitespace_pattern":null,"constrained_json_disable_any_whitespace":false,"attention_backend":"dsv4","decode_attention_backend":null,"prefill_attention_backend":null,"sampling_backend":"flashinfer","grammar_backend":"xgrammar","radix_cache_backend":null,"mm_attention_backend":null,"fp8_gemm_runner_backend":"auto","fp4_gemm_runner_backend":"auto","bf16_gemm_backend":"auto","dsa_prefill_backend":null,"dsa_decode_backend":null,"dsa_paged_mqa_logits_backend":"auto","dsa_topk_backend":"sgl-kernel","disable_flashinfer_autotune":false,"flashinfer_autotune_skip_ops":null,"mamba_backend":"triton","cuda_graph_config":{"decode":{"backend":"full","max_bs":512,"bs":[1,2,4,8,12,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,272,288,304,320,336,352,368,384,400,416,432,448,464,480,496,512],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null},"prefill":{"backend":"disabled","max_bs":16384,"bs":[4,8,12,16,20,24,28,32,48,64,80,96,112,128,144,160,176,192,208,224,240,256,288,320,352,384,416,448,480,512,576,640,704,768,832,896,960,1024,1280,1536,1792,2048,2304,2560,2816,3072,3328,3584,3840,4096,4608,5120,5632,6144,6656,7168,7680,8192,8704,9216,9728,10240,10752,11264,11776,12288,12800,13312,13824,14336,14848,15360,15872,16384],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null}},"cuda_graph_backend_decode":null,"cuda_graph_backend_prefill":null,"cuda_graph_max_bs_decode":null,"cuda_graph_max_bs_prefill":null,"cuda_graph_bs_decode":null,"cuda_graph_bs_prefill":null,"cuda_graph_tc_compiler":null,"disable_prefill_cuda_graph":false,"disable_decode_cuda_graph":false,"disable_cuda_graph":false,"disable_cuda_graph_padding":false,"enable_profile_cuda_graph":false,"enable_cudagraph_gc":false,"debug_cuda_graph":false,"enable_layerwise_nvtx_marker":false,"enable_nccl_nvls":false,"enable_symm_mem":false,"triton_attention_reduce_in_fp32":false,"triton_attention_num_kv_splits":8,"triton_attention_split_tile_size":null,"flashinfer_mla_disable_ragged":false,"enable_fused_qk_norm_rope":false,"enable_precise_embedding_interpolation":false,"enable_fused_moe_sum_all_reduce":false,"enable_deepseek_v4_fp4_indexer":false,"disable_custom_all_reduce":false,"enable_mscclpp":false,"enable_torch_symm_mem":false,"enable_scattered_sconv":false,"pre_warm_nccl":false,"enable_quant_communications":false,"enable_flashinfer_allreduce_fusion":false,"enforce_disable_flashinfer_allreduce_fusion":false,"flashinfer_allreduce_fusion_backend":null,"enable_aiter_allreduce_fusion":false,"enable_torch_compile":false,"enable_torch_compile_debug_mode":false,"torch_compile_max_bs":32,"speculative_algorithm":null,"speculative_draft_model_path":null,"speculative_draft_model_revision":null,"speculative_draft_load_format":null,"speculative_num_steps":null,"speculative_eagle_topk":null,"speculative_num_draft_tokens":null,"speculative_dflash_block_size":null,"speculative_dspark_block_size":null,"speculative_dspark_sps_table_path":null,"speculative_dspark_confidence_sts_path":null,"speculative_dspark_align_verify_tokens_to_graph_tier":false,"speculative_accept_threshold_single":1.0,"speculative_accept_threshold_acc":1.0,"speculative_use_rejection_sampling":false,"speculative_token_map":null,"speculative_attention_mode":"prefill","speculative_draft_attention_backend":null,"speculative_draft_kv_cache_dtype":null,"speculative_draft_window_size":null,"speculative_moe_runner_backend":"auto","speculative_moe_a2a_backend":null,"speculative_draft_model_quantization":null,"_speculative_draft_quantization_explicitly_set":false,"speculative_skip_dp_mlp_sync":false,"enable_multi_layer_eagle":false,"speculative_adaptive":false,"speculative_adaptive_config":null,"decoupled_spec_bind_endpoint":null,"decoupled_spec_connect_endpoints":null,"decoupled_spec_rank":null,"decoupled_spec_role":"null","spec_trace_dir":null,"speculative_ngram_min_bfs_breadth":1,"speculative_ngram_max_bfs_breadth":10,"speculative_ngram_match_type":"BFS","speculative_ngram_max_trie_depth":18,"speculative_ngram_capacity":10000000,"speculative_ngram_external_corpus_path":null,"speculative_ngram_external_sam_budget":0,"speculative_ngram_external_corpus_max_tokens":10000000,"ep_size":4,"moe_a2a_backend":"deepep","moe_runner_backend":"auto","flashinfer_mxfp4_moe_precision":"default","deepep_mode":"auto","fuseep_mode":2,"deepep_dispatcher_output_dtype":"auto","ep_num_redundant_experts":0,"ep_dispatch_algorithm":null,"init_expert_location":"trivial","enable_eplb":false,"eplb_algorithm":"auto","eplb_rebalance_num_iterations":1000,"eplb_rebalance_layers_per_chunk":null,"eplb_min_rebalancing_utilization_threshold":1.0,"expert_distribution_recorder_mode":null,"expert_distribution_recorder_buffer_size":1000,"expert_balancedness_report_mode":"off","deepep_config":"{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}","moe_dense_tp_size":null,"elastic_ep_backend":null,"enable_elastic_expert_backup":false,"mooncake_ib_device":null,"enable_waterfill":false,"ep_join_mode":null,"ep_join_rank_offset":0,"elastic_ep_initial_size":null,"max_ep_size":null,"elastic_ep_scale_timeout":600,"elastic_ep_rejoin":false,"disable_flashinfer_cutlass_moe_fp4_allgather":false,"disable_shared_experts_fusion":false,"enforce_shared_experts_fusion":false,"max_mamba_cache_size":null,"mamba_ssm_dtype":null,"mamba_max_states_per_path":-1,"enable_mamba_cache_stochastic_rounding":false,"mamba_cache_philox_rounds":0,"mamba_full_memory_ratio":0.9,"mamba_radix_cache_strategy":"auto","uses_mamba_radix_cache":false,"mamba_track_interval":256,"enable_int8_mamba_checkpoint":false,"int8_mamba_ckpt_size":null,"linear_attn_backend":"triton","linear_attn_decode_backend":null,"linear_attn_prefill_backend":null,"linear_attn_verify_backend":null,"enable_linear_replayssm":false,"linear_replayssm_cache_len":16,"enable_linear_replayssm_spec":false,"enable_hierarchical_cache":false,"hicache_ratio":2.0,"hicache_size":0,"hicache_write_policy":"write_through","hicache_io_backend":"kernel","hicache_mem_layout":"page_first","hicache_storage_backend":null,"hicache_storage_prefetch_policy":"timeout","hicache_storage_backend_extra_config":null,"enable_hisparse":false,"hisparse_config":null,"enable_broadcast_mm_inputs_process":false,"enable_prefix_mm_cache":false,"mm_enable_dp_encoder":false,"mm_process_config":{},"mm_processor_worker_num":0,"mm_io_worker_num":0,"allowed_media_domains":[],"media_url_max_file_size_mb":64,"mm_preprocess_cache_size_mb":null,"trust_mm_content_hashes":false,"limit_mm_data_per_request":null,"enable_mm_global_cache":false,"image_processor_backend":"auto","mm_global_cache_backend":"mooncake","disable_fast_image_processor":false,"mm_feature_transport":"cpu","keep_mm_feature_on_device":false,"enable_lora":null,"enable_lora_overlap_loading":null,"max_lora_rank":null,"lora_target_modules":null,"lora_paths":null,"max_loaded_loras":null,"max_loras_per_batch":8,"lora_eviction_policy":"lru","lora_backend":"csgmv","max_lora_chunk_size":16,"experts_shared_outer_loras":null,"lora_use_virtual_experts":false,"lora_strict_loading":false,"lora_drain_wait_threshold":0.0,"enable_two_batch_overlap":false,"enable_single_batch_overlap":false,"tbo_token_distribution_threshold":0.48,"cpu_offload_gb":0,"offload_group_size":-1,"offload_num_in_group":1,"offload_prefetch_step":1,"offload_mode":"cpu","enable_lmcache":false,"lmcache_config_file":null,"enable_flexkv":false,"flexkv_config_file":null,"kt_weight_path":null,"kt_method":"AMXINT4","kt_cpuinfer":null,"kt_threadpool_count":2,"kt_num_gpu_experts":null,"kt_max_deferred_experts_per_token":null,"dllm_algorithm":null,"dllm_algorithm_config":null,"dllm_fdfo":true,"disaggregation_mode":"null","disaggregation_transfer_backend":"mooncake","disaggregation_bootstrap_port":8998,"disaggregation_ib_device":null,"disaggregation_decode_enable_radix_cache":false,"disaggregation_decode_enable_offload_kvcache":false,"disaggregation_decode_retraction_backup":"cpu_tensor","num_reserved_decode_tokens":512,"disaggregation_decode_extra_slots":null,"disaggregation_decode_polling_interval":1,"optimistic_prefill_attempts":0,"encoder_only":false,"language_only":false,"language_model_only":false,"encoder_transfer_backend":"zmq_to_scheduler","encoder_urls":[],"encoder_bootstrap_port":8997,"encoder_register_urls":[],"enable_adaptive_dispatch_to_encoder":false,"enable_pdmux":false,"pdmux_config_path":null,"sm_group_num":8,"startup_weight_load_mode":"serial","custom_weight_loader":[],"weight_loader_disable_mmap":false,"weight_loader_prefetch_checkpoints":false,"weight_loader_prefetch_num_threads":4,"weight_loader_drop_cache_after_load":false,"remote_instance_weight_loader_seed_instance_ip":null,"remote_instance_weight_loader_seed_instance_service_port":null,"remote_instance_weight_loader_send_weights_group_ports":null,"remote_instance_weight_loader_backend":"nccl","remote_instance_weight_loader_start_seed_via_transfer_engine":false,"engine_info_bootstrap_port":6789,"modelexpress_config":null,"download_dir":null,"model_checksum":null,"delete_ckpt_after_loading":false,"decrypted_config_file":null,"decrypted_draft_config_file":null,"checkpoint_engine_wait_weights_before_ready":false,"enable_prefill_delayer":false,"prefill_delayer_max_delay_passes":30,"prefill_delayer_token_usage_low_watermark":null,"prefill_delayer_forward_passes_buckets":null,"prefill_delayer_wait_seconds_buckets":null,"prefill_delayer_queue_min_ratio":null,"prefill_delayer_max_delay_ms":null,"min_free_slots_delay":null,"enable_deterministic_inference":false,"rl_on_policy_target":null,"kv_canary":"none","kv_canary_real_data":"none","kv_canary_sweep_interval":0,"enable_dynamic_batch_tokenizer":false,"dynamic_batch_tokenizer_batch_size":32,"dynamic_batch_tokenizer_batch_timeout":0.002,"enable_tokenizer_batch_encode":false,"disable_tokenizer_batch_decode":false,"debug_tensor_dump_output_folder":null,"debug_tensor_dump_layers":null,"debug_tensor_dump_input_file":null,"enable_memory_saver":false,"enable_weights_cpu_backup":false,"enable_draft_weights_cpu_backup":false,"enable_custom_logit_processor":false,"enable_return_hidden_states":false,"return_hidden_states_mode":null,"enable_return_routed_experts":false,"enable_return_indexer_topk":false,"disable_outlines_disk_cache":false,"enable_mis":false,"weight_cache_mode":"off","weight_cache_socket":null,"weight_cache_timeout":1800,"forward_hooks":null,"msprobe_dump_config":null,"_resolved_overrides":[["_deepseek_v4_overrides",{"attention_backend":"dsv4","page_size":256,"swa_full_tokens_ratio":0.1}],["_deepseek_v4_kv_cache_dtype",{"kv_cache_dtype":"fp8_e4m3"}],["_sampling_backend_default",{"sampling_backend":"flashinfer"}],["_tp_lm_head_all_to_all_default",{"enable_tp_lm_head_all_to_all":false}],["_a2a_ep_size",{"ep_size":4}],["_speculative_moe_runner_default",{"speculative_moe_runner_backend":"auto"}]],"grpc_worker_threads":4,"_quantization_explicitly_unset":false,"_cuda_graph_config_locked":[],"_declarations_materialized":true,"_mx_config_cache":{},"max_speculative_num_draft_tokens":null,"last_gen_throughput":0.0,"memory_usage":{"weight":42.971,"kvcache":0,"startup_available":14.253,"token_capacity":25820928,"token_capacity_swa":2582016,"graph":{"prefill":0.0,"decode":19.33,"target_verify":0.0,"draft_prefill":0.0,"draft_decode":0.0,"draft_extend":0.0}},"startup_time":{"load_weight":348.43866326799616,"kv_cache_allocation":0.8657742600189522,"scheduler_e2e":508.1397862319136,"cuda_graph":{"prefill":0.0,"decode":89.43686953000724,"target_verify":0.0,"draft_prefill":0.0,"draft_decode":0.0,"draft_extend":0.0}},"effective_max_running_requests_per_dp":64},{"model_path":"/models","tokenizer_path":"/models","tokenizer_mode":"auto","tokenizer_backend":"huggingface","tokenizer_worker_num":1,"detokenizer_worker_num":1,"skip_tokenizer_init":false,"load_format":"auto","model_loader_extra_config":"{}","trust_remote_code":true,"context_length":null,"is_embedding":false,"enable_multimodal":null,"revision":null,"model_impl":"auto","model_config_parser":"auto","json_model_override_args":"{}","dtype":"auto","quantization":null,"quantization_param_path":null,"kv_cache_dtype":"fp8_e4m3","enable_fp32_lm_head":false,"modelopt_quant":null,"modelopt_checkpoint_restore_path":null,"modelopt_checkpoint_save_path":null,"modelopt_export_path":null,"quantize_and_serve":false,"rl_quant_profile":null,"enable_tf32_matmul":false,"mem_fraction_static":0.862,"max_running_requests":256,"max_queued_requests":null,"max_total_tokens":null,"chunked_prefill_size":4096,"enable_dynamic_chunking":false,"prefill_reserve_budget":0,"max_prefill_tokens":16384,"prefill_max_requests":null,"schedule_policy":"fcfs","enable_priority_scheduling":false,"disable_priority_preemption":false,"default_priority_value":null,"abort_on_priority_when_disabled":false,"schedule_low_priority_values_first":false,"priority_scheduling_preemption_threshold":10,"retraction_policy":"length","schedule_conservativeness":0.3,"page_size":256,"c128_page_size":16,"swa_full_tokens_ratio":0.1,"disable_hybrid_swa_memory":false,"radix_eviction_policy":"lru","prefill_only_disable_kv_cache":false,"disable_radix_cache":false,"enable_page_major_kv_layout":false,"enable_unified_memory":false,"disable_chunked_prefix_cache":true,"disable_overlap_schedule":false,"num_continuous_decode_steps":1,"scheduler_recv_interval":1,"enable_mixed_chunk":false,"nccl_port":null,"dist_timeout":null,"dist_init_addr":null,"nnodes":1,"node_rank":0,"tp_size":4,"dcp_size":1,"pp_size":1,"pp_max_micro_batch_size":64,"pp_async_batch_depth":0,"dp_size":4,"load_balance_method":"round_robin","prefix_affinity_fallback":"total_tokens","prefix_affinity_max_load_skew":1.5,"prefix_affinity_hash_tokens":4096,"prefix_affinity_disable_token_fallback":false,"attn_cp_size":1,"moe_dp_size":1,"dwdp_size":1,"dcp_comm_backend":"ag_rs","dcp_replicate_q_proj":null,"enable_prefill_cp":false,"cp_strategy":null,"enable_dsa_cache_layer_split":false,"enable_dsa_prefill_context_parallel":false,"dsa_prefill_cp_mode":"round-robin-split","enable_prefill_context_parallel":false,"prefill_cp_mode":"in-seq-split","enable_cp_decode_attn_tp":false,"enable_dp_attention":true,"enable_dp_attention_local_control_broadcast":false,"enable_dp_lm_head":false,"enable_tp_lm_head_all_to_all":false,"enable_attn_tp_input_scattered":false,"disable_attn_tp_gather":false,"enable_p2p_check":false,"device":"cuda","base_gpu_id":0,"gpu_id_step":1,"random_seed":903930395,"mlx_enable_sampling":false,"watchdog_timeout":300,"soft_watchdog_timeout":null,"sleep_on_idle":false,"use_ray":false,"numa_node":null,"gc_threshold":null,"host":"0.0.0.0","port":30000,"fastapi_root_path":"","smg_grpc_mode":false,"grpc_mode":false,"grpc_port":null,"sidecar":null,"sidecar_args":null,"skip_server_warmup":false,"warmups":null,"enable_http2":false,"http2_max_concurrent_streams":200,"ssl_keyfile":null,"ssl_certfile":null,"ssl_ca_certs":null,"ssl_keyfile_password":null,"enable_ssl_refresh":false,"api_key":null,"admin_api_key":null,"served_model_name":"/models","weight_version":"default","chat_template":null,"hf_chat_template_name":null,"completion_template":null,"file_storage_path":"sglang_storage","enable_cache_report":false,"reasoning_parser":"deepseek-v4","default_chat_template_kwargs":null,"strip_thinking_cache":false,"enable_strict_thinking":false,"tool_call_parser":"deepseekv4","tool_server":null,"sampling_defaults":"model","asr_max_buffer_seconds":60,"asr_max_concurrent_sessions":32,"preferred_sampling_params":null,"allow_auto_truncate":false,"stream_interval":1,"batch_notify_size":16,"stream_response_default_include_usage":false,"incremental_streaming_output":false,"enable_streaming_session":false,"enable_session_radix_cache":false,"log_level":"info","log_level_http":null,"log_requests":false,"log_requests_level":2,"log_requests_format":"text","log_requests_target":null,"uvicorn_access_log_exclude_prefixes":[],"crash_dump_folder":null,"show_time_cost":false,"enable_metrics":false,"smg_http_sidecar_port":null,"enable_mfu_metrics":false,"enable_metrics_for_all_schedulers":false,"load_snapshot_publish_interval":15,"tokenizer_metrics_custom_labels_header":"x-custom-labels","tokenizer_metrics_allowed_custom_labels":null,"extra_metric_labels":null,"bucket_time_to_first_token":null,"bucket_inter_token_latency":null,"bucket_e2e_request_latency":null,"prompt_tokens_buckets":null,"generation_tokens_buckets":null,"gc_warning_threshold_secs":0.0,"decode_log_interval":40,"enable_request_time_stats_logging":false,"kv_events_config":null,"enable_forward_pass_metrics":false,"forward_pass_metrics_worker_id":"","forward_pass_metrics_ipc_name":null,"enable_trace":false,"trace_modules":"request","otlp_traces_endpoint":"localhost:4317","export_metrics_to_file":false,"export_metrics_to_file_dir":null,"stat_loggers":null,"constrained_json_whitespace_pattern":null,"constrained_json_disable_any_whitespace":false,"attention_backend":"dsv4","decode_attention_backend":null,"prefill_attention_backend":null,"sampling_backend":"flashinfer","grammar_backend":"xgrammar","radix_cache_backend":null,"mm_attention_backend":null,"fp8_gemm_runner_backend":"auto","fp4_gemm_runner_backend":"auto","bf16_gemm_backend":"auto","dsa_prefill_backend":null,"dsa_decode_backend":null,"dsa_paged_mqa_logits_backend":"auto","dsa_topk_backend":"sgl-kernel","disable_flashinfer_autotune":false,"flashinfer_autotune_skip_ops":null,"mamba_backend":"triton","cuda_graph_config":{"decode":{"backend":"full","max_bs":512,"bs":[1,2,4,8,12,16,24,32,40,48,56,64,72,80,88,96,104,112,120,128,136,144,152,160,168,176,184,192,200,208,216,224,232,240,248,256,272,288,304,320,336,352,368,384,400,416,432,448,464,480,496,512],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null},"prefill":{"backend":"disabled","max_bs":16384,"bs":[4,8,12,16,20,24,28,32,48,64,80,96,112,128,144,160,176,192,208,224,240,256,288,320,352,384,416,448,480,512,576,640,704,768,832,896,960,1024,1280,1536,1792,2048,2304,2560,2816,3072,3328,3584,3840,4096,4608,5120,5632,6144,6656,7168,7680,8192,8704,9216,9728,10240,10752,11264,11776,12288,12800,13312,13824,14336,14848,15360,15872,16384],"tc_compiler":"eager","full_prefill_max_req":null,"full_prefill_prefix_chunk_tokens":null}},"cuda_graph_backend_decode":null,"cuda_graph_backend_prefill":null,"cuda_graph_max_bs_decode":null,"cuda_graph_max_bs_prefill":null,"cuda_graph_bs_decode":null,"cuda_graph_bs_prefill":null,"cuda_graph_tc_compiler":null,"disable_prefill_cuda_graph":false,"disable_decode_cuda_graph":false,"disable_cuda_graph":false,"disable_cuda_graph_padding":false,"enable_profile_cuda_graph":false,"enable_cudagraph_gc":false,"debug_cuda_graph":false,"enable_layerwise_nvtx_marker":false,"enable_nccl_nvls":false,"enable_symm_mem":false,"triton_attention_reduce_in_fp32":false,"triton_attention_num_kv_splits":8,"triton_attention_split_tile_size":null,"flashinfer_mla_disable_ragged":false,"enable_fused_qk_norm_rope":false,"enable_precise_embedding_interpolation":false,"enable_fused_moe_sum_all_reduce":false,"enable_deepseek_v4_fp4_indexer":false,"disable_custom_all_reduce":false,"enable_mscclpp":false,"enable_torch_symm_mem":false,"enable_scattered_sconv":false,"pre_warm_nccl":false,"enable_quant_communications":false,"enable_flashinfer_allreduce_fusion":false,"enforce_disable_flashinfer_allreduce_fusion":false,"flashinfer_allreduce_fusion_backend":null,"enable_aiter_allreduce_fusion":false,"enable_torch_compile":false,"enable_torch_compile_debug_mode":false,"torch_compile_max_bs":32,"speculative_algorithm":null,"speculative_draft_model_path":null,"speculative_draft_model_revision":null,"speculative_draft_load_format":null,"speculative_num_steps":null,"speculative_eagle_topk":null,"speculative_num_draft_tokens":null,"speculative_dflash_block_size":null,"speculative_dspark_block_size":null,"speculative_dspark_sps_table_path":null,"speculative_dspark_confidence_sts_path":null,"speculative_dspark_align_verify_tokens_to_graph_tier":false,"speculative_accept_threshold_single":1.0,"speculative_accept_threshold_acc":1.0,"speculative_use_rejection_sampling":false,"speculative_token_map":null,"speculative_attention_mode":"prefill","speculative_draft_attention_backend":null,"speculative_draft_kv_cache_dtype":null,"speculative_draft_window_size":null,"speculative_moe_runner_backend":"auto","speculative_moe_a2a_backend":null,"speculative_draft_model_quantization":null,"_speculative_draft_quantization_explicitly_set":false,"speculative_skip_dp_mlp_sync":false,"enable_multi_layer_eagle":false,"speculative_adaptive":false,"speculative_adaptive_config":null,"decoupled_spec_bind_endpoint":null,"decoupled_spec_connect_endpoints":null,"decoupled_spec_rank":null,"decoupled_spec_role":"null","spec_trace_dir":null,"speculative_ngram_min_bfs_breadth":1,"speculative_ngram_max_bfs_breadth":10,"speculative_ngram_match_type":"BFS","speculative_ngram_max_trie_depth":18,"speculative_ngram_capacity":10000000,"speculative_ngram_external_corpus_path":null,"speculative_ngram_external_sam_budget":0,"speculative_ngram_external_corpus_max_tokens":10000000,"ep_size":4,"moe_a2a_backend":"deepep","moe_runner_backend":"auto","flashinfer_mxfp4_moe_precision":"default","deepep_mode":"auto","fuseep_mode":2,"deepep_dispatcher_output_dtype":"auto","ep_num_redundant_experts":0,"ep_dispatch_algorithm":null,"init_expert_location":"trivial","enable_eplb":false,"eplb_algorithm":"auto","eplb_rebalance_num_iterations":1000,"eplb_rebalance_layers_per_chunk":null,"eplb_min_rebalancing_utilization_threshold":1.0,"expert_distribution_recorder_mode":null,"expert_distribution_recorder_buffer_size":1000,"expert_balancedness_report_mode":"off","deepep_config":"{\"normal_dispatch\":{\"num_sms\":96},\"normal_combine\":{\"num_sms\":96}}","moe_dense_tp_size":null,"elastic_ep_backend":null,"enable_elastic_expert_backup":false,"mooncake_ib_device":null,"enable_waterfill":false,"ep_join_mode":null,"ep_join_rank_offset":0,"elastic_ep_initial_size":null,"max_ep_size":null,"elastic_ep_scale_timeout":600,"elastic_ep_rejoin":false,"disable_flashinfer_cutlass_moe_fp4_allgather":false,"disable_shared_experts_fusion":false,"enforce_shared_experts_fusion":false,"max_mamba_cache_size":null,"mamba_ssm_dtype":null,"mamba_max_states_per_path":-1,"enable_mamba_cache_stochastic_rounding":false,"mamba_cache_philox_rounds":0,"mamba_full_memory_ratio":0.9,"mamba_radix_cache_strategy":"auto","uses_mamba_radix_cache":false,"mamba_track_interval":256,"enable_int8_mamba_checkpoint":false,"int8_mamba_ckpt_size":null,"linear_attn_backend":"triton","linear_attn_decode_backend":null,"linear_attn_prefill_backend":null,"linear_attn_verify_backend":null,"enable_linear_replayssm":false,"linear_replayssm_cache_len":16,"enable_linear_replayssm_spec":false,"enable_hierarchical_cache":false,"hicache_ratio":2.0,"hicache_size":0,"hicache_write_policy":"write_through","hicache_io_backend":"kernel","hicache_mem_layout":"page_first","hicache_storage_backend":null,"hicache_storage_prefetch_policy":"timeout","hicache_storage_backend_extra_config":null,"enable_hisparse":false,"hisparse_config":null,"enable_broadcast_mm_inputs_process":false,"enable_prefix_mm_cache":false,"mm_enable_dp_encoder":false,"mm_process_config":{},"mm_processor_worker_num":0,"mm_io_worker_num":0,"allowed_media_domains":[],"media_url_max_file_size_mb":64,"mm_preprocess_cache_size_mb":null,"trust_mm_content_hashes":false,"limit_mm_data_per_request":null,"enable_mm_global_cache":false,"image_processor_backend":"auto","mm_global_cache_backend":"mooncake","disable_fast_image_processor":false,"mm_feature_transport":"cpu","keep_mm_feature_on_device":false,"enable_lora":null,"enable_lora_overlap_loading":null,"max_lora_rank":null,"lora_target_modules":null,"lora_paths":null,"max_loaded_loras":null,"max_loras_per_batch":8,"lora_eviction_policy":"lru","lora_backend":"csgmv","max_lora_chunk_size":16,"experts_shared_outer_loras":null,"lora_use_virtual_experts":false,"lora_strict_loading":false,"lora_drain_wait_threshold":0.0,"enable_two_batch_overlap":false,"enable_single_batch_overlap":false,"tbo_token_distribution_threshold":0.48,"cpu_offload_gb":0,"offload_group_size":-1,"offload_num_in_group":1,"offload_prefetch_step":1,"offload_mode":"cpu","enable_lmcache":false,"lmcache_config_file":null,"enable_flexkv":false,"flexkv_config_file":null,"kt_weight_path":null,"kt_method":"AMXINT4","kt_cpuinfer":null,"kt_threadpool_count":2,"kt_num_gpu_experts":null,"kt_max_deferred_experts_per_token":null,"dllm_algorithm":null,"dllm_algorithm_config":null,"dllm_fdfo":true,"disaggregation_mode":"null","disaggregation_transfer_backend":"mooncake","disaggregation_bootstrap_port":8998,"disaggregation_ib_device":null,"disaggregation_decode_enable_radix_cache":false,"disaggregation_decode_enable_offload_kvcache":false,"disaggregation_decode_retraction_backup":"cpu_tensor","num_reserved_decode_tokens":512,"disaggregation_decode_extra_slots":null,"disaggregation_decode_polling_interval":1,"optimistic_prefill_attempts":0,"encoder_only":false,"language_only":false,"language_model_only":false,"encoder_transfer_backend":"zmq_to_scheduler","encoder_urls":[],"encoder_bootstrap_port":8997,"encoder_register_urls":[],"enable_adaptive_dispatch_to_encoder":false,"enable_pdmux":false,"pdmux_config_path":null,"sm_group_num":8,"startup_weight_load_mode":"serial","custom_weight_loader":[],"weight_loader_disable_mmap":false,"weight_loader_prefetch_checkpoints":false,"weight_loader_prefetch_num_threads":4,"weight_loader_drop_cache_after_load":false,"remote_instance_weight_loader_seed_instance_ip":null,"remote_instance_weight_loader_seed_instance_service_port":null,"remote_instance_weight_loader_send_weights_group_ports":null,"remote_instance_weight_loader_backend":"nccl","remote_instance_weight_loader_start_seed_via_transfer_engine":false,"engine_info_bootstrap_port":6789,"modelexpress_config":null,"download_dir":null,"model_checksum":null,"delete_ckpt_after_loading":false,"decrypted_config_file":null,"decrypted_draft_config_file":null,"checkpoint_engine_wait_weights_before_ready":false,"enable_prefill_delayer":false,"prefill_delayer_max_delay_passes":30,"prefill_delayer_token_usage_low_watermark":null,"prefill_delayer_forward_passes_buckets":null,"prefill_delayer_wait_seconds_buckets":null,"prefill_delayer_queue_min_ratio":null,"prefill_delayer_max_delay_ms":null,"min_free_slots_delay":null,"enable_deterministic_inference":false,"rl_on_policy_target":null,"kv_canary":"none","kv_canary_real_data":"none","kv_canary_sweep_interval":0,"enable_dynamic_batch_tokenizer":false,"dynamic_batch_tokenizer_batch_size":32,"dynamic_batch_tokenizer_batch_timeout":0.002,"enable_tokenizer_batch_encode":false,"disable_tokenizer_batch_decode":false,"debug_tensor_dump_output_folder":null,"debug_tensor_dump_layers":null,"debug_tensor_dump_input_file":null,"enable_memory_saver":false,"enable_weights_cpu_backup":false,"enable_draft_weights_cpu_backup":false,"enable_custom_logit_processor":false,"enable_return_hidden_states":false,"return_hidden_states_mode":null,"enable_return_routed_experts":false,"enable_return_indexer_topk":false,"disable_outlines_disk_cache":false,"enable_mis":false,"weight_cache_mode":"off","weight_cache_socket":null,"weight_cache_timeout":1800,"forward_hooks":null,"msprobe_dump_config":null,"_resolved_overrides":[["_deepseek_v4_overrides",{"attention_backend":"dsv4","page_size":256,"swa_full_tokens_ratio":0.1}],["_deepseek_v4_kv_cache_dtype",{"kv_cache_dtype":"fp8_e4m3"}],["_sampling_backend_default",{"sampling_backend":"flashinfer"}],["_tp_lm_head_all_to_all_default",{"enable_tp_lm_head_all_to_all":false}],["_a2a_ep_size",{"ep_size":4}],["_speculative_moe_runner_default",{"speculative_moe_runner_backend":"auto"}]],"grpc_worker_threads":4,"_quantization_explicitly_unset":false,"_cuda_graph_config_locked":[],"_declarations_materialized":true,"_mx_config_cache":{},"max_speculative_num_draft_tokens":null,"last_gen_throughput":0.0,"memory_usage":{"weight":42.971,"kvcache":0,"startup_available":15.19,"token_capacity":25820928,"token_capacity_swa":2582016,"graph":{"prefill":0.0,"decode":18.549,"target_verify":0.0,"draft_prefill":0.0,"draft_decode":0.0,"draft_extend":0.0}},"startup_time":{"load_weight":348.5880510379793,"kv_cache_allocation":0.8970330059528351,"scheduler_e2e":508.1053619449958,"cuda_graph":{"prefill":0.0,"decode":89.43910579301883,"target_verify":0.0,"draft_prefill":0.0,"draft_decode":0.0,"draft_extend":0.0}},"effective_max_running_requests_per_dp":64}],"version":"0.5.18","kv_events":null}
\ No newline at end of file
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/comparison.md b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/comparison.md
new file mode 100644
index 0000000..87ef16a
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/comparison.md
@@ -0,0 +1,121 @@
+# B300 DeepSeek-V4-Flash / GLM-5.3 Comparison: b300-dsv4-glm53-dev4-20260909-084427
+
+## Run Status
+
+- Finished: 78.39%
+- Pass: 164
+- Timeout pass: 3
+- Invalid: 0
+- Fail: 18
+
+## Service Configuration And Capacity
+
+| Profile | TP | DP | EP | Max running | Max tokens/rank | Chunk | KV dtype | Attention | MoE runner | MoE A2A | Speculative |
+|---|---:|---:|---:|---:|---:|---:|---|---|---|---|---|
+| dsv4-low-latency | - | - | - | - | - | - | - | - | - | - | - |
+| dsv4-balanced | - | - | - | - | - | - | - | - | - | - | - |
+| glm53-low-latency | 8 | 1 | 1 | 48 | 2696000 | 16384 | fp8_e4m3 | dsa | flashinfer_trtllm | none | EAGLE |
+| glm53-high-throughput | 8 | 8 | 8 | 256 | 2542208 | 4096 | fp8_e4m3 | dsa | auto | deepep | EAGLE |
+
+## Throughput Peaks And Their Latency
+
+Only validated PASS points are considered.
+
+| Shape | Mode | Peak input TPS @ C | TTFT P95 (ms) | Peak output TPS @ C | TPOT P95 (ms) |
+|---|---|---:|---:|---:|---:|
+| 16k_512 | dsv4-low-latency | 60485.71 @ C256 | 48385.38 | 1890.18 @ C256 | 226.23 |
+| 16k_512 | dsv4-balanced | 57436.40 @ C256 | 62849.59 | 1794.89 @ C256 | 134.28 |
+| 16k_512 | glm53-low-latency | 24021.20 @ C256 | 144384.42 | 750.66 @ C256 | 102.36 |
+| 16k_512 | glm53-high-throughput | 31889.21 @ C128 | 30890.29 | 996.54 @ C128 | 165.33 |
+| 16k_1 | dsv4-low-latency | 66726.27 @ C32 | 7810.68 | 4.07 @ C32 | 0.00 |
+| 16k_1 | dsv4-balanced | 65093.70 @ C32 | 7943.21 | 3.97 @ C32 | 0.00 |
+| 1k_128 | dsv4-low-latency | 29913.48 @ C256 | 3544.47 | 3739.19 @ C256 | 99.14 |
+| 1k_128 | dsv4-balanced | 36265.26 @ C256 | 4505.51 | 4533.16 @ C256 | 59.65 |
+| 1k_128 | glm53-low-latency | 14308.93 @ C64 | 1848.75 | 1788.62 @ C64 | 40.49 |
+| 1k_128 | glm53-high-throughput | 15193.07 @ C128 | 2569.96 | 1899.13 @ C128 | 87.74 |
+| 1k_4k | dsv4-low-latency | 5283.08 @ C256 | 3658.46 | 21132.32 @ C256 | 17.31 |
+| 1k_4k | dsv4-balanced | 2441.07 @ C256 | 4386.00 | 9764.26 @ C256 | 26.51 |
+| 1k_4k | glm53-low-latency | 1530.74 @ C128 | 56046.13 | 6122.95 @ C128 | 11.08 |
+| 1k_4k | glm53-high-throughput | 2653.31 @ C256 | 4540.61 | 10613.25 @ C256 | 22.36 |
+| 64k_1 | dsv4-low-latency | 58913.97 @ C4 | 4440.74 | 0.90 @ C4 | 0.00 |
+| 64k_1 | dsv4-balanced | 65864.61 @ C16 | 15823.61 | 1.01 @ C16 | 0.00 |
+| 64k_1 | glm53-low-latency | 25336.86 @ C8 | 20633.79 | 0.39 @ C8 | 0.00 |
+| 64k_1 | glm53-high-throughput | 53720.01 @ C8 | 9891.34 | 0.82 @ C8 | 0.00 |
+| 64k_512 | dsv4-low-latency | 56712.78 @ C64 | 55290.66 | 443.07 @ C64 | 238.49 |
+| 64k_512 | glm53-low-latency | 24220.17 @ C64 | 128260.27 | 189.22 @ C64 | 323.81 |
+| 64k_512 | glm53-high-throughput | 33437.72 @ C64 | 57344.18 | 261.23 @ C64 | 329.77 |
+| 128k_1 | dsv4-low-latency | 51351.81 @ C2 | 5095.93 | 0.39 @ C2 | 0.00 |
+| 128k_1 | glm53-low-latency | 22544.03 @ C2 | 11620.01 | 0.17 @ C2 | 0.00 |
+| 128k_512 | dsv4-low-latency | 48253.24 @ C32 | 62228.62 | 188.49 @ C32 | 214.99 |
+| 128k_512 | glm53-low-latency | 21787.55 @ C32 | 142627.99 | 85.11 @ C32 | 339.34 |
+| 128k_512 | glm53-high-throughput | 27503.67 @ C32 | 59905.46 | 107.44 @ C32 | 401.75 |
+| 256k_1 | dsv4-low-latency | 40426.30 @ C2 | 13069.41 | 0.15 @ C2 | 0.00 |
+| 256k_1 | glm53-low-latency | 17620.27 @ C2 | 29920.79 | 0.07 @ C2 | 0.00 |
+| 512k_1 | glm53-low-latency | 12497.10 @ C2 | 84423.37 | 0.02 @ C2 | 0.00 |
+| 512k_1 | glm53-high-throughput | 24677.77 @ C4 | 86777.89 | 0.05 @ C4 | 0.00 |
+| 1m_boundary_1 | glm53-low-latency | 7914.10 @ C4 | 509514.56 | 0.01 @ C4 | 0.00 |
+| 1m_boundary_1 | glm53-high-throughput | 18580.87 @ C4 | 225402.20 | 0.02 @ C4 | 0.00 |
+
+## Capacity And Failure Boundaries
+
+| Mode | Shape | C | Status | Evidence |
+|---|---|---:|---|---|
+| dsv4-low-latency | 512k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-low-latency/points/512k_1_c1.failed` |
+| dsv4-balanced | 1k_4k | 1 | TIMEOUT_PASS | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/points/1k_4k_c1.json` |
+| glm53-low-latency | 16k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/16k_1_c1.json` |
+| glm53-low-latency | 64k_1 | 4 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c4.json` |
+| glm53-low-latency | 64k_1 | 16 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c16.json` |
+| glm53-low-latency | 64k_1 | 32 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c32.json` |
+| glm53-low-latency | 64k_1 | 64 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c64.json` |
+| glm53-low-latency | 64k_1 | 128 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/64k_1_c128.json` |
+| glm53-low-latency | 128k_1 | 8 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/128k_1_c8.json` |
+| glm53-low-latency | 128k_1 | 16 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/128k_1_c16.json` |
+| glm53-low-latency | 128k_1 | 32 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/128k_1_c32.json` |
+| glm53-low-latency | 128k_1 | 64 | TIMEOUT_PASS | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/128k_1_c64.json` |
+| glm53-low-latency | 256k_1 | 4 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/points/256k_1_c4.json` |
+| glm53-high-throughput | 16k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/16k_1_c1.json` |
+| glm53-high-throughput | 1k_4k | 1 | TIMEOUT_PASS | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/1k_4k_c1.json` |
+| glm53-high-throughput | 64k_1 | 16 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/64k_1_c16.json` |
+| glm53-high-throughput | 64k_1 | 32 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/64k_1_c32.json` |
+| glm53-high-throughput | 64k_1 | 64 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/64k_1_c64.json` |
+| glm53-high-throughput | 64k_1 | 128 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/64k_1_c128.json` |
+| glm53-high-throughput | 128k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/128k_1_c1.json` |
+| glm53-high-throughput | 256k_1 | 1 | FAIL | `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/points/256k_1_c1.json` |
+
+## Service Error Summary
+
+### dsv4-low-latency
+
+- Matched lines: 319
+- Source: `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-low-latency/server.final.log`
+- `[rank0]:[W910 05:05:37.307446434 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 2415919104 bytes (free: 753532928, total: 287428771840).`
+- `[rank1]:[W910 05:05:37.312576349 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 1 while trying to allocate 2415919104 bytes (free: 724172800, total: 287428771840).`
+- `[rank3]:[W910 05:05:37.312868926 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 3 while trying to allocate 2415919104 bytes (free: 1066008576, total: 287428771840).`
+- `[rank2]:[W910 05:05:37.314989750 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 2 while trying to allocate 2415919104 bytes (free: 724172800, total: 287428771840).`
+- `[rank1]:[W910 05:05:38.764535635 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 1 while trying to allocate 3221225472 bytes (free: 870973440, total: 287428771840).`
+
+### dsv4-balanced
+
+- Matched lines: 0
+- Source: `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/dsv4-balanced/server.final.log`
+
+### glm53-low-latency
+
+- Matched lines: 141
+- Source: `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-low-latency/server.final.log`
+- `[rank2]:[W909 15:01:40.129936229 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 2 while trying to allocate 8858370048 bytes (free: 2053701632, total: 287428771840).`
+- `[rank3]:[W909 15:01:40.130091786 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 3 while trying to allocate 8858370048 bytes (free: 2116616192, total: 287428771840).`
+- `[rank1]:[W909 15:01:40.130433377 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 1 while trying to allocate 8858370048 bytes (free: 2053701632, total: 287428771840).`
+- `[rank0]:[W909 15:01:40.130656418 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 0 while trying to allocate 8885633024 bytes (free: 2229862400, total: 287428771840).`
+- `[rank7]:[W909 15:01:40.131055935 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 7 while trying to allocate 8994684928 bytes (free: 2714304512, total: 287428771840).`
+
+### glm53-high-throughput
+
+- Matched lines: 586
+- Source: `/data/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/glm53-high-throughput/server.final.log`
+- `/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: /dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDAinit failed for transport: IBGDA`
+- `/dvs/p4/build/sw/rel/gpgpu/toolkit/r13.0/main_nvshmem/src/host/transport/transport.cpp:nvshmemi_transport_init:282: init failed for transport: IBGDA`
+- `[rank1]:[W909 19:58:27.770722283 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 1 while trying to allocate 1950351360 bytes (free: 1160314880, total: 287428771840).`
+- `[rank2]:[W909 19:58:42.743872031 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 2 while trying to allocate 1950351360 bytes (free: 724107264, total: 287428771840).`
+- `[rank3]:[W909 19:58:55.386477067 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 3 while trying to allocate 1883242496 bytes (free: 1875443712, total: 287428771840).`
+
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/progress.json b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/progress.json
new file mode 100644
index 0000000..7e243b1
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/progress.json
@@ -0,0 +1,47 @@
+{
+  "expected": 236,
+  "fail": 18,
+  "finished_percent": 78.39,
+  "invalid": 0,
+  "modes": {
+    "dsv4-balanced": {
+      "expected": 59,
+      "fail": 0,
+      "finished_percent": 52.54,
+      "invalid": 0,
+      "pass": 30,
+      "pending": 28,
+      "timeout_pass": 1
+    },
+    "dsv4-low-latency": {
+      "expected": 59,
+      "fail": 1,
+      "finished_percent": 91.53,
+      "invalid": 0,
+      "pass": 53,
+      "pending": 5,
+      "timeout_pass": 0
+    },
+    "glm53-high-throughput": {
+      "expected": 59,
+      "fail": 7,
+      "finished_percent": 77.97,
+      "invalid": 0,
+      "pass": 38,
+      "pending": 13,
+      "timeout_pass": 1
+    },
+    "glm53-low-latency": {
+      "expected": 59,
+      "fail": 10,
+      "finished_percent": 91.53,
+      "invalid": 0,
+      "pass": 43,
+      "pending": 5,
+      "timeout_pass": 1
+    }
+  },
+  "pass": 164,
+  "pending": 51,
+  "timeout_pass": 3
+}
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/summary.md b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/summary.md
new file mode 100644
index 0000000..0bff236
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/final-summary/summary.md
@@ -0,0 +1,200 @@
+# B300 DeepSeek-V4-Flash / GLM-5.3 Text Matrix: b300-dsv4-glm53-dev4-20260909-084427
+
+## Progress
+
+| Mode | Finished | Expected | Pass | Timeout pass | Invalid | Fail | Progress |
+|---|---:|---:|---:|---:|---:|---:|---:|
+| dsv4-low-latency | 54 | 59 | 53 | 0 | 0 | 1 | 91.53% |
+| dsv4-balanced | 31 | 59 | 30 | 1 | 0 | 0 | 52.54% |
+| glm53-low-latency | 54 | 59 | 43 | 1 | 0 | 10 | 91.53% |
+| glm53-high-throughput | 46 | 59 | 38 | 1 | 0 | 7 | 77.97% |
+
+## Results
+
+| Mode | Shape | C | Status | Input TPS | Output TPS | TTFT P95 (ms) | TPOT P95 (ms) |
+|---|---|---:|---|---:|---:|---:|---:|
+| dsv4-low-latency | 16k_512 | 1 | PASS | 13128.54 | 410.27 | 265.87 | 3.07 |
+| dsv4-low-latency | 16k_512 | 8 | PASS | 33936.55 | 1060.52 | 1517.12 | 10.23 |
+| dsv4-low-latency | 16k_512 | 32 | PASS | 49777.89 | 1555.56 | 6230.65 | 31.87 |
+| dsv4-low-latency | 16k_512 | 64 | PASS | 55645.12 | 1738.91 | 12091.63 | 57.79 |
+| dsv4-low-latency | 16k_512 | 128 | PASS | 59108.19 | 1847.13 | 24712.64 | 114.28 |
+| dsv4-low-latency | 16k_512 | 256 | PASS | 60485.71 | 1890.18 | 48385.38 | 226.23 |
+| dsv4-low-latency | 16k_1 | 1 | PASS | 61163.57 | 3.73 | 266.91 | 0.00 |
+| dsv4-low-latency | 16k_1 | 8 | PASS | 66664.03 | 4.07 | 1950.41 | 0.00 |
+| dsv4-low-latency | 16k_1 | 32 | PASS | 66726.27 | 4.07 | 7810.68 | 0.00 |
+| dsv4-low-latency | 16k_1 | 64 | PASS | 66288.99 | 4.05 | 16147.44 | 0.00 |
+| dsv4-low-latency | 16k_1 | 128 | PASS | 66596.05 | 4.06 | 31378.16 | 0.00 |
+| dsv4-low-latency | 16k_1 | 256 | PASS | 66600.10 | 4.06 | 62914.28 | 0.00 |
+| dsv4-low-latency | 1k_128 | 1 | PASS | 2041.91 | 255.24 | 183.12 | 3.53 |
+| dsv4-low-latency | 1k_128 | 8 | PASS | 3578.88 | 447.36 | 3387.56 | 31.29 |
+| dsv4-low-latency | 1k_128 | 32 | PASS | 7184.95 | 898.12 | 1708.71 | 49.89 |
+| dsv4-low-latency | 1k_128 | 64 | PASS | 11724.62 | 1465.58 | 1205.29 | 62.99 |
+| dsv4-low-latency | 1k_128 | 128 | PASS | 20916.51 | 2614.56 | 2101.48 | 68.93 |
+| dsv4-low-latency | 1k_128 | 256 | PASS | 29913.48 | 3739.19 | 3544.47 | 99.14 |
+| dsv4-low-latency | 1k_4k | 1 | PASS | 151.34 | 605.35 | 183.93 | 2.28 |
+| dsv4-low-latency | 1k_4k | 8 | PASS | 709.27 | 2837.08 | 451.52 | 3.52 |
+| dsv4-low-latency | 1k_4k | 32 | PASS | 1679.25 | 6717.00 | 754.10 | 6.34 |
+| dsv4-low-latency | 1k_4k | 64 | PASS | 2459.31 | 9837.23 | 1200.11 | 9.86 |
+| dsv4-low-latency | 1k_4k | 128 | PASS | 3217.13 | 12868.50 | 2188.22 | 14.14 |
+| dsv4-low-latency | 1k_4k | 256 | PASS | 5283.08 | 21132.32 | 3658.46 | 17.31 |
+| dsv4-low-latency | 64k_1 | 1 | PASS | 57079.88 | 0.87 | 1180.60 | 0.00 |
+| dsv4-low-latency | 64k_1 | 4 | PASS | 58913.97 | 0.90 | 4440.74 | 0.00 |
+| dsv4-low-latency | 64k_1 | 8 | PASS | 58818.80 | 0.90 | 8883.26 | 0.00 |
+| dsv4-low-latency | 64k_1 | 16 | PASS | 58871.47 | 0.90 | 17757.38 | 0.00 |
+| dsv4-low-latency | 64k_1 | 32 | PASS | 58821.54 | 0.90 | 35523.52 | 0.00 |
+| dsv4-low-latency | 64k_1 | 64 | PASS | 58852.44 | 0.90 | 71226.16 | 0.00 |
+| dsv4-low-latency | 64k_1 | 128 | PASS | 58701.82 | 0.90 | 142742.12 | 0.00 |
+| dsv4-low-latency | 64k_512 | 1 | PASS | 32785.22 | 256.13 | 1146.10 | 2.91 |
+| dsv4-low-latency | 64k_512 | 4 | PASS | 47367.52 | 370.06 | 3651.92 | 13.00 |
+| dsv4-low-latency | 64k_512 | 8 | PASS | 51470.88 | 402.12 | 8142.53 | 27.01 |
+| dsv4-low-latency | 64k_512 | 16 | PASS | 54390.07 | 424.92 | 13764.60 | 53.61 |
+| dsv4-low-latency | 64k_512 | 32 | PASS | 55570.81 | 434.15 | 27317.01 | 113.22 |
+| dsv4-low-latency | 64k_512 | 64 | PASS | 56712.78 | 443.07 | 55290.66 | 238.49 |
+| dsv4-low-latency | 128k_1 | 1 | PASS | 50299.31 | 0.38 | 2606.88 | 0.00 |
+| dsv4-low-latency | 128k_1 | 2 | PASS | 51351.81 | 0.39 | 5095.93 | 0.00 |
+| dsv4-low-latency | 128k_1 | 4 | PASS | 51297.32 | 0.39 | 10307.95 | 0.00 |
+| dsv4-low-latency | 128k_1 | 8 | PASS | 51293.08 | 0.39 | 20440.51 | 0.00 |
+| dsv4-low-latency | 128k_1 | 16 | PASS | 51305.80 | 0.39 | 40813.75 | 0.00 |
+| dsv4-low-latency | 128k_1 | 32 | PASS | 51293.67 | 0.39 | 81674.88 | 0.00 |
+| dsv4-low-latency | 128k_1 | 64 | PASS | 51257.15 | 0.39 | 163688.22 | 0.00 |
+| dsv4-low-latency | 128k_512 | 1 | PASS | 28968.88 | 113.16 | 2613.09 | 4.47 |
+| dsv4-low-latency | 128k_512 | 2 | PASS | 35668.55 | 139.33 | 2886.10 | 14.56 |
+| dsv4-low-latency | 128k_512 | 4 | PASS | 41221.32 | 161.02 | 5149.64 | 25.35 |
+| dsv4-low-latency | 128k_512 | 8 | PASS | 44544.04 | 174.00 | 13131.55 | 55.16 |
+| dsv4-low-latency | 128k_512 | 16 | PASS | 46962.57 | 183.45 | 31281.83 | 107.27 |
+| dsv4-low-latency | 128k_512 | 32 | PASS | 48253.24 | 188.49 | 62228.62 | 214.99 |
+| dsv4-low-latency | 256k_1 | 1 | PASS | 39875.71 | 0.15 | 6610.03 | 0.00 |
+| dsv4-low-latency | 256k_1 | 2 | PASS | 40426.30 | 0.15 | 13069.41 | 0.00 |
+| dsv4-low-latency | 256k_1 | 4 | PASS | 40371.52 | 0.15 | 26029.44 | 0.00 |
+| dsv4-low-latency | 512k_1 | 1 | FAIL | - | - | - | - |
+| dsv4-balanced | 16k_512 | 1 | PASS | 2517.53 | 78.67 | 1067.18 | 11.26 |
+| dsv4-balanced | 16k_512 | 8 | PASS | 16954.74 | 529.84 | 2578.91 | 12.69 |
+| dsv4-balanced | 16k_512 | 32 | PASS | 36100.44 | 1128.14 | 8087.71 | 25.54 |
+| dsv4-balanced | 16k_512 | 64 | PASS | 45336.60 | 1416.77 | 16042.56 | 41.92 |
+| dsv4-balanced | 16k_512 | 128 | PASS | 52115.96 | 1628.62 | 31955.08 | 73.50 |
+| dsv4-balanced | 16k_512 | 256 | PASS | 57436.40 | 1794.89 | 62849.59 | 134.28 |
+| dsv4-balanced | 16k_1 | 1 | PASS | 16704.37 | 1.02 | 1001.65 | 0.00 |
+| dsv4-balanced | 16k_1 | 8 | PASS | 63912.26 | 3.90 | 2334.27 | 0.00 |
+| dsv4-balanced | 16k_1 | 32 | PASS | 65093.70 | 3.97 | 7943.21 | 0.00 |
+| dsv4-balanced | 16k_1 | 64 | PASS | 64887.20 | 3.96 | 16398.83 | 0.00 |
+| dsv4-balanced | 16k_1 | 128 | PASS | 64875.87 | 3.96 | 32514.97 | 0.00 |
+| dsv4-balanced | 16k_1 | 256 | PASS | 64245.92 | 3.92 | 65033.17 | 0.00 |
+| dsv4-balanced | 1k_128 | 1 | PASS | 643.09 | 80.39 | 342.42 | 10.62 |
+| dsv4-balanced | 1k_128 | 8 | PASS | 4391.78 | 548.97 | 649.01 | 10.73 |
+| dsv4-balanced | 1k_128 | 32 | PASS | 13757.02 | 1719.63 | 943.32 | 12.48 |
+| dsv4-balanced | 1k_128 | 64 | PASS | 21219.84 | 2652.48 | 1451.23 | 17.88 |
+| dsv4-balanced | 1k_128 | 128 | PASS | 29472.67 | 3684.08 | 2841.20 | 27.68 |
+| dsv4-balanced | 1k_128 | 256 | PASS | 36265.26 | 4533.16 | 4505.51 | 59.65 |
+| dsv4-balanced | 1k_4k | 1 | TIMEOUT_PASS | 21.98 | 87.94 | 370.83 | 14.15 |
+| dsv4-balanced | 1k_4k | 8 | PASS | 187.97 | 751.87 | 799.79 | 10.62 |
+| dsv4-balanced | 1k_4k | 32 | PASS | 631.65 | 2526.62 | 916.72 | 12.59 |
+| dsv4-balanced | 1k_4k | 64 | PASS | 1112.31 | 4449.26 | 1374.52 | 14.35 |
+| dsv4-balanced | 1k_4k | 128 | PASS | 1913.20 | 7652.81 | 3162.08 | 16.57 |
+| dsv4-balanced | 1k_4k | 256 | PASS | 2441.07 | 9764.26 | 4386.00 | 26.51 |
+| dsv4-balanced | 64k_1 | 1 | PASS | 17118.25 | 0.26 | 3874.73 | 0.00 |
+| dsv4-balanced | 64k_1 | 4 | PASS | 61303.08 | 0.94 | 4407.47 | 0.00 |
+| dsv4-balanced | 64k_1 | 8 | PASS | 65564.19 | 1.00 | 8356.27 | 0.00 |
+| dsv4-balanced | 64k_1 | 16 | PASS | 65864.61 | 1.01 | 15823.61 | 0.00 |
+| dsv4-balanced | 64k_1 | 32 | PASS | 65504.60 | 1.00 | 32070.29 | 0.00 |
+| dsv4-balanced | 64k_1 | 64 | PASS | 65616.93 | 1.00 | 64081.25 | 0.00 |
+| dsv4-balanced | 64k_1 | 128 | PASS | 65401.37 | 1.00 | 128671.94 | 0.00 |
+| glm53-low-latency | 16k_512 | 1 | PASS | 7882.31 | 246.32 | 612.89 | 4.36 |
+| glm53-low-latency | 16k_512 | 8 | PASS | 16384.46 | 512.01 | 3892.78 | 22.46 |
+| glm53-low-latency | 16k_512 | 32 | PASS | 21303.26 | 665.73 | 15127.36 | 69.03 |
+| glm53-low-latency | 16k_512 | 64 | PASS | 23502.93 | 734.47 | 29490.35 | 105.62 |
+| glm53-low-latency | 16k_512 | 128 | PASS | 23920.46 | 747.51 | 63407.51 | 101.14 |
+| glm53-low-latency | 16k_512 | 256 | PASS | 24021.20 | 750.66 | 144384.42 | 102.36 |
+| glm53-low-latency | 16k_1 | 1 | FAIL | 26686.87 | 1.63 | 606.55 | 0.00 |
+| glm53-low-latency | 1k_128 | 1 | PASS | 2005.88 | 250.73 | 70.19 | 5.55 |
+| glm53-low-latency | 1k_128 | 8 | PASS | 6062.38 | 757.80 | 813.10 | 15.41 |
+| glm53-low-latency | 1k_128 | 32 | PASS | 11946.48 | 1493.31 | 1204.54 | 31.01 |
+| glm53-low-latency | 1k_128 | 64 | PASS | 14308.93 | 1788.62 | 1848.75 | 40.49 |
+| glm53-low-latency | 1k_128 | 128 | PASS | 14124.94 | 1765.62 | 6332.63 | 42.03 |
+| glm53-low-latency | 1k_128 | 256 | PASS | 13944.40 | 1743.05 | 15823.61 | 43.56 |
+| glm53-low-latency | 1k_4k | 1 | PASS | 109.08 | 436.33 | 70.69 | 3.32 |
+| glm53-low-latency | 1k_4k | 8 | PASS | 477.85 | 1911.42 | 435.51 | 5.56 |
+| glm53-low-latency | 1k_4k | 32 | PASS | 1191.77 | 4767.10 | 892.67 | 9.33 |
+| glm53-low-latency | 1k_4k | 64 | PASS | 1458.93 | 5835.71 | 19807.67 | 10.84 |
+| glm53-low-latency | 1k_4k | 128 | PASS | 1530.74 | 6122.95 | 56046.13 | 11.08 |
+| glm53-low-latency | 1k_4k | 256 | PASS | 1526.22 | 6104.87 | 140390.00 | 11.26 |
+| glm53-low-latency | 64k_1 | 1 | PASS | 24918.24 | 0.38 | 2625.80 | 0.00 |
+| glm53-low-latency | 64k_1 | 4 | FAIL | 25330.44 | 0.39 | 10319.69 | 0.00 |
+| glm53-low-latency | 64k_1 | 8 | PASS | 25336.86 | 0.39 | 20633.79 | 0.00 |
+| glm53-low-latency | 64k_1 | 16 | FAIL | 25368.13 | 0.39 | 41257.75 | 0.00 |
+| glm53-low-latency | 64k_1 | 32 | FAIL | 25376.71 | 0.39 | 82506.66 | 0.00 |
+| glm53-low-latency | 64k_1 | 64 | FAIL | 25399.08 | 0.39 | 164948.88 | 0.00 |
+| glm53-low-latency | 64k_1 | 128 | FAIL | 25411.76 | 0.39 | 329925.47 | 0.00 |
+| glm53-low-latency | 64k_512 | 1 | PASS | 15809.05 | 123.51 | 2626.48 | 4.40 |
+| glm53-low-latency | 64k_512 | 4 | PASS | 20932.05 | 163.53 | 5658.47 | 26.76 |
+| glm53-low-latency | 64k_512 | 8 | PASS | 22427.36 | 175.21 | 13072.49 | 61.58 |
+| glm53-low-latency | 64k_512 | 16 | PASS | 23295.48 | 182.00 | 31634.83 | 127.45 |
+| glm53-low-latency | 64k_512 | 32 | PASS | 23917.39 | 186.85 | 62694.72 | 243.87 |
+| glm53-low-latency | 64k_512 | 64 | PASS | 24220.17 | 189.22 | 128260.27 | 323.81 |
+| glm53-low-latency | 128k_1 | 1 | PASS | 22283.97 | 0.17 | 5887.63 | 0.00 |
+| glm53-low-latency | 128k_1 | 2 | PASS | 22544.03 | 0.17 | 11620.01 | 0.00 |
+| glm53-low-latency | 128k_1 | 4 | PASS | 22540.08 | 0.17 | 23234.84 | 0.00 |
+| glm53-low-latency | 128k_1 | 8 | FAIL | 22556.35 | 0.17 | 46447.98 | 0.00 |
+| glm53-low-latency | 128k_1 | 16 | FAIL | 22547.53 | 0.17 | 92876.53 | 0.00 |
+| glm53-low-latency | 128k_1 | 32 | FAIL | 22549.85 | 0.17 | 185810.07 | 0.00 |
+| glm53-low-latency | 128k_1 | 64 | TIMEOUT_PASS | 22567.39 | 0.17 | 371590.24 | 0.00 |
+| glm53-low-latency | 128k_512 | 1 | PASS | 17525.45 | 68.46 | 5894.95 | 4.62 |
+| glm53-low-latency | 128k_512 | 2 | PASS | 19570.06 | 76.45 | 6325.56 | 27.21 |
+| glm53-low-latency | 128k_512 | 4 | PASS | 20526.29 | 80.18 | 11700.22 | 62.53 |
+| glm53-low-latency | 128k_512 | 8 | PASS | 21132.30 | 82.55 | 28726.77 | 145.04 |
+| glm53-low-latency | 128k_512 | 16 | PASS | 21643.46 | 84.54 | 70612.08 | 282.81 |
+| glm53-low-latency | 128k_512 | 32 | PASS | 21787.55 | 85.11 | 142627.99 | 339.34 |
+| glm53-low-latency | 256k_1 | 1 | PASS | 17456.68 | 0.07 | 15141.04 | 0.00 |
+| glm53-low-latency | 256k_1 | 2 | PASS | 17620.27 | 0.07 | 29920.79 | 0.00 |
+| glm53-low-latency | 256k_1 | 4 | FAIL | 17584.82 | 0.07 | 59691.73 | 0.00 |
+| glm53-low-latency | 512k_1 | 1 | PASS | 12420.94 | 0.02 | 42470.03 | 0.00 |
+| glm53-low-latency | 512k_1 | 2 | PASS | 12497.10 | 0.02 | 84423.37 | 0.00 |
+| glm53-low-latency | 512k_1 | 4 | PASS | 12494.82 | 0.02 | 168091.37 | 0.00 |
+| glm53-low-latency | 1m_boundary_1 | 1 | PASS | 7897.30 | 0.01 | 132967.28 | 0.00 |
+| glm53-low-latency | 1m_boundary_1 | 2 | PASS | 7846.52 | 0.01 | 269850.67 | 0.00 |
+| glm53-low-latency | 1m_boundary_1 | 4 | PASS | 7914.10 | 0.01 | 509514.56 | 0.00 |
+| glm53-high-throughput | 16k_512 | 1 | PASS | 1753.70 | 54.80 | 1706.01 | 16.40 |
+| glm53-high-throughput | 16k_512 | 8 | PASS | 7602.48 | 237.58 | 2952.39 | 38.86 |
+| glm53-high-throughput | 16k_512 | 32 | PASS | 15478.02 | 483.69 | 10709.46 | 82.99 |
+| glm53-high-throughput | 16k_512 | 64 | PASS | 22864.39 | 714.51 | 17177.98 | 115.88 |
+| glm53-high-throughput | 16k_512 | 128 | PASS | 31889.21 | 996.54 | 30890.29 | 165.33 |
+| glm53-high-throughput | 16k_512 | 256 | PASS | 25722.47 | 803.83 | 58208.90 | 435.35 |
+| glm53-high-throughput | 16k_1 | 1 | FAIL | 10265.87 | 0.63 | 1635.66 | 0.00 |
+| glm53-high-throughput | 1k_128 | 1 | PASS | 440.61 | 55.08 | 354.14 | 17.25 |
+| glm53-high-throughput | 1k_128 | 8 | PASS | 2216.13 | 277.02 | 658.97 | 33.33 |
+| glm53-high-throughput | 1k_128 | 32 | PASS | 5512.82 | 689.10 | 1351.79 | 55.76 |
+| glm53-high-throughput | 1k_128 | 64 | PASS | 8781.68 | 1097.71 | 1544.35 | 75.33 |
+| glm53-high-throughput | 1k_128 | 128 | PASS | 15193.07 | 1899.13 | 2569.96 | 87.74 |
+| glm53-high-throughput | 1k_128 | 256 | PASS | 14521.72 | 1815.21 | 4585.68 | 192.57 |
+| glm53-high-throughput | 1k_4k | 1 | TIMEOUT_PASS | 15.31 | 61.25 | 722.89 | 21.05 |
+| glm53-high-throughput | 1k_4k | 8 | PASS | 122.82 | 491.26 | 585.67 | 17.31 |
+| glm53-high-throughput | 1k_4k | 32 | PASS | 458.53 | 1834.10 | 964.13 | 18.18 |
+| glm53-high-throughput | 1k_4k | 64 | PASS | 772.99 | 3091.95 | 1480.98 | 20.33 |
+| glm53-high-throughput | 1k_4k | 128 | PASS | 1421.86 | 5687.44 | 2492.89 | 23.05 |
+| glm53-high-throughput | 1k_4k | 256 | PASS | 2653.31 | 10613.25 | 4540.61 | 22.36 |
+| glm53-high-throughput | 64k_1 | 1 | PASS | 10461.13 | 0.16 | 6387.30 | 0.00 |
+| glm53-high-throughput | 64k_1 | 4 | PASS | 31751.47 | 0.48 | 8371.38 | 0.00 |
+| glm53-high-throughput | 64k_1 | 8 | PASS | 53720.01 | 0.82 | 9891.34 | 0.00 |
+| glm53-high-throughput | 64k_1 | 16 | FAIL | 56023.39 | 0.85 | 19477.62 | 0.00 |
+| glm53-high-throughput | 64k_1 | 32 | FAIL | 56599.60 | 0.86 | 36913.14 | 0.00 |
+| glm53-high-throughput | 64k_1 | 64 | FAIL | 56554.68 | 0.86 | 74260.73 | 0.00 |
+| glm53-high-throughput | 64k_1 | 128 | FAIL | 56357.44 | 0.86 | 153394.58 | 0.00 |
+| glm53-high-throughput | 64k_512 | 1 | PASS | 4707.22 | 36.78 | 6471.79 | 16.01 |
+| glm53-high-throughput | 64k_512 | 4 | PASS | 8844.03 | 69.09 | 8240.16 | 54.88 |
+| glm53-high-throughput | 64k_512 | 8 | PASS | 13286.74 | 103.80 | 9928.84 | 96.89 |
+| glm53-high-throughput | 64k_512 | 16 | PASS | 18863.70 | 147.37 | 19171.29 | 135.82 |
+| glm53-high-throughput | 64k_512 | 32 | PASS | 25109.46 | 196.17 | 29462.53 | 206.58 |
+| glm53-high-throughput | 64k_512 | 64 | PASS | 33437.72 | 261.23 | 57344.18 | 329.77 |
+| glm53-high-throughput | 128k_1 | 1 | FAIL | 10100.97 | 0.08 | 13350.24 | 0.00 |
+| glm53-high-throughput | 128k_512 | 1 | PASS | 6350.91 | 24.81 | 13384.88 | 16.13 |
+| glm53-high-throughput | 128k_512 | 2 | PASS | 7862.65 | 30.71 | 13400.23 | 41.95 |
+| glm53-high-throughput | 128k_512 | 4 | PASS | 9986.29 | 39.01 | 16714.05 | 92.60 |
+| glm53-high-throughput | 128k_512 | 8 | PASS | 13503.28 | 52.75 | 20219.46 | 171.97 |
+| glm53-high-throughput | 128k_512 | 16 | PASS | 19923.94 | 77.83 | 39164.93 | 264.45 |
+| glm53-high-throughput | 128k_512 | 32 | PASS | 27503.67 | 107.44 | 59905.46 | 401.75 |
+| glm53-high-throughput | 256k_1 | 1 | FAIL | 9204.74 | 0.04 | 29442.21 | 0.00 |
+| glm53-high-throughput | 512k_1 | 1 | PASS | 7532.88 | 0.01 | 71779.28 | 0.00 |
+| glm53-high-throughput | 512k_1 | 2 | PASS | 14141.46 | 0.03 | 77207.47 | 0.00 |
+| glm53-high-throughput | 512k_1 | 4 | PASS | 24677.77 | 0.05 | 86777.89 | 0.00 |
+| glm53-high-throughput | 1m_boundary_1 | 1 | PASS | 5330.26 | 0.01 | 204815.71 | 0.00 |
+| glm53-high-throughput | 1m_boundary_1 | 2 | PASS | 9956.15 | 0.01 | 214992.66 | 0.00 |
+| glm53-high-throughput | 1m_boundary_1 | 4 | PASS | 18580.87 | 0.02 | 225402.20 | 0.00 |
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/manual-stop.complete b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/manual-stop.complete
new file mode 100644
index 0000000..aba7028
--- /dev/null
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/manual-stop.complete
@@ -0,0 +1 @@
+2026-09-10T07:46:07Z
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/matrix.log b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/matrix.log
index c85eaec..9b3fe10 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/matrix.log
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/matrix.log
@@ -518,3 +518,9 @@ run_full_text_matrix.sh: line 247: 1797795 Killed                  timeout --sig
 [2026-09-10T07:21:11+00:00] warning profile=dsv4-balanced point=64k_1_c32 zero_ttft_count=1
 [2026-09-10T07:21:11+00:00] pass profile=dsv4-balanced point=64k_1_c32
 [2026-09-10T07:21:12+00:00] run profile=dsv4-balanced shape=64k_1 isl=65536 osl=1 c=64 prompts=320
+[2026-09-10T07:26:57+00:00] warning profile=dsv4-balanced point=64k_1_c64 zero_ttft_count=3
+[2026-09-10T07:26:57+00:00] pass profile=dsv4-balanced point=64k_1_c64
+[2026-09-10T07:26:58+00:00] run profile=dsv4-balanced shape=64k_1 isl=65536 osl=1 c=128 prompts=640
+[2026-09-10T07:38:13+00:00] warning profile=dsv4-balanced point=64k_1_c128 zero_ttft_count=5
+[2026-09-10T07:38:13+00:00] pass profile=dsv4-balanced point=64k_1_c128
+[2026-09-10T07:38:14+00:00] run profile=dsv4-balanced shape=64k_512 isl=65536 osl=512 c=1 prompts=64
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/point_status.csv b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/point_status.csv
index c6042f8..aa06bdd 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/point_status.csv
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/point_status.csv
@@ -182,3 +182,5 @@ timestamp,profile,point,status,rc
 2026-09-10T07:16:30+00:00,dsv4-balanced,64k_1_c8,PASS,0
 2026-09-10T07:18:09+00:00,dsv4-balanced,64k_1_c16,PASS,0
 2026-09-10T07:21:11+00:00,dsv4-balanced,64k_1_c32,PASS,0
+2026-09-10T07:26:57+00:00,dsv4-balanced,64k_1_c64,PASS,0
+2026-09-10T07:38:13+00:00,dsv4-balanced,64k_1_c128,PASS,0
diff --git a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/started_points.csv b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/started_points.csv
index dd1601b..eefee1b 100644
--- a/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/started_points.csv
+++ b/experiments/b300/dsv4_flash_glm53_sglang_text_matrix/snapshot-20260910-024355/b300-dsv4-glm53-text-matrix/b300-dsv4-glm53-dev4-20260909-084427/started_points.csv
@@ -183,3 +183,5 @@ timestamp,profile,shape,isl,osl,concurrency,num_prompts
 2026-09-10T07:16:31+00:00,dsv4-balanced,64k_1,65536,1,16,80
 2026-09-10T07:18:10+00:00,dsv4-balanced,64k_1,65536,1,32,160
 2026-09-10T07:21:12+00:00,dsv4-balanced,64k_1,65536,1,64,320
+2026-09-10T07:26:58+00:00,dsv4-balanced,64k_1,65536,1,128,640
+2026-09-10T07:38:14+00:00,dsv4-balanced,64k_512,65536,512,1,64