From e8ff3ce1e8dd0f5cc2dd2b6dc601c984c9a73e89 Mon Sep 17 00:00:00 2001
From: Zhiyi Hong <2497491955@qq.com>
Date: Wed, 19 Aug 2026 10:47:26 +0800
Subject: [PATCH] [Test] Add exact Kimi SM120 PR validation point

---
 README.md                                     |   2 +
 .../Dockerfile.pr_validation                  |  30 ++
 .../run_pr_representative_point.sh            | 418 ++++++++++++++++++
 3 files changed, 450 insertions(+)
 create mode 100644 experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/Dockerfile.pr_validation
 create mode 100755 experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/run_pr_representative_point.sh

diff --git a/README.md b/README.md
index 5969fe5..6f674bb 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,7 @@
 # sskj — 多平台大模型推理性能基准测试项目
 
+**更新（2026-08-19 10:45:18 CST）**：新增 Kimi-K3 SM120 SGLang Draft PR 的单点代表性验收入口。使用精确 SGLang `300c87a`、FlashInfer #4460 实现 `b525c51` 和统一镜像，在 601-604 四节点 TP32×EP4 上仅复测 16K→1、C=8、Chunk=8K 的 Marlin/FlashInfer A/B，各 3 次重复；完整命令、镜像身份、原始日志和汇总统一落入单个 Run 目录。
+
 **更新（2026-08-18 23:00:42 CST）**：完成依赖 FlashInfer #4460 的 SGLang Draft PR 收敛。确认不提交任何 FlashInfer PR，也不 vendor 或 pin 未合并 kernel；在 601 GPU6 上从 #4460 源码构建 FlashInfer 0.6.18 wheel，SGLang 定向测试 8/8 通过。Draft 标题、依赖、兼容策略、测试和四机 EP4 数据见 `experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/SGLANG_DRAFT_PR.md`。
 
 **更新（2026-08-18 22:21:40 CST）**：完成 Kimi-K3 SM120 FlashInfer MXFP4 去重审计。确认 CUTLASS SiTU kernel 已由未合并的 FlashInfer PR #4460 实现，不再提交平行 kernel；SGLang 贡献收敛为 Kimi gate/up 与 scale 布局、SiTU 4.0/25.0 参数映射、非连续输入和 SM120 attention-residual guard，并保留 601-604 EP4 的全部服务级结果。详见 `experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/UPSTREAM_DUPLICATION_AUDIT.md`。
diff --git a/experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/Dockerfile.pr_validation b/experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/Dockerfile.pr_validation
new file mode 100644
index 0000000..f4efc63
--- /dev/null
+++ b/experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/Dockerfile.pr_validation
@@ -0,0 +1,30 @@
+ARG BASE_IMAGE=lmsysorg/sglang:kimi-k3-fiv617situ-warm
+FROM ${BASE_IMAGE}
+
+ARG SGLANG_COMMIT
+ARG FLASHINFER_COMMIT
+
+ENV FLASHINFER_DISABLE_VERSION_CHECK=1 \
+    PYTHONPATH=/sgl-workspace/sglang/python \
+    PYTHONUNBUFFERED=1
+
+COPY sglang_kernel-0.4.6.post1-cp310-abi3-manylinux2014_x86_64.whl /tmp/
+COPY flashinfer_python-0.6.18-py3-none-any.whl /tmp/
+RUN python3 -m pip install --no-deps --force-reinstall \
+      /tmp/sglang_kernel-0.4.6.post1-cp310-abi3-manylinux2014_x86_64.whl \
+      /tmp/flashinfer_python-0.6.18-py3-none-any.whl && \
+    rm -f /tmp/*.whl
+
+# Replace the image's older Python package with the exact Draft tree. The
+# matching 0.4.6.post1 sglang-kernel wheel is installed above.
+RUN rm -rf /sgl-workspace/sglang/python/sglang
+COPY sglang/ /sgl-workspace/sglang/python/sglang/
+COPY test_mxfp4_sm120_cutlass.py /opt/pr-tests/test_mxfp4_sm120_cutlass.py
+COPY source_identity.txt /opt/pr-build/source_identity.txt
+
+RUN python3 -m compileall -q /sgl-workspace/sglang/python/sglang && \
+    python3 -c "import inspect; from flashinfer.fused_moe import cutlass_fused_moe; from flashinfer.fused_moe.core import ActivationType; assert hasattr(ActivationType, 'Situ'); assert 'situ_beta' in inspect.signature(cutlass_fused_moe).parameters; import sglang.srt.layers.quantization.mxfp4"
+
+LABEL ai.meta-stone.purpose="Kimi-K3 SM120 SGLang Draft representative validation" \
+      ai.meta-stone.sglang.commit="${SGLANG_COMMIT}" \
+      ai.meta-stone.flashinfer.commit="${FLASHINFER_COMMIT}"
diff --git a/experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/run_pr_representative_point.sh b/experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/run_pr_representative_point.sh
new file mode 100755
index 0000000..32cc896
--- /dev/null
+++ b/experiments/pro6000/kimi3_pro6000_sglang_sm120_flashinfer_mxfp4/run_pr_representative_point.sh
@@ -0,0 +1,418 @@
+#!/usr/bin/env bash
+# Validate the exact SGLang Draft with one four-node Kimi-K3 serving point.
+set -Eeuo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
+# shellcheck source=/dev/null
+source "${REPO_ROOT}/scripts/common/lib.sh"
+
+ACTION="${1:-all}"
+RUN_ID="${RUN_ID:-kimi3-sm120-pr-representative-$(date '+%Y%m%d-%H%M%S')}"
+RESULT_ROOT="${RESULT_ROOT:-${SCRIPT_DIR}/results/${RUN_ID}}"
+
+MODEL_PATH="${MODEL_PATH:-/data/hf_models/Kimi-K3}"
+SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-kimi-k3}"
+BASE_IMAGE="${BASE_IMAGE:-lmsysorg/sglang:kimi-k3-fiv617situ-warm}"
+PR_IMAGE="${PR_IMAGE:-local/sglang:kimi-k3-sm120-pr-300c87a-fi-b525c51}"
+SGLANG_SOURCE="${SGLANG_SOURCE:-/data/hzy/src/sglang-kimi-sm120-draft-wt}"
+FLASHINFER_SOURCE="${FLASHINFER_SOURCE:-/data/hzy/src/flashinfer-pr4460-kimi-test-wt}"
+ARTIFACT_DIR="${ARTIFACT_DIR:-/data/hzy/artifacts/sglang-pr-kimi-sm120-300c87a}"
+FLASHINFER_WHEEL="${FLASHINFER_WHEEL:-${ARTIFACT_DIR}/flashinfer_python-0.6.18-py3-none-any.whl}"
+SGLANG_KERNEL_WHEEL="${SGLANG_KERNEL_WHEEL:-${ARTIFACT_DIR}/sglang_kernel-0.4.6.post1-cp310-abi3-manylinux2014_x86_64.whl}"
+BUILD_CONTEXT="${BUILD_CONTEXT:-/tmp/kimi3_sm120_pr_validation_context}"
+FLASHINFER_CACHE="${FLASHINFER_CACHE:-/data/hzy/cache/flashinfer-pr-b525c51}"
+
+HEAD_HOST="${HEAD_HOST:-174.1.60.1}"
+NODE_SSH_USER="${NODE_SSH_USER:-user}"
+NODE_HOSTS=(174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4)
+SSH_OPTS=(-o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=10)
+DIST_PORT="${DIST_PORT:-20000}"
+PORT="${PORT:-30000}"
+CONTAINER_PREFIX="kimi3_sm120_pr_validation"
+
+INPUT_LEN=16384
+OUTPUT_LEN=1
+CONCURRENCY=8
+CHUNKED_PREFILL_SIZE=8192
+NUM_PROMPTS="${NUM_PROMPTS:-40}"
+REPEATS="${REPEATS:-3}"
+WARMUP_REQUESTS="${WARMUP_REQUESTS:-2}"
+HEALTH_WAIT_S="${HEALTH_WAIT_S:-2400}"
+BACKENDS=(marlin flashinfer_mxfp4)
+
+mkdir -p "${RESULT_ROOT}"/{build,service,raw,bench,gpu}
+log_init "${RESULT_ROOT}/orchestrator.log"
+
+require_password() {
+  if [[ -z "${SUDO_PASSWORD:-}" && -n "${SUDO_PASSWORD_FILE:-}" ]]; then
+    [[ -r "${SUDO_PASSWORD_FILE}" ]] || {
+      echo "ERROR: cannot read SUDO_PASSWORD_FILE=${SUDO_PASSWORD_FILE}" >&2
+      exit 2
+    }
+    IFS= read -r SUDO_PASSWORD <"${SUDO_PASSWORD_FILE}"
+  fi
+  [[ -n "${SUDO_PASSWORD:-}" ]] || {
+    echo "ERROR: set SUDO_PASSWORD or SUDO_PASSWORD_FILE" >&2
+    exit 2
+  }
+}
+
+is_head() { [[ "$1" == "${HEAD_HOST}" ]]; }
+
+sudo_host() {
+  local host="$1"
+  shift
+  require_password
+  if is_head "$host"; then
+    printf '%s\n' "${SUDO_PASSWORD}" | sudo -S -p '' -- "$@"
+  else
+    local remote_cmd
+    printf -v remote_cmd '%q ' "$@"
+    printf '%s\n' "${SUDO_PASSWORD}" | \
+      ssh "${SSH_OPTS[@]}" "${NODE_SSH_USER}@${host}" \
+        "sudo -S -p '' -- ${remote_cmd}"
+  fi
+}
+
+check_inputs() {
+  local path
+  for path in "${MODEL_PATH}" "${SGLANG_SOURCE}/python/sglang" \
+    "${FLASHINFER_SOURCE}" "${FLASHINFER_WHEEL}" \
+    "${SGLANG_KERNEL_WHEEL}" \
+    "${SCRIPT_DIR}/Dockerfile.pr_validation"; do
+    [[ -e "$path" ]] || { echo "ERROR: missing ${path}" >&2; exit 2; }
+  done
+
+  SGLANG_COMMIT="$(git -C "${SGLANG_SOURCE}" rev-parse HEAD)"
+  FLASHINFER_COMMIT="$(git -C "${FLASHINFER_SOURCE}" rev-parse HEAD~1)"
+  [[ "${SGLANG_COMMIT}" == 300c87a431ac40d3e7817246376b7fe20932db09 ]] || {
+    echo "ERROR: unexpected SGLang commit ${SGLANG_COMMIT}" >&2
+    exit 2
+  }
+  [[ "${FLASHINFER_COMMIT}" == b525c51* ]] || {
+    echo "ERROR: unexpected FlashInfer implementation commit ${FLASHINFER_COMMIT}" >&2
+    exit 2
+  }
+  export SGLANG_COMMIT FLASHINFER_COMMIT
+}
+
+prepare_build_context() {
+  log "preparing exact Draft build context"
+  rm -rf "${BUILD_CONTEXT}"
+  mkdir -p "${BUILD_CONTEXT}/sglang"
+  cp "${SCRIPT_DIR}/Dockerfile.pr_validation" "${BUILD_CONTEXT}/Dockerfile"
+  cp "${FLASHINFER_WHEEL}" "${BUILD_CONTEXT}/"
+  cp "${SGLANG_KERNEL_WHEEL}" "${BUILD_CONTEXT}/"
+  cp "${SGLANG_SOURCE}/test/registered/unit/layers/quantization/test_mxfp4_sm120_cutlass.py" \
+    "${BUILD_CONTEXT}/test_mxfp4_sm120_cutlass.py"
+  tar -C "${SGLANG_SOURCE}/python/sglang" -cf - . | \
+    tar -C "${BUILD_CONTEXT}/sglang" -xf -
+  {
+    printf 'sglang=%s\n' "${SGLANG_COMMIT}"
+    printf 'flashinfer_implementation=%s\n' "${FLASHINFER_COMMIT}"
+    sha256sum "${FLASHINFER_WHEEL}" "${SGLANG_KERNEL_WHEEL}"
+  } >"${BUILD_CONTEXT}/source_identity.txt"
+  cp "${BUILD_CONTEXT}/source_identity.txt" "${RESULT_ROOT}/build/"
+  du -sh "${BUILD_CONTEXT}" | tee "${RESULT_ROOT}/build/context_size.txt"
+}
+
+stage_context() {
+  local host="$1"
+  is_head "$host" && return
+  ssh "${SSH_OPTS[@]}" "${NODE_SSH_USER}@${host}" \
+    "rm -rf '${BUILD_CONTEXT}' && mkdir -p '${BUILD_CONTEXT}'"
+  tar -C "${BUILD_CONTEXT}" -cf - . | \
+    ssh "${SSH_OPTS[@]}" "${NODE_SSH_USER}@${host}" \
+      "tar -C '${BUILD_CONTEXT}' -xf -"
+}
+
+build_one_node() {
+  local host="$1"
+  sudo_host "$host" docker build \
+    --build-arg "BASE_IMAGE=${BASE_IMAGE}" \
+    --build-arg "SGLANG_COMMIT=${SGLANG_COMMIT}" \
+    --build-arg "FLASHINFER_COMMIT=${FLASHINFER_COMMIT}" \
+    --tag "${PR_IMAGE}" "${BUILD_CONTEXT}" \
+    >"${RESULT_ROOT}/build/${host}.log" 2>&1
+  sudo_host "$host" docker image inspect "${PR_IMAGE}" \
+    --format '{{.Id}} {{.Size}} {{json .Config.Labels}}' \
+    >"${RESULT_ROOT}/build/${host}.image.txt"
+  sudo_host "$host" docker run --rm --entrypoint python3 "${PR_IMAGE}" -c \
+    "import importlib.metadata as m; print('sglang=' + m.version('sglang')); print('sglang-kernel=' + m.version('sglang-kernel')); print('flashinfer-python=' + m.version('flashinfer-python')); print(open('/opt/pr-build/source_identity.txt').read(), end='')" \
+    >"${RESULT_ROOT}/build/${host}.packages.txt"
+}
+
+build_all_nodes() {
+  check_inputs
+  prepare_build_context
+  local host pid rc=0
+  for host in "${NODE_HOSTS[@]}"; do stage_context "$host"; done
+  local -a pids=()
+  for host in "${NODE_HOSTS[@]}"; do
+    build_one_node "$host" &
+    pids+=("$!")
+  done
+  for pid in "${pids[@]}"; do wait "$pid" || rc=1; done
+  (( rc == 0 )) || { log "ERROR: image build failed"; return 1; }
+  log "exact Draft image built on all nodes"
+}
+
+prewarm_one_node() {
+  local host="$1"
+  sudo_host "$host" mkdir -p "${FLASHINFER_CACHE}"
+  sudo_host "$host" docker run --rm --gpus device=0 \
+    -v "${FLASHINFER_CACHE}:/root/.cache/flashinfer" \
+    -e FLASHINFER_DISABLE_VERSION_CHECK=1 \
+    --entrypoint python3 "${PR_IMAGE}" -m pytest -q -s \
+    /opt/pr-tests/test_mxfp4_sm120_cutlass.py \
+    -k kimi_k3_sm120_situ_layout_and_noncontiguous_input \
+    >"${RESULT_ROOT}/build/${host}.prewarm.log" 2>&1
+}
+
+prewarm_all_nodes() {
+  local host pid rc=0
+  local -a pids=()
+  for host in "${NODE_HOSTS[@]}"; do
+    prewarm_one_node "$host" &
+    pids+=("$!")
+  done
+  for pid in "${pids[@]}"; do wait "$pid" || rc=1; done
+  (( rc == 0 )) || { log "ERROR: FlashInfer prewarm failed"; return 1; }
+  log "exact #4460 kernel prewarmed on all nodes"
+}
+
+container_name() { printf '%s_node%s' "${CONTAINER_PREFIX}" "$1"; }
+
+stop_service() {
+  local rank host
+  for rank in 0 1 2 3; do
+    host="${NODE_HOSTS[$rank]}"
+    sudo_host "$host" docker rm -f "$(container_name "$rank")" \
+      >/dev/null 2>&1 || true
+  done
+}
+
+collect_service_logs() {
+  local label="$1" rank host
+  for rank in 0 1 2 3; do
+    host="${NODE_HOSTS[$rank]}"
+    sudo_host "$host" docker logs "$(container_name "$rank")" \
+      >"${RESULT_ROOT}/service/${label}_node${rank}.log" 2>&1 || true
+  done
+}
+
+collect_gpu() {
+  local label="$1" rank host
+  for rank in 0 1 2 3; do
+    host="${NODE_HOSTS[$rank]}"
+    sudo_host "$host" nvidia-smi \
+      --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu,power.draw \
+      --format=csv,noheader,nounits \
+      >"${RESULT_ROOT}/gpu/${label}_node${rank}.csv" 2>&1 || true
+  done
+}
+
+verify_service_logs() {
+  local label="$1"
+  local pattern='CUDA out of memory|torch\.OutOfMemoryError|Traceback|EngineDeadError|NCCL[^[:cntrl:]]*(error|failed)|connection refused|Terminated'
+  if grep -Ein "${pattern}" "${RESULT_ROOT}/service/${label}_node"*.log \
+      >"${RESULT_ROOT}/service/${label}_fatal_scan.txt"; then
+    log "ERROR: fatal pattern found in service logs label=${label}"
+    return 1
+  fi
+  : >"${RESULT_ROOT}/service/${label}_fatal_scan.txt"
+}
+
+start_node() {
+  local rank="$1" backend="$2"
+  local host="${NODE_HOSTS[$rank]}" name bootstrap
+  name="$(container_name "$rank")"
+  bootstrap="export SGLANG_HOST_IP=174.1.60.$((rank + 1)); exec python3 -m sglang.launch_server --model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size 32 --ep-size 4 --nnodes 4 --node-rank ${rank} --dist-init-addr ${HEAD_HOST}:${DIST_PORT} --trust-remote-code --moe-runner-backend ${backend} --chunked-prefill-size ${CHUNKED_PREFILL_SIZE} --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --host 0.0.0.0 --port ${PORT}"
+  local -a cmd=(
+    docker run -d --name "$name"
+    --gpus all --network host --ipc=host --ulimit memlock=-1
+    --device /dev/infiniband --shm-size 32g --entrypoint bash
+    -v "${MODEL_PATH}:${MODEL_PATH}:ro"
+    -v "${FLASHINFER_CACHE}:/root/.cache/flashinfer"
+    -e CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+    -e NCCL_SOCKET_IFNAME=bond0 -e GLOO_SOCKET_IFNAME=bond0
+    -e NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3
+    -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7
+    -e NCCL_CUMEM_ENABLE=1 -e PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
+    -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1
+    -e FLASHINFER_DISABLE_VERSION_CHECK=1
+    "${PR_IMAGE}" -lc "$bootstrap"
+  )
+  printf '%q ' "${cmd[@]}" \
+    >"${RESULT_ROOT}/service/${backend}_node${rank}.cmd.txt"
+  printf '\n' >>"${RESULT_ROOT}/service/${backend}_node${rank}.cmd.txt"
+  sudo_host "$host" "${cmd[@]}" >/dev/null
+}
+
+wait_health() {
+  local backend="$1" i
+  for ((i = 1; i <= HEALTH_WAIT_S; i++)); do
+    if curl --fail --silent --max-time 5 \
+        "http://${HEAD_HOST}:${PORT}/health" >/dev/null 2>&1; then
+      log "service healthy backend=${backend} wait_s=${i}"
+      return 0
+    fi
+    if (( i % 30 == 0 )); then
+      log "waiting backend=${backend} elapsed_s=${i}"
+      collect_service_logs "${backend}_starting"
+      if grep -Eiq 'Traceback|CUDA out of memory|NCCL.*(error|failed)|EngineDeadError' \
+          "${RESULT_ROOT}/service/${backend}_starting_node"*.log; then
+        return 1
+      fi
+    fi
+    sleep 1
+  done
+  return 1
+}
+
+start_service() {
+  local backend="$1"
+  stop_service
+  log "starting TP32/EP4 backend=${backend} chunk=8192"
+  start_node 1 "$backend"
+  start_node 2 "$backend"
+  start_node 3 "$backend"
+  sleep 5
+  start_node 0 "$backend"
+  wait_health "$backend" || {
+    collect_service_logs "${backend}_startup_failed"
+    return 1
+  }
+  collect_service_logs "${backend}_healthy"
+  collect_gpu "${backend}_healthy"
+}
+
+run_bench() {
+  local backend="$1" repeat="$2"
+  local stem="${backend}_chunk8192_c8_r${repeat}"
+  local output="${RESULT_ROOT}/raw/${stem}.jsonl"
+  rm -f "$output"
+  log "bench backend=${backend} repeat=${repeat}/${REPEATS}"
+  sudo_host "${HEAD_HOST}" docker run --rm --network host \
+    -v "${MODEL_PATH}:${MODEL_PATH}:ro" \
+    -v "${RESULT_ROOT}:${RESULT_ROOT}" \
+    -e PYTHONUNBUFFERED=1 --entrypoint python3 "${PR_IMAGE}" \
+    -m sglang.benchmark.serving \
+    --backend sglang --host "${HEAD_HOST}" --port "${PORT}" \
+    --tokenizer "${MODEL_PATH}" --dataset-name random-ids \
+    --random-input-len "${INPUT_LEN}" --random-output-len "${OUTPUT_LEN}" \
+    --random-range-ratio 1.0 --num-prompts "${NUM_PROMPTS}" \
+    --max-concurrency "${CONCURRENCY}" --request-rate 10000 \
+    --warmup-requests "${WARMUP_REQUESTS}" --output-file "$output" \
+    --output-details --disable-tqdm \
+    >"${RESULT_ROOT}/bench/${stem}.log" 2>&1
+  python3 - "$output" "${NUM_PROMPTS}" <<'PY'
+import json, sys
+path, expected = sys.argv[1], int(sys.argv[2])
+rows = [json.loads(x) for x in open(path, encoding="utf-8") if x.strip()]
+assert len(rows) == 1, (path, len(rows))
+assert rows[0].get("completed") == expected, rows[0].get("completed")
+assert not rows[0].get("errors"), rows[0].get("errors")
+PY
+}
+
+run_ab() {
+  local backend repeat
+  collect_gpu before
+  for backend in "${BACKENDS[@]}"; do
+    start_service "$backend"
+    for ((repeat = 1; repeat <= REPEATS; repeat++)); do
+      run_bench "$backend" "$repeat"
+    done
+    collect_service_logs "${backend}_completed"
+    verify_service_logs "${backend}_completed"
+    collect_gpu "${backend}_completed"
+    stop_service
+    sleep 5
+  done
+  collect_gpu after
+}
+
+summarize() {
+  python3 - "${RESULT_ROOT}" <<'PY'
+import csv, json, re, statistics, sys
+from pathlib import Path
+
+root = Path(sys.argv[1])
+pat = re.compile(r"(.+)_chunk8192_c8_r(\d+)\.jsonl$")
+rows = []
+for path in sorted((root / "raw").glob("*.jsonl")):
+    m = pat.match(path.name)
+    if not m:
+        continue
+    data = next(json.loads(x) for x in path.read_text().splitlines() if x.strip())
+    rows.append({"backend": m.group(1), "repeat": int(m.group(2)), **data})
+
+metrics = ["request_throughput", "input_throughput", "total_throughput",
+           "median_ttft_ms", "p95_ttft_ms", "median_e2e_latency_ms"]
+summary = []
+for backend in ("marlin", "flashinfer_mxfp4"):
+    group = [x for x in rows if x["backend"] == backend]
+    if len(group) != 3:
+        raise SystemExit(f"expected 3 repeats for {backend}, got {len(group)}")
+    item = {"backend": backend, "repeats": len(group),
+            "completed_each": [x.get("completed") for x in group]}
+    for metric in metrics:
+        item[f"median_{metric}"] = statistics.median(float(x[metric]) for x in group)
+    summary.append(item)
+idx = {x["backend"]: x for x in summary}
+base, cand = idx["marlin"], idx["flashinfer_mxfp4"]
+comparison = {
+    "input_throughput_change_pct":
+        (cand["median_input_throughput"] / base["median_input_throughput"] - 1) * 100,
+    "median_ttft_change_pct":
+        (cand["median_median_ttft_ms"] / base["median_median_ttft_ms"] - 1) * 100,
+    "p95_ttft_change_pct":
+        (cand["median_p95_ttft_ms"] / base["median_p95_ttft_ms"] - 1) * 100,
+}
+payload = {
+    "run_id": root.name,
+    "shape": {"input_len": 16384, "output_len": 1,
+              "concurrency": 8, "chunked_prefill_size": 8192,
+              "tp": 32, "ep": 4},
+    "summary": summary,
+    "comparison": comparison,
+}
+(root / "summary.json").write_text(json.dumps(payload, indent=2))
+with (root / "results.csv").open("w", newline="") as f:
+    fields = ["backend", "repeat", "completed", *metrics]
+    w = csv.DictWriter(f, fieldnames=fields, extrasaction="ignore")
+    w.writeheader(); w.writerows(rows)
+print(json.dumps(payload, indent=2))
+PY
+}
+
+cleanup() {
+  collect_service_logs cleanup 2>/dev/null || true
+  stop_service 2>/dev/null || true
+}
+trap cleanup EXIT INT TERM
+
+case "${ACTION}" in
+  build)
+    require_password; check_inputs; build_all_nodes; prewarm_all_nodes
+    ;;
+  run)
+    require_password; check_inputs; run_ab; summarize
+    ;;
+  all)
+    require_password; check_inputs; build_all_nodes; prewarm_all_nodes
+    run_ab; summarize
+    ;;
+  summarize)
+    summarize
+    ;;
+  stop)
+    require_password; stop_service
+    ;;
+  *)
+    echo "Usage: $0 {all|build|run|summarize|stop}" >&2
+    exit 2
+    ;;
+esac