From 13944079fa5eca143c7a8d061cd9db582dba161b Mon Sep 17 00:00:00 2001 From: Zhiyi Hong <2497491955@qq.com> Date: Tue, 18 Aug 2026 13:57:31 +0800 Subject: [PATCH] [Test] Add EP4 maximum-pressure capacity probe --- README.md | 2 ++ .../README.md | 9 +++++++-- .../config.env | 8 +++++--- .../run_moe_backend_prefill.sh | 4 ++-- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index c7b1b56..6f11b1c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # sskj — 多平台大模型推理性能基准测试项目 +**更新（2026-08-18 13:55:30 CST）**：Kimi-K3 四节点 MoE backend 验收入口支持通过环境变量覆盖 EP size 及 backend/chunk/concurrency 子矩阵，用于先执行 TP32×EP4、16K Chunk、16K→1、C=16 的最大压力容量验证，同时保留默认 TP32×EP32 全矩阵。 + **更新（2026-08-18 13:03:33 CST）**：Phase 5 候选镜像改为针对 Kimi 基础镜像原生 SGLang 源码应用最小兼容补丁，仅补齐 SM120 FlashInfer MXFP4 backend 分发、权重布局与 SiTU 激活映射，避免整体替换新版 Python 包造成 `sglang-kernel` 版本错配。 **更新（2026-08-18 12:24:19 CST）**：新增 Kimi-K3 / 601-604 / SGLang TP32×EP32 的真实 Prefill MoE backend 验收实验；固定比较 Marlin 与 FlashInfer MXFP4 在 16K→1、C=8/16、Chunk=8K/16K 下的 TTFT、E2E 与 Input TPS，并提供四节点一致候选镜像构建、原始证据和自动汇总，详见 `experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/README.md`。 diff --git a/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/README.md b/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/README.md index 5b1694b..151eda1 100644 --- a/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/README.md +++ b/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/README.md @@ -22,6 +22,11 @@ length without depending on a ShareGPT file or text truncation. Radix cache is disabled. `OSL=1` makes TTFT, E2E latency and input TPS the primary metrics; TPOT is not meaningful for this matrix. +The defaults above can be narrowed for a capacity-first run with `EP_SIZE`, +`BACKENDS_CSV`, `CHUNKED_PREFILL_SIZES_CSV`, and `CONCURRENCIES_CSV`. For +example, the TP32/EP4 maximum-pressure probe uses Marlin, 16K chunk, and C=16 +before spending time on the complete backend comparison. + ## Unique entrypoint Run on 601 only. The sudo password is supplied at runtime and is never written @@ -72,6 +77,6 @@ The run aborts on a failed service start, any failed request, a missing result, or an image capability smoke failure. Containers are removed on exit. The candidate image preserves the Kimi base image's dependency set. It installs -the patched FlashInfer wheel/AOT libraries and applies only the two production -source files from SGLang commit `fb929bb`; it does not replace the complete +the patched FlashInfer wheel/AOT libraries and applies a minimal diff against +the image's exact two SGLang source files; it does not replace the complete SGLang Python tree or require a newer `sglang-kernel` package. diff --git a/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/config.env b/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/config.env index 2086892..a70c281 100644 --- a/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/config.env +++ b/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/config.env @@ -10,6 +10,7 @@ HEAD_HOST="${HEAD_HOST:-174.1.60.1}" NODE_SSH_USER="${NODE_SSH_USER:-user}" NODE_HOSTS=(174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4) DIST_PORT="${DIST_PORT:-20000}" +EP_SIZE="${EP_SIZE:-32}" BASE_IMAGE="${BASE_IMAGE:-lmsysorg/sglang:kimi-k3-fiv617situ-warm}" CANDIDATE_IMAGE="${CANDIDATE_IMAGE:-local/sglang:kimi-k3-sm120-flashinfer-mxfp4-phase5}" @@ -27,6 +28,7 @@ REPEATS="${REPEATS:-3}" WARMUP_REQUESTS="${WARMUP_REQUESTS:-2}" HEALTH_WAIT_S="${HEALTH_WAIT_S:-2400}" -BACKENDS=(marlin flashinfer_mxfp4) -CHUNKED_PREFILL_SIZES=(8192 16384) -CONCURRENCIES=(8 16) +IFS=',' read -r -a BACKENDS <<<"${BACKENDS_CSV:-marlin,flashinfer_mxfp4}" +IFS=',' read -r -a CHUNKED_PREFILL_SIZES \ + <<<"${CHUNKED_PREFILL_SIZES_CSV:-8192,16384}" +IFS=',' read -r -a CONCURRENCIES <<<"${CONCURRENCIES_CSV:-8,16}" diff --git a/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/run_moe_backend_prefill.sh b/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/run_moe_backend_prefill.sh index f3ee816..6a22020 100755 --- a/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/run_moe_backend_prefill.sh +++ b/experiments/pro6000/kimi3_pro6000_sglang_tp32ep32_moe_backend_prefill/run_moe_backend_prefill.sh @@ -192,7 +192,7 @@ start_node() { local rank="$1" backend="$2" chunk="$3" local host="${NODE_HOSTS[$rank]}" name bootstrap name="$(container_name "$rank")" - bootstrap="export SGLANG_HOST_IP=174.1.60.$((rank + 1)); exec python3 -m sglang.launch_server --model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size 32 --ep-size 32 --nnodes 4 --node-rank ${rank} --dist-init-addr ${HEAD_HOST}:${DIST_PORT} --trust-remote-code --moe-runner-backend ${backend} --chunked-prefill-size ${chunk} --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --host 0.0.0.0 --port ${PORT}" + bootstrap="export SGLANG_HOST_IP=174.1.60.$((rank + 1)); exec python3 -m sglang.launch_server --model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size 32 --ep-size ${EP_SIZE} --nnodes 4 --node-rank ${rank} --dist-init-addr ${HEAD_HOST}:${DIST_PORT} --trust-remote-code --moe-runner-backend ${backend} --chunked-prefill-size ${chunk} --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --host 0.0.0.0 --port ${PORT}" local -a cmd=( docker run -d --name "$name" --gpus all --network host --ipc=host --ulimit memlock=-1 @@ -236,7 +236,7 @@ wait_health() { start_service() { local backend="$1" chunk="$2" stop_service - log "starting TP32/EP32 backend=${backend} chunk=${chunk} on 601-604" + log "starting TP32/EP${EP_SIZE} backend=${backend} chunk=${chunk} on 601-604" start_node 1 "$backend" "$chunk" start_node 2 "$backend" "$chunk" start_node 3 "$backend" "$chunk"