feat(pro6000): Kimi-K3 PD 分离部署（MoonCake RDMA）- 8 节点 P/D 双 profile + deploy_pd.sh 编排 + 文档
- 新增 P 组(prefill)/D 组(decode) deploy profiles（mooncake RDMA + 计算网 mlx5_0~3） - 新增 experiments/pro6000/kimi3_pro6000_pd_rdma/ 编排脚本（mc-master+P+D+router） - 关键修复: 必须关闭 PYTORCH_CUDA_ALLOC_CONF=expandable_segments （mooncake RDMA 注册 expandable GPU 段报 Bad address，GitHub #2511） - 容器内升级 mooncake 0.3.12.post1（含 dmabuf 修复 #2035） - docs/KIMI_K3_DEPLOY.md 新增附录 B，README 更新实验索引
This commit is contained in:
shishi
parent
3bd04698bb
commit
d72dbff689
@ -45,6 +45,7 @@
|
|||||||
| `experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/` | RTX 6000D + vLLM |
|
| `experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/` | RTX 6000D + vLLM |
|
||||||
| `experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/` | RTX 6000D + SGLang |
|
| `experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/` | RTX 6000D + SGLang |
|
||||||
| `experiments/pro6000/kimi3_pro6000_sglang_tp32ep32/` | RTX 6000D + SGLang，Kimi-K3（TP32×EP32，部署手册见 docs/KIMI_K3_DEPLOY.md） |
|
| `experiments/pro6000/kimi3_pro6000_sglang_tp32ep32/` | RTX 6000D + SGLang，Kimi-K3（TP32×EP32，部署手册见 docs/KIMI_K3_DEPLOY.md） |
|
||||||
|
| `experiments/pro6000/kimi3_pro6000_pd_rdma/` | RTX 6000D + SGLang，Kimi-K3 PD 分离（MoonCake RDMA，8 节点，见 deploy_pd.sh + docs/KIMI_K3_DEPLOY.md 附录 B） |
|
||||||
|
|
||||||
每个目录内：`run_bench.sh` 跑固定并发矩阵；`run_adaptive_concurrency.sh` 从 C=1 指数倍增搜饱和点；`run_adaptive_concurrency_add16.sh` 从 C=16 线性 +16 步进、带 TTFT SLO 停止与回退（当前主力用法，见 `experiments/ADAPTIVE_CONCURRENCY_USAGE.md`）。
|
每个目录内：`run_bench.sh` 跑固定并发矩阵；`run_adaptive_concurrency.sh` 从 C=1 指数倍增搜饱和点；`run_adaptive_concurrency_add16.sh` 从 C=16 线性 +16 步进、带 TTFT SLO 停止与回退（当前主力用法，见 `experiments/ADAPTIVE_CONCURRENCY_USAGE.md`）。
|
||||||
|
|
||||||
|
|||||||
54
deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env
Normal file
54
deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env
Normal file
@ -0,0 +1,54 @@
|
|||||||
|
# Kimi-K3 PD 分离 - D 组 (decode) deployment profile (4x RTX 6000D).
|
||||||
|
# Nodes: 174.1.60.5~8 (rank 0~3), 32x NVIDIA RTX 6000D (85GB, sm_120).
|
||||||
|
#
|
||||||
|
# 这是 PD (Prefill/Decode) 分离部署的 D 组 (decode) 端 profile。
|
||||||
|
# 配套 P 组 profile: kimi3_pro6000_pd_prefill.env
|
||||||
|
# 配套编排脚本: experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh
|
||||||
|
#
|
||||||
|
# 与 P 组相同的传输/网络配置（mooncake RDMA + 计算网），区别仅在:
|
||||||
|
# - 节点是 174.1.60.5~8
|
||||||
|
# - --disaggregation-mode decode --port 30000
|
||||||
|
# - dist-init-addr 174.1.60.5:20000
|
||||||
|
# - 必须 D 组在 P 组之后启动（见 deploy_pd.sh）
|
||||||
|
#
|
||||||
|
# Model-team only. Ops only run benchmark against the served router URL.
|
||||||
|
|
||||||
|
PLATFORM=pro6000
|
||||||
|
EXPERIMENT=kimi3_pro6000_pd_decode
|
||||||
|
MODEL_NAME=Kimi-K3
|
||||||
|
ENGINE=sglang
|
||||||
|
RUNTIME=docker
|
||||||
|
DOCKER_IMAGE=lmsysorg/sglang:kimi-k3
|
||||||
|
CONTAINER_NAME=${EXPERIMENT}_node${NODE_RANK}
|
||||||
|
MODEL_PATH=/data/hf_models/Kimi-K3
|
||||||
|
SERVED_MODEL_NAME=kimi-k3
|
||||||
|
PORT=30000
|
||||||
|
HEALTH_PATH=/health
|
||||||
|
HEALTH_HOST=174.1.60.5
|
||||||
|
HEALTH_WAIT_S=2400
|
||||||
|
CONTAINER_PYTHON=python3
|
||||||
|
|
||||||
|
# ---- 并行度（固定，勿改）----
|
||||||
|
TP=32
|
||||||
|
DP=1
|
||||||
|
EP_SIZE=32
|
||||||
|
|
||||||
|
# ---- Multi-node topology (D 组 = decode 4 节点) ----
|
||||||
|
NNODES=4
|
||||||
|
NODE_HOSTS="174.1.60.5 174.1.60.6 174.1.60.7 174.1.60.8"
|
||||||
|
NODE_SSH_USER=root
|
||||||
|
LOCAL_NODE_RANK=0
|
||||||
|
MASTER_IP=174.1.60.5
|
||||||
|
DIST_PORT=20000
|
||||||
|
|
||||||
|
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
||||||
|
ENGINE_ENV="MOONCAKE_MASTER=174.1.60.1:50051 MOONCAKE_PROTOCOL=rdma NCCL_SOCKET_IFNAME=bond1 GLOO_SOCKET_IFNAME=bond1 NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3 NCCL_IB_GID_INDEX=3 NCCL_IB_TIMEOUT=22 NCCL_IB_RETRY_CNT=7 NCCL_CUMEM_ENABLE=1 SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 SGLANG_MOE_FUSED_GATE_RADIX=1 SGLANG_ENABLE_REQUEST_HEADER_OVERRIDES=1"
|
||||||
|
|
||||||
|
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''"
|
||||||
|
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
|
||||||
|
PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /data/flashkda_deploy/wheels:/mc_wheels:ro"
|
||||||
|
|
||||||
|
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 decode。
|
||||||
|
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((5 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"
|
||||||
|
|
||||||
|
LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode decode --host 0.0.0.0 --port ${PORT}"
|
||||||
60
deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env
Normal file
60
deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env
Normal file
@ -0,0 +1,60 @@
|
|||||||
|
# Kimi-K3 PD 分离 - P 组 (prefill) deployment profile (4x RTX 6000D).
|
||||||
|
# Nodes: 174.1.60.1~4 (rank 0~3), 32x NVIDIA RTX 6000D (85GB, sm_120).
|
||||||
|
#
|
||||||
|
# 这是 PD (Prefill/Decode) 分离部署的 P 组 (prefill) 端 profile。
|
||||||
|
# 配套 D 组 profile: kimi3_pro6000_pd_decode.env
|
||||||
|
# 配套编排脚本: experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh
|
||||||
|
#
|
||||||
|
# 关键点（实测踩坑，勿随意改）:
|
||||||
|
# - 传输后端 mooncake + MOONCAKE_PROTOCOL=rdma（计算网 mlx5_0~3，4 链路）
|
||||||
|
# - 必须关掉 PYTORCH_CUDA_ALLOC_CONF=expandable_segments
|
||||||
|
# （实测: expandable_segments 分配的 GPU 段 mooncake RDMA 注册失败
|
||||||
|
# Bad address [14]，见飞书文档 PD 章节 & GitHub kvcache-ai/Mooncake#2511）
|
||||||
|
# - --disaggregation-ib-device mlx5_0~3 让 mooncake 走计算网 RDMA
|
||||||
|
# - NCCL/GLOO 走 bond1（计算网），NCCL_IB_HCA=mlx5_0..3
|
||||||
|
# - P 组先启动，D 组后启动（见 deploy_pd.sh）
|
||||||
|
# - mooncake master 需先在 174.1.60.1 运行（MOONCAKE_MASTER=174.1.60.1:50051）
|
||||||
|
# - 容器挂载 /mc_wheels 并在 BOOTSTRAP 里 pip install mooncake 0.3.12.post1
|
||||||
|
# （镜像自带 0.3.11.post1 无 dmabuf 修复，必须升级）
|
||||||
|
#
|
||||||
|
# Model-team only. Ops only run benchmark against the served router URL.
|
||||||
|
|
||||||
|
PLATFORM=pro6000
|
||||||
|
EXPERIMENT=kimi3_pro6000_pd_prefill
|
||||||
|
MODEL_NAME=Kimi-K3
|
||||||
|
ENGINE=sglang
|
||||||
|
RUNTIME=docker
|
||||||
|
DOCKER_IMAGE=lmsysorg/sglang:kimi-k3
|
||||||
|
CONTAINER_NAME=${EXPERIMENT}_node${NODE_RANK}
|
||||||
|
MODEL_PATH=/data/hf_models/Kimi-K3
|
||||||
|
SERVED_MODEL_NAME=kimi-k3
|
||||||
|
PORT=30000
|
||||||
|
HEALTH_PATH=/health
|
||||||
|
HEALTH_HOST=174.1.60.1
|
||||||
|
HEALTH_WAIT_S=2400
|
||||||
|
CONTAINER_PYTHON=python3
|
||||||
|
|
||||||
|
# ---- 并行度（固定，勿改）----
|
||||||
|
TP=32
|
||||||
|
DP=1
|
||||||
|
EP_SIZE=32
|
||||||
|
|
||||||
|
# ---- Multi-node topology (P 组 = prefill 4 节点) ----
|
||||||
|
NNODES=4
|
||||||
|
NODE_HOSTS="174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4"
|
||||||
|
NODE_SSH_USER=root
|
||||||
|
LOCAL_NODE_RANK=0
|
||||||
|
MASTER_IP=174.1.60.1
|
||||||
|
DIST_PORT=20000
|
||||||
|
|
||||||
|
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
||||||
|
ENGINE_ENV="MOONCAKE_MASTER=174.1.60.1:50051 MOONCAKE_PROTOCOL=rdma NCCL_SOCKET_IFNAME=bond1 GLOO_SOCKET_IFNAME=bond1 NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3 NCCL_IB_GID_INDEX=3 NCCL_IB_TIMEOUT=22 NCCL_IB_RETRY_CNT=7 NCCL_CUMEM_ENABLE=1 SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 SGLANG_MOE_FUSED_GATE_RADIX=1"
|
||||||
|
|
||||||
|
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''"
|
||||||
|
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
|
||||||
|
PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /data/flashkda_deploy/wheels:/mc_wheels:ro"
|
||||||
|
|
||||||
|
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 prefill。
|
||||||
|
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((1 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"
|
||||||
|
|
||||||
|
LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --disaggregation-transfer-backend mooncake --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0,mlx5_1,mlx5_2,mlx5_3 --disaggregation-mode prefill --linear-attn-prefill-backend flashkda --host 0.0.0.0 --port ${PORT}"
|
||||||
@ -192,3 +192,68 @@ FlashKDA（MoonshotAI CUTLASS KDA 内核，支持 sm_120）可作为 KDA prefill
|
|||||||
`--linear-attn-prefill-backend flashkda`
|
`--linear-attn-prefill-backend flashkda`
|
||||||
|
|
||||||
已知问题：flashkda 后端在 4/8 并发压测下服务异常（纯 triton 无此问题），修复前不建议生产使用。
|
已知问题：flashkda 后端在 4/8 并发压测下服务异常（纯 triton 无此问题），修复前不建议生产使用。
|
||||||
|
|
||||||
|
## 附录 B：PD 分离部署（MoonCake RDMA，8 节点）
|
||||||
|
|
||||||
|
非 PD 单组部署之外的另一种形态：**Prefill/Decode 分离**（P 组 4 节点做 prefill，D 组 4 节点做 decode），KV 传输走 **MoonCake RDMA**。适合长上下文、吞吐优先的场景。
|
||||||
|
|
||||||
|
### B.1 拓扑
|
||||||
|
|
||||||
|
```
|
||||||
|
┌───────────────────┐
|
||||||
|
client ──31000──▶ │ router (MiniLB) │ 174.1.60.5
|
||||||
|
└───────┬───────────┘
|
||||||
|
┌───────▼───────────┐
|
||||||
|
P 组 (prefill) │ 174.1.60.1~4 │ TP32×EP32，4 节点 32 卡
|
||||||
|
└── prefill 计算 │ 计算完产出 KV │
|
||||||
|
└───────┬───────────┘
|
||||||
|
┌───────▼───────────┐
|
||||||
|
MoonCake RDMA│ mlx5_0~3 (RoCE) │ 4 链路并行
|
||||||
|
└───────┬───────────┘
|
||||||
|
┌───────▼───────────┐
|
||||||
|
D 组 (decode) │ 174.1.60.5~8 │ TP32×EP32，4 节点 32 卡
|
||||||
|
└── decode 生成 │ 接收 KV 逐 token │
|
||||||
|
└───────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### B.2 一键部署
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /data/yy/sskj/experiments/pro6000/kimi3_pro6000_pd_rdma
|
||||||
|
bash deploy_pd.sh start # mc-master → P 组 → D 组 → router
|
||||||
|
bash deploy_pd.sh status
|
||||||
|
bash deploy_pd.sh stop
|
||||||
|
bash deploy_pd.sh restart
|
||||||
|
```
|
||||||
|
|
||||||
|
### B.3 验证
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 端到端（router 自动双发 P/D）
|
||||||
|
curl -s http://174.1.60.5:31000/generate -H "Content-Type: application/json" \
|
||||||
|
-d '{"text":"Hello","sampling_params":{"max_new_tokens":16}}'
|
||||||
|
```
|
||||||
|
|
||||||
|
长输入验证（2026-08-11 实测）：1680 / 5280 tokens 输入均正常，P 组 prefill ~112 tok/s，D 组 decode ~26 tok/s。
|
||||||
|
|
||||||
|
### B.4 关键注意事项（务必遵守）
|
||||||
|
|
||||||
|
1. **禁止设置 `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`**
|
||||||
|
expandable_segments 分配的 GPU 段，mooncake RDMA 注册报 `Bad address [14]`
|
||||||
|
（GitHub kvcache-ai/Mooncake **#2511**）。移除后 KV 注册从 2376 条失败 → 0。
|
||||||
|
2. **P 组必须先于 D 组启动**（prefill 需先注册 bootstrap）。
|
||||||
|
3. **容器内必须升级 mooncake 到 0.3.12.post1**（镜像自带 0.3.11.post1 无 dmabuf 修复 #2035）。
|
||||||
|
wheel 在 `/data/flashkda_deploy/wheels/`，profile 的 BOOTSTRAP 已含 pip install。
|
||||||
|
4. NCCL/GLOO 走计算网（`bond1` + `NCCL_IB_HCA=mlx5_0..3`），不要走管理网 bond0。
|
||||||
|
5. mc-master（mooncake 元数据服务）需先在 174.1.60.1 运行（deploy_pd.sh 自动处理）。
|
||||||
|
|
||||||
|
### B.5 与单组部署的取舍
|
||||||
|
|
||||||
|
| 维度 | 单组 TP32×EP32 | PD 分离（MoonCake RDMA） |
|
||||||
|
|---|---|---|
|
||||||
|
| 节点 | 174.1.60.5~8（4 节点） | P 174.1.60.1~4 + D 174.1.60.5~8（8 节点） |
|
||||||
|
| KV 传输 | 无（本地） | MoonCake RDMA（计算网 4 链路） |
|
||||||
|
| 传输后端 | - | mooncake（必须 0.3.12.post1） |
|
||||||
|
| 适用 | 单组吞吐、简单 | 长上下文、PD 分离 |
|
||||||
|
| profile | kimi3_pro6000_sglang_tp32ep32 | kimi3_pro6000_pd_prefill / _pd_decode |
|
||||||
|
| 编排 | `python -m sskj.deploy` | `deploy_pd.sh` |
|
||||||
90
experiments/pro6000/kimi3_pro6000_pd_rdma/README.md
Normal file
90
experiments/pro6000/kimi3_pro6000_pd_rdma/README.md
Normal file
@ -0,0 +1,90 @@
|
|||||||
|
# Kimi-K3 PD 分离部署（MoonCake RDMA）实验
|
||||||
|
|
||||||
|
RTX 6000D（sm_120，8 节点 64 卡）上 Kimi-K3 的 **Prefill/Decode 分离部署**，KV 传输使用 **MoonCake RDMA**（计算网 mlx5_0~3，4 链路 RoCE）。
|
||||||
|
|
||||||
|
## 背景与动机
|
||||||
|
|
||||||
|
PD 分离部署（PD Disaggregation）将 prefill（预填充，计算密集型）与 decode（逐 token 生成，访存密集型）拆分到不同节点组，是长上下文下提升吞吐/利用率的行业标准做法。本实验聚焦 KV 传输后端的选型验证与稳定部署。
|
||||||
|
|
||||||
|
### 为什么不用 NIXL
|
||||||
|
|
||||||
|
实测 NIXL（UCX 后端）在 RTX 6000D（无 GDR/GDAKI）上 **VRAM 单次传输上限约 20MB**（24/32/64MB 均报 `Input/output error` → `NIXL_ERR_REMOTE_DISCONNECT`），且 sglang NIXL 后端不做按字节分块（一次传输整个请求全部 KV），连续负载下必断。根因：UCX 依赖 GDAKI（GPU Direct Async）接口，当前未启用（缺 `NVreg_RegistryDwords="PeerMappingOverride=1;"`）。详见飞书 wiki「Kimi-K3 部署手册」PD 章节。
|
||||||
|
|
||||||
|
### 为什么 MoonCake RDMA 可行
|
||||||
|
|
||||||
|
- MoonCake 走 **libibverbs 直连**（传统 GDR + dmabuf 注册），不依赖 UCX/GDAKI
|
||||||
|
- 实测跨节点 GPU 显存传输 **1GB 无压力**，连续 10×64MB 全部成功
|
||||||
|
- **必须用 0.3.12.post1**（含 dmabuf 注册修复 #2035；镜像自带的 0.3.11.post1 无修复）
|
||||||
|
|
||||||
|
### 关键坑（务必遵守）
|
||||||
|
|
||||||
|
1. **不能设 `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`**
|
||||||
|
expandable_segments 分配的 GPU 段，mooncake RDMA 注册报 `Bad address [14]`（GitHub kvcache-ai/Mooncake **#2511**）。移除后 KV 注册 2376 条失败 → 0 失败。
|
||||||
|
2. **P 组先启动，D 组后启动**（prefill 需先注册 bootstrap）。
|
||||||
|
3. **容器内必须升级 mooncake 到 0.3.12.post1**（wheel 在 `/data/flashkda_deploy/wheels/`，profile 的 BOOTSTRAP 里已含 pip install）。
|
||||||
|
4. NCCL/GLOO 走计算网（`bond1` + `NCCL_IB_HCA=mlx5_0..3`），不要走管理网 bond0。
|
||||||
|
|
||||||
|
## 架构
|
||||||
|
|
||||||
|
```
|
||||||
|
┌───────────────────┐
|
||||||
|
client ──31000──▶ │ router (MiniLB) │ 174.1.60.5
|
||||||
|
└───────┬───────────┘
|
||||||
|
┌───────▼───────────┐
|
||||||
|
P 组 (prefill) │ 174.1.60.1~4 │ TP32×EP32，4 节点 32 卡
|
||||||
|
└── prefill 计算 │ 计算完产出 KV │
|
||||||
|
└───────┬───────────┘
|
||||||
|
┌───────▼───────────┐
|
||||||
|
MoonCake RDMA│ mlx5_0~3 (RoCE) │ 4 链路并行
|
||||||
|
└───────┬───────────┘
|
||||||
|
┌───────▼───────────┐
|
||||||
|
D 组 (decode) │ 174.1.60.5~8 │ TP32×EP32，4 节点 32 卡
|
||||||
|
└── decode 生成 │ 接收 KV 逐 token │
|
||||||
|
└───────────────────┘
|
||||||
|
|
||||||
|
mc-master (174.1.60.1:50051) MoonCake 元数据服务
|
||||||
|
```
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# 一键启动（mc-master → P 组 → D 组 → router）
|
||||||
|
bash deploy_pd.sh start
|
||||||
|
|
||||||
|
# 查看状态
|
||||||
|
bash deploy_pd.sh status
|
||||||
|
|
||||||
|
# 停止 / 重启
|
||||||
|
bash deploy_pd.sh stop
|
||||||
|
bash deploy_pd.sh restart
|
||||||
|
|
||||||
|
# 端到端验证（router 入口，自动双发 P/D）
|
||||||
|
curl -s http://174.1.60.5:31000/generate -H "Content-Type: application/json" \
|
||||||
|
-d '{"text":"Hello","sampling_params":{"max_new_tokens":16}}'
|
||||||
|
```
|
||||||
|
|
||||||
|
## 实测验证（2026-08-11）
|
||||||
|
|
||||||
|
| 测试 | 输入 tokens | 输出 tokens | 结果 |
|
||||||
|
|---|---|---|---|
|
||||||
|
| warmup | 6 | 16 | ✅ 200 OK |
|
||||||
|
| 中等 | 1680 | 512 | ✅ e2e 31.4s |
|
||||||
|
| 长输入 | 5280 | 256 | ✅ e2e 13.0s |
|
||||||
|
|
||||||
|
- P 组 prefill throughput ~112 tok/s（5280 token 输入），D 组 decode ~26 tok/s
|
||||||
|
- 两端 GPU 同步工作，**PD 分离链路完整跑通**
|
||||||
|
- 对比 NIXL：MoonCake RDMA 处理长 KV 传输稳定，无 20MB 上限问题
|
||||||
|
|
||||||
|
## 文件
|
||||||
|
|
||||||
|
| 文件 | 说明 |
|
||||||
|
|---|---|
|
||||||
|
| `deploy_pd.sh` | PD 编排脚本（mc-master + P + D + router 全生命周期） |
|
||||||
|
| `config.env` | 实验配置 |
|
||||||
|
| `deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env` | P 组（prefill）部署 profile |
|
||||||
|
| `deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env` | D 组（decode）部署 profile |
|
||||||
|
|
||||||
|
## 参考
|
||||||
|
|
||||||
|
- 飞书 wiki「Kimi-K3 部署手册」PD 分离章节（七~十一章）
|
||||||
|
- GitHub kvcache-ai/Mooncake: #2511（expandable_segments 注册失败）、#2035（dmabuf 修复）、#351（Bad address）
|
||||||
31
experiments/pro6000/kimi3_pro6000_pd_rdma/config.env
Executable file
31
experiments/pro6000/kimi3_pro6000_pd_rdma/config.env
Executable file
@ -0,0 +1,31 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Kimi-K3 PD 分离部署（MoonCake RDMA）实验配置。
|
||||||
|
# 服务器生命周期走 deploy_pd.sh 编排脚本 + deploy profiles：
|
||||||
|
# - P 组 profile: pro6000/kimi3_pro6000_pd_prefill
|
||||||
|
# - D 组 profile: pro6000/kimi3_pro6000_pd_decode
|
||||||
|
# - 编排: experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh
|
||||||
|
# 部署参数以 profile / deploy_pd.sh 为准，勿在此重复。
|
||||||
|
|
||||||
|
EXPERIMENT="kimi3_pro6000_pd_rdma"
|
||||||
|
MODEL_NAME="Kimi-K3"
|
||||||
|
MODEL_PATH="/data/hf_models/Kimi-K3"
|
||||||
|
SERVED_MODEL_NAME="kimi-k3"
|
||||||
|
|
||||||
|
# Router (MiniLB) 入口端口。
|
||||||
|
SGLANG_PORT="${SGLANG_PORT:-31000}"
|
||||||
|
|
||||||
|
# MoonCake master 地址（P 组头节点）。
|
||||||
|
MOONCAKE_MASTER="${MOONCAKE_MASTER:-174.1.60.1:50051}"
|
||||||
|
|
||||||
|
# Python interpreter for orchestration scripts.
|
||||||
|
VENV_CLIENT="${VENV_CLIENT:-/root/miniconda3/envs/mas}"
|
||||||
|
|
||||||
|
# Run the benchmark client natively (0) or inside Docker (1).
|
||||||
|
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
||||||
|
|
||||||
|
# Deployment profiles used by deploy_pd.sh.
|
||||||
|
P_DEPLOY_PROFILE="${P_DEPLOY_PROFILE:-pro6000/kimi3_pro6000_pd_prefill}"
|
||||||
|
D_DEPLOY_PROFILE="${D_DEPLOY_PROFILE:-pro6000/kimi3_pro6000_pd_decode}"
|
||||||
|
|
||||||
|
# Per-scenario timeout to avoid hangs (seconds).
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-3600}"
|
||||||
114
experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh
Executable file
114
experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh
Executable file
@ -0,0 +1,114 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Kimi-K3 PD 分离部署编排脚本（MoonCake RDMA 传输）。
|
||||||
|
# 管理 mc-master（元数据服务）+ P 组（prefill）+ D 组（decode）+ router（MiniLB 入口）。
|
||||||
|
#
|
||||||
|
# 用法:
|
||||||
|
# bash deploy_pd.sh start # 按顺序启动: mc-master → P 组 → D 组 → router
|
||||||
|
# bash deploy_pd.sh stop # 逆序停止: router → D 组 → P 组 → mc-master
|
||||||
|
# bash deploy_pd.sh status # 查看各组件状态
|
||||||
|
# bash deploy_pd.sh restart # stop + start
|
||||||
|
#
|
||||||
|
# 依赖:
|
||||||
|
# - 8 台节点镜像 lmsysorg/sglang:kimi-k3、patch、flashkda/mooncake wheel 已就位
|
||||||
|
# （/data/flashkda_deploy/wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-*.whl）
|
||||||
|
# - 8 台节点可 ssh（NODE_SSH_USER），有 docker 权限
|
||||||
|
# - 本脚本从任一可 ssh 全集群的节点执行
|
||||||
|
#
|
||||||
|
# 重要:
|
||||||
|
# - 必须先启动 P 组再启动 D 组（prefill 需先注册 bootstrap）
|
||||||
|
# - 不能设置 PYTORCH_CUDA_ALLOC_CONF=expandable_segments
|
||||||
|
# （mooncake RDMA 注册 expandable GPU 段报 Bad address，见 GitHub #2511）
|
||||||
|
# - P/D 组的容器启停复用 deploy 层（python -m sskj.deploy），profile 为单一事实来源
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ROOT_DIR="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||||
|
|
||||||
|
MASTER_IP="174.1.60.1"
|
||||||
|
ROUTER_IP="174.1.60.5"
|
||||||
|
IMAGE="lmsysorg/sglang:kimi-k3"
|
||||||
|
MC_MASTER_CONTAINER="mc-master"
|
||||||
|
ROUTER_CONTAINER="router"
|
||||||
|
MODEL_PATH="/data/hf_models/Kimi-K3"
|
||||||
|
|
||||||
|
P_PROFILE="${ROOT_DIR}/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env"
|
||||||
|
D_PROFILE="${ROOT_DIR}/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env"
|
||||||
|
|
||||||
|
P_HOSTS=(174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4)
|
||||||
|
D_HOSTS=(174.1.60.5 174.1.60.6 174.1.60.7 174.1.60.8)
|
||||||
|
|
||||||
|
log() { echo "[$(date +%H:%M:%S)] $*"; }
|
||||||
|
|
||||||
|
ssh_node() { ssh -o ConnectTimeout=15 -o StrictHostKeyChecking=no "$@"; }
|
||||||
|
|
||||||
|
deploy_util() {
|
||||||
|
local profile="$1" container="$2" cmd="$3"
|
||||||
|
PYTHONPATH="${ROOT_DIR}/src" python3 -m sskj.deploy "$cmd" \
|
||||||
|
--profile "$profile" \
|
||||||
|
--tp 32 --dp 1 --port 30000 --model-path "$MODEL_PATH" \
|
||||||
|
--container-name "$container"
|
||||||
|
}
|
||||||
|
|
||||||
|
start_mc_master() {
|
||||||
|
log "starting mc-master on ${MASTER_IP}"
|
||||||
|
ssh_node "$MASTER_IP" "docker rm -f ${MC_MASTER_CONTAINER} >/dev/null 2>&1 || true; docker run -d --name ${MC_MASTER_CONTAINER} --network host ${IMAGE} mooncake_master" >/dev/null
|
||||||
|
sleep 3
|
||||||
|
if ssh_node "$MASTER_IP" "ss -tlnp 2>/dev/null | grep -q :50051"; then
|
||||||
|
log "mc-master is up (50051)"
|
||||||
|
else
|
||||||
|
log "WARN: mc-master 50051 not listening"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
start_p() {
|
||||||
|
log "starting P 组 (prefill) via deploy layer"
|
||||||
|
deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" start
|
||||||
|
}
|
||||||
|
|
||||||
|
start_d() {
|
||||||
|
log "starting D 组 (decode) via deploy layer"
|
||||||
|
deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" start
|
||||||
|
}
|
||||||
|
|
||||||
|
start_router() {
|
||||||
|
log "starting router (MiniLB) on ${ROUTER_IP}"
|
||||||
|
ssh_node "$ROUTER_IP" "docker rm -f ${ROUTER_CONTAINER} >/dev/null 2>&1 || true; docker run -d --name ${ROUTER_CONTAINER} --network host ${IMAGE} python3 -m sglang_router.launch_router --pd-disaggregation --mini-lb --prefill http://${MASTER_IP}:30000 28800 --decode http://${ROUTER_IP}:30000 --host 0.0.0.0 --port 31000" >/dev/null
|
||||||
|
sleep 3
|
||||||
|
if ssh_node "$ROUTER_IP" "ss -tlnp 2>/dev/null | grep -q :31000"; then
|
||||||
|
log "router is up (31000)"
|
||||||
|
else
|
||||||
|
log "WARN: router 31000 not listening"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_all() {
|
||||||
|
log "stopping router (${ROUTER_IP})"
|
||||||
|
ssh_node "$ROUTER_IP" "docker rm -f ${ROUTER_CONTAINER} >/dev/null 2>&1 || true" || true
|
||||||
|
log "stopping D 组 via deploy layer"
|
||||||
|
deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" stop || true
|
||||||
|
log "stopping P 组 via deploy layer"
|
||||||
|
deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" stop || true
|
||||||
|
log "stopping mc-master (${MASTER_IP})"
|
||||||
|
ssh_node "$MASTER_IP" "docker rm -f ${MC_MASTER_CONTAINER} >/dev/null 2>&1 || true" || true
|
||||||
|
log "all stopped"
|
||||||
|
}
|
||||||
|
|
||||||
|
status_all() {
|
||||||
|
echo "=== mc-master ==="
|
||||||
|
ssh_node "$MASTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${MC_MASTER_CONTAINER} || echo stopped"
|
||||||
|
echo "=== P 组 ==="
|
||||||
|
deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" status || true
|
||||||
|
echo "=== D 组 ==="
|
||||||
|
deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" status || true
|
||||||
|
echo "=== router ==="
|
||||||
|
ssh_node "$ROUTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${ROUTER_CONTAINER} || echo stopped"
|
||||||
|
}
|
||||||
|
|
||||||
|
case "${1:-}" in
|
||||||
|
start) start_mc_master; start_p; sleep 30; start_d; sleep 30; start_router ;;
|
||||||
|
stop) stop_all ;;
|
||||||
|
status) status_all ;;
|
||||||
|
restart) stop_all; sleep 5; start_mc_master; start_p; sleep 30; start_d; sleep 30; start_router ;;
|
||||||
|
*) echo "用法: $0 {start|stop|status|restart}"; exit 1 ;;
|
||||||
|
esac
|
||||||
|
echo "done."
|
||||||
Loading…
x
Reference in New Issue
Block a user