feat(pro6000): Kimi-K3 DP=2 部署（方案 B：两个独立 TP32×EP32 实例 + router 负载均衡）
- 实例 A profile 加 --disable-radix-cache（bench 测量纯净） - 新增实例 B profile（kimi3_pro6000_sglang_tp32ep32_instB，.1-.4） - 新增 deploy_dp2.sh 编排脚本（A + B + router --worker-urls） - 新增 README
This commit is contained in:
shishi
parent
c69831f258
commit
987f1db4b0
@ -51,4 +51,4 @@ PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro"
|
||||
# SGLANG_HOST_IP 必须为本节点实际 IP（174.1.60.5~8 = 5 + NODE_RANK）。
|
||||
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && export SGLANG_HOST_IP=\"174.1.60.$((5 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"
|
||||
|
||||
LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer_lazy --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --host 0.0.0.0 --port ${PORT}"
|
||||
LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --host 0.0.0.0 --port ${PORT}"
|
||||
|
||||
@ -0,0 +1,61 @@
|
||||
# Kimi-K3 SGLang multi-node TP=32 EP=32 deployment profile — 实例 B (DP=2 的第二个实例).
|
||||
# Nodes: 174.1.60.1~4 (rank 0~3), 32x NVIDIA RTX 6000D (85GB, sm_120).
|
||||
#
|
||||
# DP=2 方案 B：两个独立实例（实例 A: .5-.8, 实例 B: .1-.4），各自 TP32×EP32，
|
||||
# 前面用 sglang_router（--worker-urls）做负载均衡。实例间零通信。
|
||||
#
|
||||
# 与实例 A（kimi3_pro6000_sglang_tp32ep32）的差异仅在于节点拓扑：
|
||||
# - NODE_HOSTS = .1-.4
|
||||
# - MASTER_IP = 174.1.60.1（从 .1 执行部署，LOCAL_NODE_RANK=0 本地起 rank0）
|
||||
# - SGLANG_HOST_IP = 174.1.60.$(1 + NODE_RANK)
|
||||
# 其余并行度/网络配置与实例 A 一致。
|
||||
#
|
||||
# 关键点（与实例 A 相同，勿随意改）:
|
||||
# - MoE 后端必须 marlin（K3 的 MXFP4 缩放因子为 uint8，DeepGEMM 只接受 fp32/UE8M0）
|
||||
# - RoCE: NCCL_IB_HCA=mlx5_0..3（4 张独立卡, 10.100.21-24/24, RoCEv2 GID index 3）
|
||||
# - 容器必须 --ulimit memlock=-1（否则 ibv_create_cq 报 Cannot allocate memory）
|
||||
# - 不要设 NCCL_ALGO=TREE
|
||||
# - --disable-radix-cache 关闭前缀缓存（bench 测量纯净）
|
||||
#
|
||||
# Model-team only. Ops only run `python -m sskj.bench` against the router URL.
|
||||
|
||||
PLATFORM=pro6000
|
||||
EXPERIMENT=kimi3_pro6000_sglang_tp32ep32_instB
|
||||
MODEL_NAME=Kimi-K3
|
||||
ENGINE=sglang
|
||||
RUNTIME=docker
|
||||
DOCKER_IMAGE=lmsysorg/sglang:kimi-k3
|
||||
CONTAINER_NAME=${EXPERIMENT}_node${NODE_RANK}
|
||||
MODEL_PATH=/data/hf_models/Kimi-K3
|
||||
SERVED_MODEL_NAME=kimi-k3
|
||||
PORT=30000
|
||||
HEALTH_PATH=/health
|
||||
HEALTH_HOST=174.1.60.1
|
||||
HEALTH_WAIT_S=2400
|
||||
CONTAINER_PYTHON=python3
|
||||
|
||||
# ---- 并行度（固定，勿改）----
|
||||
TP=32
|
||||
DP=1
|
||||
EP_SIZE=32
|
||||
|
||||
# ---- Multi-node topology (实例 B = .1-.4; rank 0 exposes the HTTP API) ----
|
||||
NNODES=4
|
||||
NODE_HOSTS="174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4"
|
||||
NODE_SSH_USER=root
|
||||
LOCAL_NODE_RANK=0
|
||||
MASTER_IP=174.1.60.1
|
||||
DIST_PORT=20000
|
||||
|
||||
DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
|
||||
ENGINE_ENV="NCCL_SOCKET_IFNAME=bond0 GLOO_SOCKET_IFNAME=bond0 NCCL_IB_HCA=mlx5_0,mlx5_1,mlx5_2,mlx5_3 NCCL_IB_GID_INDEX=3 NCCL_IB_TIMEOUT=22 NCCL_IB_RETRY_CNT=7 NCCL_CUMEM_ENABLE=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 SGLANG_MOE_FUSED_GATE_RADIX=1"
|
||||
|
||||
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''"
|
||||
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
|
||||
PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro"
|
||||
|
||||
# BOOTSTRAP 在容器内执行: 打 sm_120 补丁 → 按节点 rank 计算 SGLANG_HOST_IP → 启动 sglang。
|
||||
# SGLANG_HOST_IP 必须为本节点实际 IP（174.1.60.1~4 = 1 + NODE_RANK）。
|
||||
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && export SGLANG_HOST_IP=\"174.1.60.$((1 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"
|
||||
|
||||
LAUNCH_ARGS="--model-path ${MODEL_PATH} --served-model-name ${SERVED_MODEL_NAME} --tp-size ${TP} --ep-size 32 --nnodes ${NNODES} --node-rank ${NODE_RANK} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --trust-remote-code --moe-runner-backend marlin --mem-fraction-static 0.88 --cuda-graph-max-bs-decode 16 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --dist-timeout 3600 --mamba-full-memory-ratio 0.36 --host 0.0.0.0 --port ${PORT}"
|
||||
63
experiments/pro6000/kimi3_pro6000_dp2/README.md
Normal file
63
experiments/pro6000/kimi3_pro6000_dp2/README.md
Normal file
@ -0,0 +1,63 @@
|
||||
# Kimi-K3 DP=2 部署（两个独立 TP32×EP32 实例 + router 负载均衡）
|
||||
|
||||
RTX 6000D 8 节点上 Kimi-K3 的 **DP=2** 部署：方案 B——**两个完全独立的 TP32×EP32 实例**，前面用 sglang_router 负载均衡。实例间零通信。
|
||||
|
||||
## 架构
|
||||
|
||||
```
|
||||
┌───────────────────────────────┐
|
||||
client ──32000──▶ │ router (sglang_router) .5 │ --worker-urls
|
||||
│ http://174.1.60.1:30000 │ ──▶ 实例 B (.1-.4, TP32×EP32)
|
||||
│ http://174.1.60.5:30000 │ ──▶ 实例 A (.5-.8, TP32×EP32)
|
||||
└───────────────────────────────┘
|
||||
```
|
||||
|
||||
- 实例 A：174.1.60.5~8，profile `kimi3_pro6000_sglang_tp32ep32`（现有）
|
||||
- 实例 B：174.1.60.1~4，profile `kimi3_pro6000_sglang_tp32ep32_instB`（新增）
|
||||
- router：`sglang_router.launch_router --worker-urls`（Rust Router 普通模式），对外 `.5:32000`
|
||||
|
||||
## 使用
|
||||
|
||||
```bash
|
||||
cd /data/yy/sskj/experiments/pro6000/kimi3_pro6000_dp2
|
||||
bash deploy_dp2.sh start # 实例 A → 实例 B → router（约 15 分钟）
|
||||
bash deploy_dp2.sh status
|
||||
bash deploy_dp2.sh stop
|
||||
bash deploy_dp2.sh restart
|
||||
```
|
||||
|
||||
## 验证
|
||||
|
||||
```bash
|
||||
# router 入口（自动负载均衡到两个实例）
|
||||
curl -s http://174.1.60.5:32000/generate -H "Content-Type: application/json" \
|
||||
-d '{"text":"What is 23*47?","sampling_params":{"max_new_tokens":16}}'
|
||||
|
||||
# 两个实例的 head 分别可直连
|
||||
curl -s http://174.1.60.1:30000/health
|
||||
curl -s http://174.1.60.5:30000/health
|
||||
```
|
||||
|
||||
## 关键配置
|
||||
|
||||
- 两个实例都是 **TP32×EP32**，`--disable-radix-cache`（bench 测量纯净）
|
||||
- 网络：NCCL/GLOO 走 bond0（管理网）？不——实例 A/B 各自 **NCCL_IB_HCA=mlx5_0..3**（计算网），`NCCL_SOCKET_IFNAME=bond0`
|
||||
- 每个实例的容器名：`kimi3_pro6000_sglang_tp32ep32_node{0-3}` / `..._instB_node{0-3}`
|
||||
- 部署从各自 master 执行：实例 A 从 .5（LOCAL_NODE_RANK=0 本地起 rank0）、实例 B 从 .1
|
||||
|
||||
## 前置条件
|
||||
|
||||
1. 8 节点镜像 `lmsysorg/sglang:kimi-k3`、`/tmp/patch_k3_sm120.py` 就位
|
||||
2. .5 与 .1 都有 sskj 仓库（`/data/yy/sskj`，deploy 层在各自 master 本地执行）
|
||||
3. 各节点 docker 权限、.5 可 ssh .1、.1 可 ssh .2/.3/.4
|
||||
4. 部署前需释放 GPU（如之前跑着 PD，先 `bash deploy_pd.sh stop`）
|
||||
|
||||
## 与 PD 分离的区别
|
||||
|
||||
| 维度 | PD 分离（MoonCake RDMA） | DP=2（方案 B） |
|
||||
|---|---|---|
|
||||
| 实例数 | 2 角色（prefill/decode） | 2 相同实例 |
|
||||
| 实例间通信 | 有（KV 经 MoonCake RDMA） | 无（零通信） |
|
||||
| 传输后端 | mooncake rdma | 无 |
|
||||
| router | PD 模式（--prefill/--decode） | 普通模式（--worker-urls） |
|
||||
| 用途 | 长上下文、KV 复用 | 横向扩容、吞吐 |
|
||||
94
experiments/pro6000/kimi3_pro6000_dp2/deploy_dp2.sh
Executable file
94
experiments/pro6000/kimi3_pro6000_dp2/deploy_dp2.sh
Executable file
@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
# Kimi-K3 DP=2 部署编排脚本（方案 B：两个独立 TP32×EP32 实例 + router 负载均衡）。
|
||||
#
|
||||
# 架构:
|
||||
# 实例 A: 174.1.60.5~8（TP32×EP32，现有 kimi3 profile）
|
||||
# 实例 B: 174.1.60.1~4（TP32×EP32，instB profile）
|
||||
# router: sglang_router --worker-urls 指向两个实例的 head（.5:30000 与 .1:30000）
|
||||
# 对外入口: .5:32000（router），实例间零通信。
|
||||
#
|
||||
# 用法:
|
||||
# bash deploy_dp2.sh start # 起 实例 A → 实例 B → router
|
||||
# bash deploy_dp2.sh stop # 停 router → 实例 B → 实例 A
|
||||
# bash deploy_dp2.sh status # 查看状态
|
||||
# bash deploy_dp2.sh restart
|
||||
#
|
||||
# 依赖:
|
||||
# - 8 节点镜像 lmsysorg/sglang:kimi-k3、patch_k3_sm120.py 已就位
|
||||
# - .5 与 .1 上都有 sskj 仓库（deploy 层在各自 master 本地执行）
|
||||
# - 各节点 docker 权限、.5 可 ssh .1
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT_DIR="$(cd "${SCRIPT_DIR}/../../.." && pwd)"
|
||||
|
||||
IMAGE="lmsysorg/sglang:kimi-k3"
|
||||
REPO_PATH="/data/yy/sskj"
|
||||
MODEL_PATH="/data/hf_models/Kimi-K3"
|
||||
|
||||
A_HEAD="174.1.60.5"
|
||||
B_HEAD="174.1.60.1"
|
||||
ROUTER_IP="174.1.60.5"
|
||||
ROUTER_CONTAINER="router_dp2"
|
||||
ROUTER_PORT="32000"
|
||||
|
||||
PROFILE_A="pro6000/kimi3_pro6000_sglang_tp32ep32"
|
||||
PROFILE_B="pro6000/kimi3_pro6000_sglang_tp32ep32_instB"
|
||||
|
||||
log() { echo "[$(date +%H:%M:%S)] $*"; }
|
||||
|
||||
ssh_node() { ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no "$@"; }
|
||||
|
||||
run_deploy_on() {
|
||||
local host="$1" profile="$2" container="$3" cmd="$4"
|
||||
ssh_node "$host" "cd ${REPO_PATH} && PYTHONPATH=src python3 -m sskj.deploy ${cmd} --profile ${profile} --tp 32 --dp 1 --port 30000 --model-path ${MODEL_PATH} --container-name ${container}"
|
||||
}
|
||||
|
||||
start_instance_a() {
|
||||
log "starting 实例 A (prefill .5-.8) on ${A_HEAD}"
|
||||
run_deploy_on "$A_HEAD" "$PROFILE_A" "kimi3_pro6000_sglang_tp32ep32" start
|
||||
}
|
||||
|
||||
start_instance_b() {
|
||||
log "starting 实例 B (.1-.4) on ${B_HEAD}"
|
||||
run_deploy_on "$B_HEAD" "$PROFILE_B" "kimi3_pro6000_sglang_tp32ep32_instB" start
|
||||
}
|
||||
|
||||
start_router() {
|
||||
log "starting router (--worker-urls) on ${ROUTER_IP}:${ROUTER_PORT}"
|
||||
ssh_node "$ROUTER_IP" "docker rm -f ${ROUTER_CONTAINER} >/dev/null 2>&1 || true; docker run -d --name ${ROUTER_CONTAINER} --network host ${IMAGE} python3 -m sglang_router.launch_router --worker-urls http://${B_HEAD}:30000 http://${A_HEAD}:30000 --host 0.0.0.0 --port ${ROUTER_PORT}" >/dev/null
|
||||
sleep 5
|
||||
if ssh_node "$ROUTER_IP" "ss -tlnp 2>/dev/null | grep -q :${ROUTER_PORT}"; then
|
||||
log "router is up (${ROUTER_PORT})"
|
||||
else
|
||||
log "WARN: router ${ROUTER_PORT} not listening"
|
||||
fi
|
||||
}
|
||||
|
||||
stop_all() {
|
||||
log "stopping router"
|
||||
ssh_node "$ROUTER_IP" "docker rm -f ${ROUTER_CONTAINER} >/dev/null 2>&1 || true" || true
|
||||
log "stopping 实例 B"
|
||||
run_deploy_on "$B_HEAD" "$PROFILE_B" "kimi3_pro6000_sglang_tp32ep32_instB" stop || true
|
||||
log "stopping 实例 A"
|
||||
run_deploy_on "$A_HEAD" "$PROFILE_A" "kimi3_pro6000_sglang_tp32ep32" stop || true
|
||||
log "all stopped"
|
||||
}
|
||||
|
||||
status_all() {
|
||||
echo "=== 实例 A (.5-.8) ==="
|
||||
run_deploy_on "$A_HEAD" "$PROFILE_A" "kimi3_pro6000_sglang_tp32ep32" status || true
|
||||
echo "=== 实例 B (.1-.4) ==="
|
||||
run_deploy_on "$B_HEAD" "$PROFILE_B" "kimi3_pro6000_sglang_tp32ep32_instB" status || true
|
||||
echo "=== router ==="
|
||||
ssh_node "$ROUTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${ROUTER_CONTAINER} || echo stopped"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
start) start_instance_a; sleep 20; start_instance_b; sleep 20; start_router ;;
|
||||
stop) stop_all ;;
|
||||
status) status_all ;;
|
||||
restart) stop_all; sleep 5; start_instance_a; sleep 20; start_instance_b; sleep 20; start_router ;;
|
||||
*) echo "用法: $0 {start|stop|status|restart}"; exit 1 ;;
|
||||
esac
|
||||
echo "done."
|
||||
Loading…
x
Reference in New Issue
Block a user