qqtang/sskj
1
3
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(pd): PATCH_MOUNTS 补充 flashkda wheel 挂载（BOOTSTRAP 需要 /flash_kda-*.whl）

Browse Source
This commit is contained in:
shishi 2026-08-11 10:50:43 +08:00
parent d72dbff689
commit 04dfa31583
3 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env
View File

@ -46,7 +46,7 @@ ENGINE_ENV="MOONCAKE_MASTER=174.1.60.1:50051 MOONCAKE_PROTOCOL=rdma NCCL_SOCKET_
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /data/flashkda_deploy/wheels:/mc_wheels:ro"
PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:ro /data/flashkda_deploy/wheels:/mc_wheels:ro"
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 decode。
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((5 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"

2
deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env
View File

@ -52,7 +52,7 @@ ENGINE_ENV="MOONCAKE_MASTER=174.1.60.1:50051 MOONCAKE_PROTOCOL=rdma NCCL_SOCKET_
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /data/flashkda_deploy/wheels:/mc_wheels:ro"
PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:ro /data/flashkda_deploy/wheels:/mc_wheels:ro"
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 prefill。
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((1 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"

46
experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh
View File

@ -11,10 +11,12 @@
# 依赖:
# - 8 台节点镜像 lmsysorg/sglang:kimi-k3、patch、flashkda/mooncake wheel 已就位
# （/data/flashkda_deploy/wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-*.whl）
# - 8 台节点可 ssh（NODE_SSH_USER），有 docker 权限
# - 本脚本从任一可 ssh 全集群的节点执行
# - 本脚本在任一可 ssh 全集群的节点执行（建议 .5，sskj 仓库所在）
# - 各节点有 docker 权限
#
# 重要:
# 重要（实测踩坑）:
# - P 组部署必须在 174.1.60.1 上执行（deploy 层 LOCAL_NODE_RANK=0 在本地起 node0）
# - D 组部署必须在 174.1.60.5 上执行
# - 必须先启动 P 组再启动 D 组（prefill 需先注册 bootstrap）
# - 不能设置 PYTORCH_CUDA_ALLOC_CONF=expandable_segments
# （mooncake RDMA 注册 expandable GPU 段报 Bad address，见 GitHub #2511）
@ -30,23 +32,21 @@ IMAGE="lmsysorg/sglang:kimi-k3"
MC_MASTER_CONTAINER="mc-master"
ROUTER_CONTAINER="router"
MODEL_PATH="/data/hf_models/Kimi-K3"
REPO_PATH="/data/yy/sskj"
P_PROFILE="${ROOT_DIR}/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env"
D_PROFILE="${ROOT_DIR}/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env"
P_HOSTS=(174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4)
D_HOSTS=(174.1.60.5 174.1.60.6 174.1.60.7 174.1.60.8)
P_PROFILE="pro6000/kimi3_pro6000_pd_prefill"
D_PROFILE="pro6000/kimi3_pro6000_pd_decode"
P_HEAD="174.1.60.1"
D_HEAD="174.1.60.5"
log() { echo "[$(date +%H:%M:%S)] $*"; }
ssh_node() { ssh -o ConnectTimeout=15 -o StrictHostKeyChecking=no "$@"; }
ssh_node() { ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no "$@"; }
deploy_util() {
local profile="$1" container="$2" cmd="$3"
PYTHONPATH="${ROOT_DIR}/src" python3 -m sskj.deploy "$cmd" \
--profile "$profile" \
--tp 32 --dp 1 --port 30000 --model-path "$MODEL_PATH" \
--container-name "$container"
# 在指定头节点上执行 deploy 层命令（头节点本地起 rank0，其余节点 ssh 分发）。
run_deploy_on() {
local host="$1" profile="$2" container="$3" cmd="$4"
ssh_node "$host" "cd ${REPO_PATH} && PYTHONPATH=src python3 -m sskj.deploy ${cmd} --profile ${profile} --tp 32 --dp 1 --port 30000 --model-path ${MODEL_PATH} --container-name ${container}"
}
start_mc_master() {
@ -61,13 +61,13 @@ start_mc_master() {
}
start_p() {
log "starting P 组 (prefill) via deploy layer"
deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" start
log "starting P 组 (prefill) on ${P_HEAD}"
run_deploy_on "$P_HEAD" "$P_PROFILE" "kimi3_pro6000_pd_prefill" start
}
start_d() {
log "starting D 组 (decode) via deploy layer"
deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" start
log "starting D 组 (decode) on ${D_HEAD}"
run_deploy_on "$D_HEAD" "$D_PROFILE" "kimi3_pro6000_pd_decode" start
}
start_router() {
@ -85,9 +85,9 @@ stop_all() {
log "stopping router (${ROUTER_IP})"
ssh_node "$ROUTER_IP" "docker rm -f ${ROUTER_CONTAINER} >/dev/null 2>&1 || true" || true
log "stopping D 组 via deploy layer"
deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" stop || true
run_deploy_on "$D_HEAD" "$D_PROFILE" "kimi3_pro6000_pd_decode" stop || true
log "stopping P 组 via deploy layer"
deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" stop || true
run_deploy_on "$P_HEAD" "$P_PROFILE" "kimi3_pro6000_pd_prefill" stop || true
log "stopping mc-master (${MASTER_IP})"
ssh_node "$MASTER_IP" "docker rm -f ${MC_MASTER_CONTAINER} >/dev/null 2>&1 || true" || true
log "all stopped"
@ -97,9 +97,9 @@ status_all() {
echo "=== mc-master ==="
ssh_node "$MASTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${MC_MASTER_CONTAINER} || echo stopped"
echo "=== P 组 ==="
deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" status || true
run_deploy_on "$P_HEAD" "$P_PROFILE" "kimi3_pro6000_pd_prefill" status || true
echo "=== D 组 ==="
deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" status || true
run_deploy_on "$D_HEAD" "$D_PROFILE" "kimi3_pro6000_pd_decode" status || true
echo "=== router ==="
ssh_node "$ROUTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${ROUTER_CONTAINER} || echo stopped"
}