qqtang/sskj
1
3
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(pd): PATCH_MOUNTS 补充 flashkda wheel 挂载（BOOTSTRAP 需要 /flash_kda-*.whl）

Browse Source
This commit is contained in:
shishi 2026-08-11 10:50:43 +08:00
parent d72dbff689
commit 04dfa31583
3 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env
View File

@ -46,7 +46,7 @@ ENGINE_ENV="MOONCAKE_MASTER=174.1.60.1:50051 MOONCAKE_PROTOCOL=rdma NCCL_SOCKET_
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''" DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro" VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /data/flashkda_deploy/wheels:/mc_wheels:ro" PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:ro /data/flashkda_deploy/wheels:/mc_wheels:ro"
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 decode。 # BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 decode。
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((5 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}" BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((5 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"

2
deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env
View File

@ -52,7 +52,7 @@ ENGINE_ENV="MOONCAKE_MASTER=174.1.60.1:50051 MOONCAKE_PROTOCOL=rdma NCCL_SOCKET_
DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''" DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro" VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro"
PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /data/flashkda_deploy/wheels:/mc_wheels:ro" PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:ro /data/flashkda_deploy/wheels:/mc_wheels:ro"
# BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 prefill。 # BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 prefill。
BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((1 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}" BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((1 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}"

46
experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh
View File

@ -11,10 +11,12 @@
# 依赖: # 依赖:
# - 8 台节点镜像 lmsysorg/sglang:kimi-k3、patch、flashkda/mooncake wheel 已就位 # - 8 台节点镜像 lmsysorg/sglang:kimi-k3、patch、flashkda/mooncake wheel 已就位
# （/data/flashkda_deploy/wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-*.whl） # （/data/flashkda_deploy/wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-*.whl）
# - 8 台节点可 ssh（NODE_SSH_USER），有 docker 权限 # - 本脚本在任一可 ssh 全集群的节点执行（建议 .5，sskj 仓库所在）
# - 本脚本从任一可 ssh 全集群的节点执行 # - 各节点有 docker 权限
# #
# 重要: # 重要（实测踩坑）:
# - P 组部署必须在 174.1.60.1 上执行（deploy 层 LOCAL_NODE_RANK=0 在本地起 node0）
# - D 组部署必须在 174.1.60.5 上执行
# - 必须先启动 P 组再启动 D 组（prefill 需先注册 bootstrap） # - 必须先启动 P 组再启动 D 组（prefill 需先注册 bootstrap）
# - 不能设置 PYTORCH_CUDA_ALLOC_CONF=expandable_segments # - 不能设置 PYTORCH_CUDA_ALLOC_CONF=expandable_segments
# （mooncake RDMA 注册 expandable GPU 段报 Bad address，见 GitHub #2511） # （mooncake RDMA 注册 expandable GPU 段报 Bad address，见 GitHub #2511）
@ -30,23 +32,21 @@ IMAGE="lmsysorg/sglang:kimi-k3"
MC_MASTER_CONTAINER="mc-master" MC_MASTER_CONTAINER="mc-master"
ROUTER_CONTAINER="router" ROUTER_CONTAINER="router"
MODEL_PATH="/data/hf_models/Kimi-K3" MODEL_PATH="/data/hf_models/Kimi-K3"
REPO_PATH="/data/yy/sskj"
P_PROFILE="${ROOT_DIR}/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env" P_PROFILE="pro6000/kimi3_pro6000_pd_prefill"
D_PROFILE="${ROOT_DIR}/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env" D_PROFILE="pro6000/kimi3_pro6000_pd_decode"
P_HEAD="174.1.60.1"
P_HOSTS=(174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4) D_HEAD="174.1.60.5"
D_HOSTS=(174.1.60.5 174.1.60.6 174.1.60.7 174.1.60.8)
log() { echo "[$(date +%H:%M:%S)] $*"; } log() { echo "[$(date +%H:%M:%S)] $*"; }
ssh_node() { ssh -o ConnectTimeout=15 -o StrictHostKeyChecking=no "$@"; } ssh_node() { ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no "$@"; }
deploy_util() { # 在指定头节点上执行 deploy 层命令（头节点本地起 rank0，其余节点 ssh 分发）。
local profile="$1" container="$2" cmd="$3" run_deploy_on() {
PYTHONPATH="${ROOT_DIR}/src" python3 -m sskj.deploy "$cmd" \ local host="$1" profile="$2" container="$3" cmd="$4"
--profile "$profile" \ ssh_node "$host" "cd ${REPO_PATH} && PYTHONPATH=src python3 -m sskj.deploy ${cmd} --profile ${profile} --tp 32 --dp 1 --port 30000 --model-path ${MODEL_PATH} --container-name ${container}"
--tp 32 --dp 1 --port 30000 --model-path "$MODEL_PATH" \
--container-name "$container"
} }
start_mc_master() { start_mc_master() {
@ -61,13 +61,13 @@ start_mc_master() {
} }
start_p() { start_p() {
log "starting P 组 (prefill) via deploy layer" log "starting P 组 (prefill) on ${P_HEAD}"
deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" start run_deploy_on "$P_HEAD" "$P_PROFILE" "kimi3_pro6000_pd_prefill" start
} }
start_d() { start_d() {
log "starting D 组 (decode) via deploy layer" log "starting D 组 (decode) on ${D_HEAD}"
deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" start run_deploy_on "$D_HEAD" "$D_PROFILE" "kimi3_pro6000_pd_decode" start
} }
start_router() { start_router() {
@ -85,9 +85,9 @@ stop_all() {
log "stopping router (${ROUTER_IP})" log "stopping router (${ROUTER_IP})"
ssh_node "$ROUTER_IP" "docker rm -f ${ROUTER_CONTAINER} >/dev/null 2>&1 || true" || true ssh_node "$ROUTER_IP" "docker rm -f ${ROUTER_CONTAINER} >/dev/null 2>&1 || true" || true
log "stopping D 组 via deploy layer" log "stopping D 组 via deploy layer"
deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" stop || true run_deploy_on "$D_HEAD" "$D_PROFILE" "kimi3_pro6000_pd_decode" stop || true
log "stopping P 组 via deploy layer" log "stopping P 组 via deploy layer"
deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" stop || true run_deploy_on "$P_HEAD" "$P_PROFILE" "kimi3_pro6000_pd_prefill" stop || true
log "stopping mc-master (${MASTER_IP})" log "stopping mc-master (${MASTER_IP})"
ssh_node "$MASTER_IP" "docker rm -f ${MC_MASTER_CONTAINER} >/dev/null 2>&1 || true" || true ssh_node "$MASTER_IP" "docker rm -f ${MC_MASTER_CONTAINER} >/dev/null 2>&1 || true" || true
log "all stopped" log "all stopped"
@ -97,9 +97,9 @@ status_all() {
echo "=== mc-master ===" echo "=== mc-master ==="
ssh_node "$MASTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${MC_MASTER_CONTAINER} || echo stopped" ssh_node "$MASTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${MC_MASTER_CONTAINER} || echo stopped"
echo "=== P 组 ===" echo "=== P 组 ==="
deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" status || true run_deploy_on "$P_HEAD" "$P_PROFILE" "kimi3_pro6000_pd_prefill" status || true
echo "=== D 组 ===" echo "=== D 组 ==="
deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" status || true run_deploy_on "$D_HEAD" "$D_PROFILE" "kimi3_pro6000_pd_decode" status || true
echo "=== router ===" echo "=== router ==="
ssh_node "$ROUTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${ROUTER_CONTAINER} || echo stopped" ssh_node "$ROUTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${ROUTER_CONTAINER} || echo stopped"
} }