From 04dfa315836db811b6ffc405d1caf38a127a8c47 Mon Sep 17 00:00:00 2001 From: shishi Date: Tue, 11 Aug 2026 10:50:43 +0800 Subject: [PATCH] =?UTF-8?q?fix(pd):=20PATCH=5FMOUNTS=20=E8=A1=A5=E5=85=85?= =?UTF-8?q?=20flashkda=20wheel=20=E6=8C=82=E8=BD=BD=EF=BC=88BOOTSTRAP=20?= =?UTF-8?q?=E9=9C=80=E8=A6=81=20/flash=5Fkda-*.whl=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pro6000/kimi3_pro6000_pd_decode.env | 2 +- .../pro6000/kimi3_pro6000_pd_prefill.env | 2 +- .../kimi3_pro6000_pd_rdma/deploy_pd.sh | 46 +++++++++---------- 3 files changed, 25 insertions(+), 25 deletions(-) diff --git a/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env b/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env index c80d4be..818a720 100644 --- a/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env +++ b/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env @@ -46,7 +46,7 @@ ENGINE_ENV="MOONCAKE_MASTER=174.1.60.1:50051 MOONCAKE_PROTOCOL=rdma NCCL_SOCKET_ DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''" VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro" -PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /data/flashkda_deploy/wheels:/mc_wheels:ro" +PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:ro /data/flashkda_deploy/wheels:/mc_wheels:ro" # BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 decode。 BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((5 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}" diff --git a/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env b/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env index 63f496d..43891f5 100644 --- a/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env +++ b/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env @@ -52,7 +52,7 @@ ENGINE_ENV="MOONCAKE_MASTER=174.1.60.1:50051 MOONCAKE_PROTOCOL=rdma NCCL_SOCKET_ DOCKER_FLAGS="--gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband --shm-size 32g --entrypoint ''" VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro" -PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /data/flashkda_deploy/wheels:/mc_wheels:ro" +PATCH_MOUNTS="/tmp/patch_k3_sm120.py:/tmp/patch_k3_sm120.py:ro /tmp/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:/flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl:ro /data/flashkda_deploy/wheels:/mc_wheels:ro" # BOOTSTRAP: 打补丁 → 装 flashkda + mooncake wheel → 按 rank 设 SGLANG_HOST_IP → 启动 prefill。 BOOTSTRAP="python3 /tmp/patch_k3_sm120.py && pip install /flash_kda-0.0.1-cp312-cp312-linux_x86_64.whl --no-deps -q && pip install /mc_wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-cp312-cp312-manylinux_2_28_x86_64.whl --no-deps -q && export SGLANG_HOST_IP=\"174.1.60.$((1 + ${NODE_RANK}))\" && exec python3 -m sglang.launch_server ${LAUNCH_ARGS}" diff --git a/experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh b/experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh index e1e7a10..20bc147 100755 --- a/experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh +++ b/experiments/pro6000/kimi3_pro6000_pd_rdma/deploy_pd.sh @@ -11,10 +11,12 @@ # 依赖: # - 8 台节点镜像 lmsysorg/sglang:kimi-k3、patch、flashkda/mooncake wheel 已就位 # （/data/flashkda_deploy/wheels/mooncake_transfer_engine_cuda13-0.3.12.post1-*.whl） -# - 8 台节点可 ssh（NODE_SSH_USER），有 docker 权限 -# - 本脚本从任一可 ssh 全集群的节点执行 +# - 本脚本在任一可 ssh 全集群的节点执行（建议 .5，sskj 仓库所在） +# - 各节点有 docker 权限 # -# 重要: +# 重要（实测踩坑）: +# - P 组部署必须在 174.1.60.1 上执行（deploy 层 LOCAL_NODE_RANK=0 在本地起 node0） +# - D 组部署必须在 174.1.60.5 上执行 # - 必须先启动 P 组再启动 D 组（prefill 需先注册 bootstrap） # - 不能设置 PYTORCH_CUDA_ALLOC_CONF=expandable_segments # （mooncake RDMA 注册 expandable GPU 段报 Bad address，见 GitHub #2511） @@ -30,23 +32,21 @@ IMAGE="lmsysorg/sglang:kimi-k3" MC_MASTER_CONTAINER="mc-master" ROUTER_CONTAINER="router" MODEL_PATH="/data/hf_models/Kimi-K3" +REPO_PATH="/data/yy/sskj" -P_PROFILE="${ROOT_DIR}/deploy/profiles/pro6000/kimi3_pro6000_pd_prefill.env" -D_PROFILE="${ROOT_DIR}/deploy/profiles/pro6000/kimi3_pro6000_pd_decode.env" - -P_HOSTS=(174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4) -D_HOSTS=(174.1.60.5 174.1.60.6 174.1.60.7 174.1.60.8) +P_PROFILE="pro6000/kimi3_pro6000_pd_prefill" +D_PROFILE="pro6000/kimi3_pro6000_pd_decode" +P_HEAD="174.1.60.1" +D_HEAD="174.1.60.5" log() { echo "[$(date +%H:%M:%S)] $*"; } -ssh_node() { ssh -o ConnectTimeout=15 -o StrictHostKeyChecking=no "$@"; } +ssh_node() { ssh -o ConnectTimeout=20 -o StrictHostKeyChecking=no "$@"; } -deploy_util() { - local profile="$1" container="$2" cmd="$3" - PYTHONPATH="${ROOT_DIR}/src" python3 -m sskj.deploy "$cmd" \ - --profile "$profile" \ - --tp 32 --dp 1 --port 30000 --model-path "$MODEL_PATH" \ - --container-name "$container" +# 在指定头节点上执行 deploy 层命令（头节点本地起 rank0，其余节点 ssh 分发）。 +run_deploy_on() { + local host="$1" profile="$2" container="$3" cmd="$4" + ssh_node "$host" "cd ${REPO_PATH} && PYTHONPATH=src python3 -m sskj.deploy ${cmd} --profile ${profile} --tp 32 --dp 1 --port 30000 --model-path ${MODEL_PATH} --container-name ${container}" } start_mc_master() { @@ -61,13 +61,13 @@ start_mc_master() { } start_p() { - log "starting P 组 (prefill) via deploy layer" - deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" start + log "starting P 组 (prefill) on ${P_HEAD}" + run_deploy_on "$P_HEAD" "$P_PROFILE" "kimi3_pro6000_pd_prefill" start } start_d() { - log "starting D 组 (decode) via deploy layer" - deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" start + log "starting D 组 (decode) on ${D_HEAD}" + run_deploy_on "$D_HEAD" "$D_PROFILE" "kimi3_pro6000_pd_decode" start } start_router() { @@ -85,9 +85,9 @@ stop_all() { log "stopping router (${ROUTER_IP})" ssh_node "$ROUTER_IP" "docker rm -f ${ROUTER_CONTAINER} >/dev/null 2>&1 || true" || true log "stopping D 组 via deploy layer" - deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" stop || true + run_deploy_on "$D_HEAD" "$D_PROFILE" "kimi3_pro6000_pd_decode" stop || true log "stopping P 组 via deploy layer" - deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" stop || true + run_deploy_on "$P_HEAD" "$P_PROFILE" "kimi3_pro6000_pd_prefill" stop || true log "stopping mc-master (${MASTER_IP})" ssh_node "$MASTER_IP" "docker rm -f ${MC_MASTER_CONTAINER} >/dev/null 2>&1 || true" || true log "all stopped" @@ -97,9 +97,9 @@ status_all() { echo "=== mc-master ===" ssh_node "$MASTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${MC_MASTER_CONTAINER} || echo stopped" echo "=== P 组 ===" - deploy_util "$P_PROFILE" "kimi3_pro6000_pd_prefill" status || true + run_deploy_on "$P_HEAD" "$P_PROFILE" "kimi3_pro6000_pd_prefill" status || true echo "=== D 组 ===" - deploy_util "$D_PROFILE" "kimi3_pro6000_pd_decode" status || true + run_deploy_on "$D_HEAD" "$D_PROFILE" "kimi3_pro6000_pd_decode" status || true echo "=== router ===" ssh_node "$ROUTER_IP" "docker ps --format '{{.Names}} {{.Status}}' | grep ${ROUTER_CONTAINER} || echo stopped" }