30 lines
3.0 KiB
Bash
30 lines
3.0 KiB
Bash
# P800 + SGLang-XPU + GLM-5.2-W8A8-INT8-Dynamic deployment profile.
|
|
# Supports the 2-node deploy previously living in deploy_glm52_2nodes.sh.
|
|
# Model-team only. Ops should not run `python -m sskj.deploy`.
|
|
|
|
PLATFORM=p800
|
|
MODEL_NAME=GLM-5.2-W8A8-INT8-Dynamic
|
|
ENGINE=sglang
|
|
RUNTIME=docker
|
|
DOCKER_IMAGE=iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202
|
|
CONTAINER_NAME=glm52_deploy
|
|
MODEL_PATH=/data1/models/GLM-5.2-W8A8-INT8-Dynamic
|
|
SERVED_MODEL_NAME=glm52
|
|
PORT=30000
|
|
HEALTH_PATH=/health
|
|
HEALTH_WAIT_S=600
|
|
NNODES=2
|
|
MASTER_IP=10.100.11.2
|
|
DIST_PORT=6311
|
|
DP_SIZE=2
|
|
|
|
DEVICE_VARS="XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 CUDA_DEVICE_ORDER=OAM_ID"
|
|
ENGINE_ENV="BKCL_ENABLE_XDR=1 BKCL_ENABLE_IBGDA=1 BKCL_SOCKET_IFNAME=eth0 BKCL_RDMA_NICS=eth1,eth2,eth3,eth4,eth5,eth6,eth7,eth8 BKCL_TREE_THRESHOLD=1048576 XSHMEM_MODE=1 XSHMEM_QP_NUM_PER_RANK=32 BKCL_RDMA_VERBS=1 NCCL_IB_GID_INDEX=3 CUDA_ENABLE_P2P_NO_UVA=1 SGLANG_USE_TRANSFORMERS_V5_TOKENIZER=1 XSGL_INTERTYPE_BFP16=1 ENABLE_FAST_BFP16_ATTN=1 XSGL_USE_DEEP_GEMM_BMM=1 XSGL_XDNN_QUANT=1 XSGL_FUSE_RMS_NORM_QUANT=1 XSGL_TRANSPOSE_MATMUL_WEIGHT=1 XINFER_QUANT_SDNN=1 XMLIR_FORCE_USE_XPU_GRAPH=1 XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1 XSGL_INT8_LM_HEAD=1 SGLANG_CLEAN_REQUEST_WHEN_RETRACT=1 SGLANG_ENABLE_SPEC_V2=True SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 USE_FAST_BFP16_FC=1 XPUAPI_SDNN_BF16_ROUND_MODE=3 XMLIR_ENABLE_FAST_FC=true XSGL_FAST_SWIGLU=1 SGLANG_NSA_DUAL_STREAM=true SGLANG_NSA_QUANT_WQ_B_WK=false XPU_FLASH_ATTENTION_DECODER_USE_BALANCE=true"
|
|
|
|
DOCKER_FLAGS="--privileged --network host --security-opt label=disable --shm-size 64m --device /dev/xpu0:/dev/xpu0 --device /dev/xpu1:/dev/xpu1 --device /dev/xpu2:/dev/xpu2 --device /dev/xpu3:/dev/xpu3 --device /dev/xpu4:/dev/xpu4 --device /dev/xpu5:/dev/xpu5 --device /dev/xpu6:/dev/xpu6 --device /dev/xpu7:/dev/xpu7 --device /dev/xpuctrl:/dev/xpuctrl"
|
|
VOLUMES="${MODEL_PATH}:/models:ro ${MODEL_PATH}:${MODEL_PATH}:ro"
|
|
|
|
LAUNCH_ARGS="--host 0.0.0.0 --port ${PORT} --model-path /models --dist-init-addr ${MASTER_IP}:${DIST_PORT} --nnodes ${NNODES} --node-rank ${NODE_RANK} --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.82 --max-prefill-tokens 65536 --chunked-prefill-size 16384 --max-running-requests 128 --tp-size 16 --ep-size 16 --dp-size ${DP_SIZE} --enable-dp-attention --moe-dense-tp-size 1 --enable-dp-lm-head --moe-a2a-backend deepep --deepep-mode auto --quantization w8a8_int8 --disable-shared-experts-fusion --kv-cache-dtype float16 --disable-radix-cache --watchdog-timeout 3000000 --tool-call-parser glm47 --reasoning-parser glm45 --enable-nsa-prefill-context-parallel --enable-single-batch-overlap"
|
|
|
|
BOOTSTRAP="source /root/miniconda/bin/activate python310_torch25_cuda; cd /workspace; pip install --no-index --find-links=/workspace/ /workspace/transformers-5.3.0-py3-none-any.whl 2>/dev/null || true; nohup python3 -m sglang.launch_server ${LAUNCH_ARGS} > /workspace/sglang_log_${NODE_RANK}.log 2>&1 & sleep infinity"
|