# GLM-5.2-FP8 SGLang multi-node TP=16 deployment profile (2x RTX 6000D).
# node0 = 174.1.51.5 (pro6000D.1, HTTP API), node1 = 174.1.51.7 (pro6000D.3,
# compute-only). Model ~700GB fp8 so 2-node TP=16 is mandatory.
# Requires the tilelang/dsa patch mounts below (experiment-dir sources).

PLATFORM=pro6000
EXPERIMENT=glm52_pro6000_sglang_multinode_tp16
MODEL_NAME=GLM-5.2-FP8
ENGINE=sglang
RUNTIME=docker
DOCKER_IMAGE=lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45
CONTAINER_NAME=${EXPERIMENT}_sglang_tp${TP}_dp${DP}_node${NODE_RANK}
MODEL_PATH=/data/hf_models/GLM-5.2-FP8
SERVED_MODEL_NAME=GLM-5.2-FP8
PORT=30031
HEALTH_PATH=/health
HEALTH_HOST=174.1.51.5
HEALTH_WAIT_S=2400
CONTAINER_PYTHON=python3
TP=16
DP=1

# ---- Multi-node topology (rank order; rank 0 exposes the HTTP API) ----
NNODES=2
NODE_HOSTS="174.1.51.5 174.1.51.7"
NODE_SSH_USER=root
MASTER_IP=174.1.51.5
DIST_PORT=50000

DEVICE_VARS="CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7"
ENGINE_ENV="PYTHONUNBUFFERED=1 HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True,max_split_size_mb:256 NCCL_IB_HCA=mlx5_0,mlx5_3 NCCL_MIN_NCHANNELS=8 NCCL_IB_QPS_PER_CONNECTION=4 NCCL_NET_GDR_LEVEL=PHB NCCL_SOCKET_IFNAME=eth1 NCCL_IB_RETRY_CNT=14 NCCL_IB_TIMEOUT=16"

DOCKER_FLAGS="--gpus all --privileged --ipc=host --network host --ulimit memlock=-1 --ulimit stack=67108864"
VOLUMES="${MODEL_PATH}:${MODEL_PATH}:ro ${ROOT_DIR}/sglang_nightly_cu13_cache:/root/.cache"
PATCH_MOUNTS="${ROOT_DIR}/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/tilelang_kernel_sm120.py:/sgl-workspace/sglang/python/sglang/kernels/ops/attention/dsa/tilelang_kernel.py:ro ${ROOT_DIR}/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/dsa_backend.py:/sgl-workspace/sglang/python/sglang/srt/layers/attention/dsa_backend.py:ro ${ROOT_DIR}/experiments/pro6000/glm52_pro6000_sglang_multinode_tp16/dsa_indexer.py:/sgl-workspace/sglang/python/sglang/srt/layers/attention/dsa/dsa_indexer.py:ro"

LAUNCH_ARGS="--model-path ${MODEL_PATH} --trust-remote-code --tp-size ${TP} --dp-size ${DP} --quantization fp8 --kv-cache-dtype bfloat16 --moe-runner-backend auto --mem-fraction-static 0.80 --context-length 131072 --max-running-requests 64 --cuda-graph-backend-decode disabled --dsa-prefill-backend tilelang --dsa-decode-backend tilelang --host 0.0.0.0 --port ${PORT} --dist-init-addr ${MASTER_IP}:${DIST_PORT} --nnodes ${NNODES} --node-rank ${NODE_RANK}"