28 lines
3.8 KiB
Bash
28 lines
3.8 KiB
Bash
# P800 + SGLang-XPU + DeepSeek-V4-Flash-INT8 deployment profile.
|
|
# Model-team only. Ops should NOT run `python -m sskj.deploy`; they only
|
|
# run `python -m sskj.bench` against the URL this profile serves.
|
|
|
|
PLATFORM=p800
|
|
MODEL_NAME=DeepSeek-V4-Flash-INT8
|
|
ENGINE=sglang
|
|
RUNTIME=docker
|
|
DOCKER_IMAGE=iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202
|
|
CONTAINER_NAME=sglang-dsv4-flash-tpdp
|
|
MODEL_PATH=/data1/models/DeepSeek-V4-Flash-INT8
|
|
SERVED_MODEL_NAME=deepseek-v4-flash-int8
|
|
PORT=30014
|
|
HEALTH_PATH=/health
|
|
HEALTH_WAIT_S=600
|
|
CONTAINER_PYTHON=/root/miniconda/envs/python310_torch25_cuda/bin/python
|
|
PATCH_ROOT=${ROOT_DIR}/platforms/patches/kunlun_p800
|
|
|
|
DEVICE_VARS="XPU_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 CUDA_DEVICE_ORDER=OAM_ID"
|
|
ENGINE_ENV="SGLANG_USE_TRANSFORMERS_V5_TOKENIZER=1 XMLIR_FORCE_USE_XPU_GRAPH=1 SGLANG_DSV4_MODE=2604 PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True SGLANG_NSA_DUAL_STREAM=true SGLANG_NSA_QUANT_WQ_B_WK=false SGLANG_FP8_PAGED_MQA_LOGITS_TORCH=1 SGLANG_OPT_DEEPGEMM_HC_PRENORM=false SGLANG_OPT_USE_TILELANG_MHC_PRE=1 SGLANG_OPT_USE_TILELANG_MHC_POST=1 SGLANG_CLEAN_REQUEST_WHEN_RETRACT=1 SGLANG_SET_CPU_AFFINITY=1 SGLANG_OPT_USE_KLX_TOPK_KERNEL=1 XSGL_INTERTYPE_BFP16=1 ENABLE_FAST_BFP16_ATTN=1 XSGL_USE_DEEP_GEMM_BMM=1 XSGL_XDNN_QUANT=1 XSGL_FUSE_RMS_NORM_QUANT=1 XSGL_TRANSPOSE_MATMUL_WEIGHT=1 XINFER_QUANT_SDNN=1 XSGL_USE_MOE_SIGMOID_GROUP_TOPK_NORM=1 XSGL_EARLY_FIRST_TOKEN=1 XSGL_ENABLE_TGEMM_FP16=1 SGLANG_ENABLE_SPEC_V2=True SGLANG_ENABLE_OVERLAP_PLAN_STREAM=1 PYTHONDONTWRITEBYTECODE=1 XTORCH_OPS_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/xtorch_ops XPU_RUNTIME_LIB_DIR=/root/miniconda/envs/python310_torch25_cuda/xcudart/lib BKCL_TREE_THRESHOLD=1048576 CUDA_ENABLE_P2P_NO_UVA=1 NCCL_IB_GID_INDEX=3 IS_DSV4=1 MC_CUSTOM_TOPO_JSON=/workspace/nic_priority_matrix_test.json SGLANG_DSV4_FP4_EXPERTS=false SGLANG_APPLY_CONFIG_BACKUP=auto BKCL_ENABLE_XDR=1 BKCL_RDMA_NICS=eth1,eth1,eth3,eth3,eth5,eth5,eth7,eth7 BKCL_RDMA_VERBS=1 XSGL_INT8_LM_HEAD=1 SGLANG_P800_ALL_GATHER_FALLBACK=0"
|
|
|
|
DOCKER_FLAGS="--privileged --network host --ipc host --device /dev/xpu0:/dev/xpu0 --device /dev/xpu1:/dev/xpu1 --device /dev/xpu2:/dev/xpu2 --device /dev/xpu3:/dev/xpu3 --device /dev/xpu4:/dev/xpu4 --device /dev/xpu5:/dev/xpu5 --device /dev/xpu6:/dev/xpu6 --device /dev/xpu7:/dev/xpu7 --device /dev/xpuctrl:/dev/xpuctrl"
|
|
VOLUMES="${MODEL_PATH}:/models:ro ${MODEL_PATH}:${MODEL_PATH}:ro ${PATCH_ROOT}/nic_priority_matrix_test.json:/workspace/nic_priority_matrix_test.json:ro"
|
|
|
|
LAUNCH_ARGS="--host 0.0.0.0 --port ${PORT} --model-path /models --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa --trust-remote-code --disable-custom-all-reduce --page-size 64 --mem-fraction-static 0.8 --tensor-parallel-size ${TP} --disable-shared-experts-fusion --quantization w8a8_int8 --kv-cache-dtype float16 --disable-piecewise-cuda-graph --cuda-graph-max-bs 32 --watchdog-timeout 3000000 --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 --constrained-json-disable-any-whitespace --enable-metrics --enable-request-time-stats-logging --context-length 65536"
|
|
DP_FLAG="--dp-size ${DP}"
|
|
BOOTSTRAP="cd /workspace; find /root/miniconda/envs/python310_torch25_cuda/lib/python3.10/site-packages/sglang -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true; ${CONTAINER_PYTHON} -m pip install --upgrade safetensors -q; ${CONTAINER_PYTHON} -m pip install https://files.pythonhosted.org/packages/14/8b/2a1333a6455c6fad401c2285dee6f58016c55b1cb44cae3a31f8a9cc7d83/apache_tvm_ffi-0.1.0b2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl -q; ${CONTAINER_PYTHON} -c \"import torch; torch.float8_e8m0fnu = torch.uint8; import runpy, sys; sys.argv[0] = 'sglang.launch_server'; runpy.run_module('sglang.launch_server', run_name='__main__')\" ${LAUNCH_ARGS}"
|