37 lines
1.4 KiB
Bash
Executable File
37 lines
1.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
RESULT_DIR=/data/wxy/sskj-h3/throughput/sglang-base/results/subblock-sm120-probe
|
|
SGLANG_BIN=/root/.miniconda3/envs/sglang-lora/bin/sglang
|
|
MEDIA_BIN_DIR=/root/.miniconda3/envs/deploy/bin
|
|
|
|
mkdir -p "$RESULT_DIR"
|
|
env -u SGLANG_CACHE_DIT_ENABLED -u SGLANG_CACHE_DIT_FN -u SGLANG_CACHE_DIT_BN \
|
|
-u SGLANG_CACHE_DIT_RDT -u SGLANG_CACHE_DIT_MC -u SGLANG_CACHE_DIT_WARMUP \
|
|
-u SGLANG_CACHE_DIT_TAYLORSEER -u SGLANG_CACHE_DIT_SCM_PRESET \
|
|
-u SGLANG_CACHE_DIT_SCM_POLICY \
|
|
CUDA_VISIBLE_DEVICES=0,1 \
|
|
PATH="$MEDIA_BIN_DIR:$PATH" \
|
|
PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \
|
|
PYTHONUNBUFFERED=1 TOKENIZERS_PARALLELISM=false SGLANG_USE_RUNAI_MODEL_STREAMER=false \
|
|
"$SGLANG_BIN" serve \
|
|
--model-path /data/hf_models/MiniMax-H3 \
|
|
--model-variant Ref2VA \
|
|
--backend sglang \
|
|
--performance-mode speed \
|
|
--num-gpus 2 \
|
|
--tp-size 2 \
|
|
--ulysses-degree 1 \
|
|
--use-fsdp-inference false \
|
|
--enable-torch-compile false \
|
|
--batching-max-size 1 \
|
|
--batching-delay-ms 0 \
|
|
--host 127.0.0.1 \
|
|
--port 34910 \
|
|
--master-port 35910 \
|
|
--scheduler-port 36910 \
|
|
--output-path "$RESULT_DIR/outputs" \
|
|
--attention-backend subblock_sparse_attn \
|
|
--component-attention-backends text_encoder=fa \
|
|
--attention-backend-config /data/wxy/sskj-h3/throughput/sglang-base/configs/subblock_sparse_075.json
|