51 lines
2.1 KiB
Bash
51 lines
2.1 KiB
Bash
# Experiment: Qwen3-235B-A22B on NVIDIA RTX 6000D (8x 84GB), SGLang, TP=8.
|
|
# Cross-machine counterpart of experiments/p800/qwen3_235b_p800_sglang_tp8
|
|
# (Kunlun P800). Same workload: isl=2048 osl=2048 concurrency=16 num_prompts=160.
|
|
#
|
|
# NVIDIA deployment uses the standard sglang qwen3_moe path (no Kunlun
|
|
# qwen3_moe.py patch, no XPU env, no --attention-backend kunlun). dtype is
|
|
# left to sglang auto (bf16) - NVIDIA native; p800.2 ran float16 (Kunlun).
|
|
|
|
EXPERIMENT="${EXPERIMENT:-qwen3_235b_pro6000_sglang_tp8}"
|
|
MODEL_NAME="${MODEL_NAME:-Qwen3-235B-A22B}"
|
|
MODEL_PATH="${MODEL_PATH:-/data/6000D/Qwen3-235B-A22B}"
|
|
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-/data/6000D/Qwen3-235B-A22B}"
|
|
BACKEND="${BACKEND:-sglang}"
|
|
ENGINE="${ENGINE:-sglang}"
|
|
DATASET="${DATASET:-random}"
|
|
DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
|
|
|
# Serving port + container.
|
|
PORT="${PORT:-30010}"
|
|
CONTAINER_NAME="${CONTAINER_NAME:-qwen3_235b_pro6000_sglang_tp8}"
|
|
|
|
# Docker image: sm120-capable SGLang (built/verified on this RTX 6000D box),
|
|
# sglang 0.5.15.post1 supports qwen3_moe. Falls back to nightly cu13 if needed.
|
|
DOCKER_IMAGE="${DOCKER_IMAGE:-sglang-sm120-dsv4:0.5.15.post1-fi0.6.14-sm120fix1}"
|
|
# Persistent JIT cache (flashinfer/sglang) shared with other pro6000 experiments.
|
|
CACHE_DIR="${CACHE_DIR:-/data/yy/sskj/sglang_sm120_cache}"
|
|
|
|
# Hardware: 8x NVIDIA RTX 6000D, TP=8.
|
|
TP="${TP:-8}"
|
|
CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
|
|
|
# SGLang launch params. Kept standard (NVIDIA natural config); only TP,
|
|
# mem-fraction, context-length, max-running-requests are pinned to match the
|
|
# p800.2 run for comparability.
|
|
CONTEXT_LENGTH="${CONTEXT_LENGTH:-8192}"
|
|
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-16}"
|
|
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}"
|
|
# dtype: auto (bf16). Set DTYPE=float16 to match p800.2 exactly (risk: fp16
|
|
# overflow on a bf16 model on NVIDIA).
|
|
DTYPE="${DTYPE:-auto}"
|
|
|
|
# Bench scenario: "concurrency input_len output_len num_prompts".
|
|
SCENARIOS=(
|
|
"16 2048 2048 160"
|
|
)
|
|
|
|
WARMUP="${WARMUP:-16}"
|
|
REQUEST_RATE="${REQUEST_RATE:-10000}"
|
|
SEED="${SEED:-1}"
|
|
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1}"
|