68 lines
2.5 KiB
Bash
68 lines
2.5 KiB
Bash
# ============================================================
|
||
# DeepSeek-V4-Pro SGLang TP=16 EP=2 + EAGLE speculative decoding
|
||
# Nodes: 10.101.0.11 (rank 0) + 10.101.0.13 (rank 1)
|
||
# GPUs: 16× RTX 6000D (85GB each)
|
||
#
|
||
# Key flags:
|
||
# SGLANG_SHARED_EXPERT_TP1=1 + --ep-size 2 → TP=16 with FP8 block=128
|
||
# --speculative-algorithm EAGLE → Eagle speculative decoding
|
||
# ============================================================
|
||
|
||
EXPERIMENT="dsv4_pro6000_sglang_tp16_eagle"
|
||
MODEL_NAME="DeepSeek-V4-Pro"
|
||
MODEL_PATH="/data/hf_models/DeepSeek-V4-Pro"
|
||
SERVED_MODEL_NAME="default"
|
||
|
||
# ---- Nodes (IPs for reliable SSH) ----
|
||
HEAD_NODE="10.101.0.11"
|
||
WORKER_NODE="10.101.0.13"
|
||
HEAD_IP="10.101.0.11"
|
||
DIST_INIT_PORT="${DIST_INIT_PORT:-20000}"
|
||
SGLANG_PORT="${SGLANG_PORT:-30000}"
|
||
|
||
# ---- Parallelism ----
|
||
TP_SIZE="${TP_SIZE:-16}"
|
||
EP_SIZE="${EP_SIZE:-2}"
|
||
NNODES="${NNODES:-2}"
|
||
|
||
# ---- Docker ----
|
||
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
||
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
||
|
||
# ---- NCCL ----
|
||
NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-eth0}"
|
||
NCCL_DEBUG="${NCCL_DEBUG:-WARN}"
|
||
|
||
# ---- SGLang memory / perf ----
|
||
# max_running_requests must >= max concurrency tested (128) to avoid queueing
|
||
# cuda_graph_max_bs_decode must >= max concurrent batch size for accurate decode perf
|
||
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.9}"
|
||
CUDA_GRAPH_MAX_BS_DECODE="${CUDA_GRAPH_MAX_BS_DECODE:-64}"
|
||
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-256}"
|
||
|
||
# ---- EAGLE speculative decoding ----
|
||
SPECULATIVE_ALGORITHM="${SPECULATIVE_ALGORITHM:-EAGLE}"
|
||
SPECULATIVE_NUM_STEPS="${SPECULATIVE_NUM_STEPS:-3}"
|
||
SPECULATIVE_EAGLE_TOPK="${SPECULATIVE_EAGLE_TOPK:-1}"
|
||
SPECULATIVE_NUM_DRAFT_TOKENS="${SPECULATIVE_NUM_DRAFT_TOKENS:-4}"
|
||
|
||
# ---- Benchmark ----
|
||
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||
DATASET_PATH="${DATASET_PATH:-/data/yy/sskj/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
|
||
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
|
||
WARMUP_REQUESTS="${WARMUP_REQUESTS:-16}"
|
||
NUM_REPETITIONS="${NUM_REPETITIONS:-3}"
|
||
|
||
# ---- Timeouts ----
|
||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||
HEALTH_CHECK_RETRIES="${HEALTH_CHECK_RETRIES:-600}"
|
||
HEALTH_CHECK_INTERVAL_S="${HEALTH_CHECK_INTERVAL_S:-5}"
|
||
|
||
# ---- Directories ----
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/results}"
|
||
|
||
# ---- Dry-run ----
|
||
DRY_RUN="${DRY_RUN:-0}"
|