81 lines
3.3 KiB
Bash
81 lines
3.3 KiB
Bash
#!/usr/bin/env bash
|
|
# Tiny adaptive-concurrency experiment for DeepSeek-V4-Flash on RTX 6000D
|
|
# (8 GPUs) using SGLang. It fixes OSL=1024 and sweeps ISL=1K..128K.
|
|
# TP=2/DP=4 is excluded because Marlin weight loading OOMs before KV Cache
|
|
# initialization on this machine.
|
|
# Tests SGLang with two parallel configurations:
|
|
# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas
|
|
# TP=8, DP=1 -> 8 GPUs, no data parallelism
|
|
|
|
EXPERIMENT="dsv4_pro6000_sglang_tiny_1k_output"
|
|
MODEL_NAME="DeepSeek-V4-Flash"
|
|
MODEL_PATH="/data/6000D/DeepSeek-V4-Flash"
|
|
SERVED_MODEL_NAME="deepseek-v4-flash"
|
|
|
|
SGLANG_PORT="${SGLANG_PORT:-30031}"
|
|
|
|
# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
|
|
# and the benchmark client. Defaults to the system python3 if the sglang venv
|
|
# does not exist on the host.
|
|
VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
|
|
|
|
# Run the benchmark client natively (0) or inside Docker (1).
|
|
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
|
|
|
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
|
|
|
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
|
|
# directory under this experiment so the benchmark is self-contained.
|
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
|
|
|
# Parallel configurations to test. Format: "TP DP"
|
|
declare -a PARALLEL_CONFIGS=(
|
|
"4 2"
|
|
"8 1"
|
|
)
|
|
|
|
# SGLang nightly server settings for this model and machine.
|
|
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.90}"
|
|
MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-auto}"
|
|
CONTEXT_LENGTH="${CONTEXT_LENGTH:-131072}"
|
|
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
|
|
# A C=1 OOM establishes a hard context-length boundary for this TP/DP group.
|
|
SKIP_CONFIG_ON_MIN_CONCURRENCY_OOM="${SKIP_CONFIG_ON_MIN_CONCURRENCY_OOM:-1}"
|
|
|
|
# Deployment switch. 0 = native sglang venv, 1 = Docker.
|
|
USE_DOCKER="${USE_DOCKER:-1}"
|
|
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:nightly-dev-cu13-20260720-b3570a45}"
|
|
|
|
# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
|
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
|
DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
|
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
|
|
CACHE_DIR="${CACHE_DIR:-${ROOT_DIR}/sglang_nightly_cu13_cache}"
|
|
|
|
# Matrix contains ISL=1K..128K with fixed OSL=1K.
|
|
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
|
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
|
|
|
# Sampling density for concurrency.
|
|
# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
|
|
# 2 = only test the low and high endpoints.
|
|
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
|
|
|
# Per-scenario timeout to avoid hangs (seconds).
|
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
|
|
|
# GPU memory sampling interval (seconds).
|
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
|
|
|
# Dry-run mode: if 1, only log the server args and scenario plan without starting
|
|
# any server or sending requests.
|
|
DRY_RUN="${DRY_RUN:-0}"
|
|
|
|
# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
|
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
|
|
|
# PyTorch CUDA allocator setting for the SGLang server. expandable_segments
|
|
# reduces fragmentation from the GiB-scale indexer temporaries that OOM the
|
|
# SM120 torch fallback (fp8_paged_mqa_logits_torch_sm120) at ISL >= 4096.
|
|
PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
|