72 lines
2.9 KiB
Bash
72 lines
2.9 KiB
Bash
#!/usr/bin/env bash
|
||
# Kimi-K3 TP=32 EP=32 benchmark experiment on 4x RTX 6000D (174.1.60.5~8).
|
||
# 固定单配置（TP32×EP32，跨 4 节点）；服务器生命周期走 deploy profile
|
||
# `pro6000/kimi3_pro6000_sglang_tp32ep32`（部署参数以 profile 为准，勿在此重复）。
|
||
|
||
EXPERIMENT="kimi3_pro6000_sglang_tp32ep32"
|
||
MODEL_NAME="Kimi-K3"
|
||
MODEL_PATH="/data/hf_models/Kimi-K3"
|
||
SERVED_MODEL_NAME="kimi-k3"
|
||
|
||
SGLANG_PORT="${SGLANG_PORT:-30000}"
|
||
|
||
# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
|
||
# and the benchmark client. Defaults to the system python3 if the sglang venv
|
||
# does not exist on the host.
|
||
VENV_CLIENT="${VENV_CLIENT:-/root/.miniconda3/envs/sglang}"
|
||
|
||
# Run the benchmark client natively (0) or inside Docker (1).
|
||
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
||
|
||
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||
|
||
# Runtime working directory for logs, pid files, and tmp.
|
||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||
|
||
# 单配置：TP=32 DP=1（EP=32 与 RoCE 等参数固定在 deploy profile 内）
|
||
declare -a PARALLEL_CONFIGS=(
|
||
"32 1"
|
||
)
|
||
|
||
# K3 服务器参数（与 deploy profile 保持一致，供本地/诊断脚本引用）。
|
||
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}"
|
||
MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}"
|
||
EP_SIZE="${EP_SIZE:-32}"
|
||
MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-64}"
|
||
|
||
# Deployment switch. 1 = Docker（走 deploy profile），0 = 本地 venv（仅单节点调试用）。
|
||
USE_DOCKER="${USE_DOCKER:-1}"
|
||
DOCKER_IMAGE="${DOCKER_IMAGE:-lmsysorg/sglang:kimi-k3}"
|
||
|
||
# Deploy profile used by start/stop scripts and the adaptive search loop.
|
||
DEPLOY_PROFILE="${DEPLOY_PROFILE:-pro6000/kimi3_pro6000_sglang_tp32ep32}"
|
||
|
||
# To use ShareGPT, set BENCH_DATASET_NAME=random and DATASET_PATH explicitly.
|
||
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||
DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/dataset/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||
SGLANG_BENCH_MODULE="${SGLANG_BENCH_MODULE:-sglang.benchmark.serving}"
|
||
|
||
# Matrix and concurrency rules are defined in matrix.json by default.
|
||
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||
|
||
# Sampling density for concurrency.
|
||
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
||
|
||
# Per-scenario timeout to avoid hangs (seconds). K3 首请求含 JIT 编译（~110s），
|
||
# 预热由 adaptive 框架处理；超时给足。
|
||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-3600}"
|
||
|
||
# GPU memory sampling interval (seconds).
|
||
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||
|
||
# Dry-run mode: if 1, only log the server args and scenario plan without starting
|
||
# any server or sending requests.
|
||
DRY_RUN="${DRY_RUN:-0}"
|
||
|
||
# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
|
||
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||
|
||
# PyTorch CUDA allocator setting for the SGLang server.
|
||
PYTORCH_CUDA_ALLOC_CONF="${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True}"
|