d783143ddb
- Run vLLM server inside vllm/vllm-openai:latest container on H200. - Run benchmark client inside lmsysorg/sglang:latest using sglang.bench_serving --backend vllm, matching the sglang reference. - Use RUNTIME_BASE for self-contained logs/pids/tmp instead of hardcoded /data/user1/yy paths. - Update model path to /data3/hf_models/DeepSeek-V4-Flash. - Sample only low/high concurrency endpoints to match sglang matrix. - Fix docker 'invalid reference format' caused by DOCKER_CLIENT_IMAGE not being exported to bash -c subshells used by run_bench_serving. - Add offline dummy ShareGPT dataset for random benchmark mode.
77 lines
2.9 KiB
Bash
77 lines
2.9 KiB
Bash
# TP×DP matrix experiment for DeepSeek-V4-Flash on H200 (8 GPUs) using vLLM.
|
||
# Tests vLLM with three parallel configurations:
|
||
# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas
|
||
# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas
|
||
# TP=8, DP=1 -> 8 GPUs, no data parallelism
|
||
|
||
EXPERIMENT="dsv4_h200_vllm_tp_dp_matrix"
|
||
MODEL_NAME="DeepSeek-V4-Flash"
|
||
MODEL_PATH="/data3/hf_models/DeepSeek-V4-Flash"
|
||
SERVED_MODEL_NAME="deepseek-v4-flash"
|
||
|
||
VLLM_PORT="${VLLM_PORT:-30030}"
|
||
|
||
# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
|
||
# and the benchmark client. Defaults to the system python3 if the sglang venv
|
||
# does not exist on the host.
|
||
VENV_CLIENT="${VENV_CLIENT:-/data/user1/yy/envs/sglang}"
|
||
|
||
# Run the benchmark client natively (0) or inside Docker (1).
|
||
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
||
|
||
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||
|
||
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
|
||
# directory under this experiment so the benchmark is self-contained.
|
||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||
|
||
# Parallel configurations to test. Format: "TP DP"
|
||
declare -a PARALLEL_CONFIGS=(
|
||
"2 4"
|
||
"4 2"
|
||
"8 1"
|
||
)
|
||
|
||
# vLLM server settings
|
||
MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
|
||
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
|
||
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
|
||
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
|
||
BLOCK_SIZE="${BLOCK_SIZE:-256}"
|
||
|
||
# Deployment switch. 0 = native vllm venv, 1 = Docker.
|
||
USE_DOCKER="${USE_DOCKER:-1}"
|
||
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
|
||
|
||
# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving,
|
||
# so the client runs inside the SGLang image and targets the vLLM backend via the
|
||
# OpenAI-compatible API.
|
||
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||
|
||
# Dataset used by sglang.bench_serving --dataset-name random.
|
||
# The random sampler needs a ShareGPT-style JSON file locally; it falls back to
|
||
# downloading from HuggingFace, which usually fails on offline H200 nodes.
|
||
DATASET_PATH="${DATASET_PATH:-${SCRIPT_DIR}/runtime/dummy_sharegpt.json}"
|
||
|
||
# Matrix and concurrency rules are defined in matrix.json by default.
|
||
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||
|
||
# Sampling density for concurrency.
|
||
# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
|
||
# 2 = only test the low and high endpoints.
|
||
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
||
|
||
# Per-scenario timeout to avoid hangs (seconds).
|
||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||
|
||
# GPU memory sampling interval (seconds).
|
||
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||
|
||
# Dry-run mode: if 1, only log the server args and scenario plan without starting
|
||
# any server or sending requests.
|
||
DRY_RUN="${DRY_RUN:-0}"
|
||
|
||
# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
|
||
GRID_LIMIT="${GRID_LIMIT:-0}"
|