372ff08eea
- Implemented a bash script to run six benchmark experiments sequentially. - Added features for fault tolerance, out-of-memory recovery, and checkpoint management. - Included detailed logging for monitoring experiment progress and results. - Integrated GPU memory cleanup checks before each experiment. - Provided functionality to resume experiments from checkpoints.
181 lines
4.3 KiB
Bash
181 lines
4.3 KiB
Bash
# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench
|
|
# Custom benchmark client for multi-service vLLM TP=2 on 8x H200.
|
|
#
|
|
# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs:
|
|
# Service 1: port 30005, GPUs 0,1
|
|
# Service 2: port 30006, GPUs 2,3
|
|
# Service 3: port 30007, GPUs 4,5
|
|
# Service 4: port 30008, GPUs 6,7
|
|
#
|
|
# bench_client.py distributes requests across all 4 services via round-robin
|
|
# load balancing, giving accurate cluster-wide throughput even at low
|
|
# per-service concurrency.
|
|
|
|
EXPERIMENT="dsv4_h200_vllm_tp4_custom_bench"
|
|
MODEL_NAME="DeepSeek-V4-Flash"
|
|
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
|
SERVED_MODEL_NAME="deepseek-v4-flash"
|
|
|
|
# Primary port (backward compatible); multi-service ports are in PORTS.
|
|
PORT="30005"
|
|
PORTS="30005,30006"
|
|
|
|
BACKEND="vllm"
|
|
ENGINE="vllm"
|
|
|
|
# Native virtual environments on the host.
|
|
VENV_SERVER="/data/user1/yy/envs/vllm"
|
|
VENV_CLIENT="/data/user1/yy/envs/vllm"
|
|
|
|
# Load balancing strategy: round_robin | random | least_conn
|
|
LB_STRATEGY="round_robin"
|
|
|
|
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
|
|
# For multi-service we include low-concurrency scenarios to show how
|
|
# requests spread across 2 services.
|
|
SCENARIOS=(
|
|
"1 512 256 20"
|
|
"2 512 256 40"
|
|
"4 512 256 80"
|
|
"8 512 256 160"
|
|
"16 512 256 320"
|
|
"32 512 256 640"
|
|
"64 512 256 1280"
|
|
"128 512 256 2560"
|
|
|
|
"1 1024 128 20"
|
|
"1 1024 256 20"
|
|
"1 1024 512 20"
|
|
"1 1024 1024 20"
|
|
"1 1024 2048 20"
|
|
"1 1024 4096 20"
|
|
|
|
"8 1024 128 160"
|
|
"8 1024 256 160"
|
|
"8 1024 512 160"
|
|
"8 1024 1024 160"
|
|
"8 1024 2048 160"
|
|
"8 1024 4096 160"
|
|
|
|
"32 1024 128 640"
|
|
"32 1024 256 640"
|
|
"32 1024 512 640"
|
|
"32 1024 1024 640"
|
|
"32 1024 2048 640"
|
|
"32 1024 4096 640"
|
|
|
|
"128 1024 128 2560"
|
|
"128 1024 256 2560"
|
|
"128 1024 512 2560"
|
|
"128 1024 1024 2560"
|
|
"128 1024 2048 2560"
|
|
"128 1024 4096 2560"
|
|
|
|
"1 2048 128 20"
|
|
"1 2048 256 20"
|
|
"1 2048 512 20"
|
|
"1 2048 1024 20"
|
|
"1 2048 2048 20"
|
|
"1 2048 4096 20"
|
|
|
|
"8 2048 128 160"
|
|
"8 2048 256 160"
|
|
"8 2048 512 160"
|
|
"8 2048 1024 160"
|
|
"8 2048 2048 160"
|
|
"8 2048 4096 160"
|
|
|
|
"32 2048 128 640"
|
|
"32 2048 256 640"
|
|
"32 2048 512 640"
|
|
"32 2048 1024 640"
|
|
"32 2048 2048 640"
|
|
"32 2048 4096 640"
|
|
|
|
"128 2048 128 2560"
|
|
"128 2048 256 2560"
|
|
"128 2048 512 2560"
|
|
"128 2048 1024 2560"
|
|
"128 2048 2048 2560"
|
|
"128 2048 4096 2560"
|
|
|
|
"1 4096 128 20"
|
|
"1 4096 256 20"
|
|
"1 4096 512 20"
|
|
"1 4096 1024 20"
|
|
"1 4096 2048 20"
|
|
"1 4096 4096 20"
|
|
|
|
"8 4096 128 160"
|
|
"8 4096 256 160"
|
|
"8 4096 512 160"
|
|
"8 4096 1024 160"
|
|
"8 4096 2048 160"
|
|
"8 4096 4096 160"
|
|
|
|
"32 4096 128 640"
|
|
"32 4096 256 640"
|
|
"32 4096 512 640"
|
|
"32 4096 1024 640"
|
|
"32 4096 2048 640"
|
|
"32 4096 4096 640"
|
|
|
|
"128 4096 128 2560"
|
|
"128 4096 256 2560"
|
|
"128 4096 512 2560"
|
|
"128 4096 1024 2560"
|
|
"128 4096 2048 2560"
|
|
"128 4096 4096 2560"
|
|
|
|
"1 16384 128 20"
|
|
"1 16384 256 20"
|
|
"1 16384 512 20"
|
|
"1 16384 1024 20"
|
|
"1 16384 2048 20"
|
|
|
|
"8 16384 128 160"
|
|
"8 16384 256 160"
|
|
"8 16384 512 160"
|
|
"8 16384 1024 160"
|
|
"8 16384 2048 160"
|
|
|
|
"32 16384 128 640"
|
|
"32 16384 256 640"
|
|
"32 16384 512 640"
|
|
"32 16384 1024 640"
|
|
"32 16384 2048 640"
|
|
|
|
"128 16384 128 2560"
|
|
"128 16384 256 2560"
|
|
"128 16384 512 2560"
|
|
"128 16384 1024 2560"
|
|
"128 16384 2048 2560"
|
|
|
|
|
|
"1 65536 128 20"
|
|
"1 65536 256 20"
|
|
"1 65536 512 20"
|
|
"1 65536 1024 20"
|
|
|
|
"8 65536 128 160"
|
|
"8 65536 256 160"
|
|
"8 65536 512 160"
|
|
"8 65536 1024 160"
|
|
"32 65536 1024 160"
|
|
|
|
"1 131072 128 20"
|
|
"1 131072 256 20"
|
|
"1 131072 512 20"
|
|
|
|
"8 131072 512 160"
|
|
|
|
"1 262144 256 20"
|
|
|
|
"8 262144 128 160"
|
|
|
|
"1 524288 128 20"
|
|
)
|
|
|
|
# Server start script bundled with this experiment.
|
|
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"
|