4d5b6a77ca
- Create configuration file for the custom benchmark experiment. - Implement result parsing script to handle JSONL outputs from the benchmark. - Develop run script to orchestrate the benchmark execution, including server management and health checks. - Add server start script to launch a single vLLM service on all available GPUs.
84 lines
2.3 KiB
Bash
84 lines
2.3 KiB
Bash
# Experiment configuration for dsv4_h200_vllm_tp2_custom_bench
|
|
# Custom benchmark client for multi-service vLLM TP=2 on 8x H200.
|
|
#
|
|
# This experiment runs 4 independent vLLM services (TP=2 each) on GPU pairs:
|
|
# Service 1: port 30005, GPUs 0,1
|
|
# Service 2: port 30006, GPUs 2,3
|
|
# Service 3: port 30007, GPUs 4,5
|
|
# Service 4: port 30008, GPUs 6,7
|
|
#
|
|
# bench_client.py distributes requests across all 4 services via round-robin
|
|
# load balancing, giving accurate cluster-wide throughput even at low
|
|
# per-service concurrency.
|
|
|
|
EXPERIMENT="dsv4_h200_vllm_tp8_custom_bench"
|
|
MODEL_NAME="DeepSeek-V4-Flash"
|
|
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
|
SERVED_MODEL_NAME="deepseek-v4-flash"
|
|
|
|
# Primary port (backward compatible); multi-service ports are in PORTS.
|
|
PORT="30005"
|
|
PORTS="30005"
|
|
|
|
BACKEND="vllm"
|
|
ENGINE="vllm"
|
|
|
|
# Native virtual environments on the host.
|
|
VENV_SERVER="/data/user1/yy/envs/vllm"
|
|
VENV_CLIENT="/data/user1/yy/envs/vllm"
|
|
|
|
# Load balancing strategy: round_robin | random | least_conn
|
|
LB_STRATEGY="round_robin"
|
|
|
|
# Benchmark scenarios: "concurrency input_len output_len num_prompts".
|
|
# For single-service TP=8 we cover standard concurrency gradients.
|
|
SCENARIOS=(
|
|
"1 512 256 20"
|
|
"2 512 256 40"
|
|
"4 512 256 80"
|
|
"8 512 256 160"
|
|
"16 512 256 320"
|
|
"32 512 256 640"
|
|
"64 512 256 1280"
|
|
"128 512 256 2560"
|
|
"1 2048 512 20"
|
|
"8 2048 512 160"
|
|
"32 2048 512 640"
|
|
"128 2048 512 2560"
|
|
"1 4096 1024 20"
|
|
"8 4096 1024 160"
|
|
"32 4096 1024 640"
|
|
"128 4096 1024 2560"
|
|
"1 8192 1024 20"
|
|
"8 8192 1024 160"
|
|
"32 8192 1024 640"
|
|
"128 8192 1024 2560"
|
|
"1 16384 1024 20"
|
|
"8 16384 1024 160"
|
|
"32 16384 1024 640"
|
|
"128 16384 1024 2560"
|
|
"1 32768 1024 20"
|
|
"8 32768 1024 160"
|
|
"32 32768 1024 640"
|
|
"128 32768 1024 2560"
|
|
"1 65536 1024 20"
|
|
"8 65536 1024 160"
|
|
"32 65536 1024 640"
|
|
"128 65536 1024 2560"
|
|
"1 131072 1024 20"
|
|
"8 131072 1024 160"
|
|
"32 131072 1024 640"
|
|
"128 131072 1024 2560"
|
|
"1 262144 1024 20"
|
|
"8 262144 1024 160"
|
|
"32 262144 1024 640"
|
|
"128 262144 1024 2560"
|
|
"1 524288 1024 20"
|
|
"8 524288 1024 160"
|
|
"32 524288 1024 640"
|
|
"128 524288 1024 2560"
|
|
)
|
|
|
|
# Server start script bundled with this experiment.
|
|
SERVER_START_SCRIPT="${SCRIPT_DIR}/start_server.sh"
|