06b3398ad0
- config.env: add SHAREGPT_DATASET, SHAREGPT_CONTEXT_LEN and SHAREGPT_SCENARIOS
- run_bench.sh: add run_sharegpt() and integrate it after Phase 1 for each backend
- parse_backend.py: support both phase{1|2} and sharegpt raw output filenames
- README.md: document ShareGPT scenarios
66 lines
2.0 KiB
Bash
66 lines
2.0 KiB
Bash
# Common configuration for the SGLang vs vLLM comparison on H200.
|
|
# All values can be overridden via environment variables.
|
|
|
|
EXPERIMENT="dsv4_h200_sglang_vs_vllm"
|
|
MODEL_NAME="DeepSeek-V4-Flash"
|
|
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
|
SERVED_MODEL_NAME="deepseek-v4-flash"
|
|
|
|
# Ports must differ so both backends can be tested independently.
|
|
SGLANG_PORT="${SGLANG_PORT:-30006}"
|
|
VLLM_PORT="${VLLM_PORT:-30005}"
|
|
|
|
# Virtual environments.
|
|
VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}"
|
|
VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}"
|
|
|
|
# Hardware: use all 8 H200 cards for both backends.
|
|
export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
|
|
TP=8
|
|
|
|
# Phase 1: short-context throughput (max_model_len 32k).
|
|
# Each element: "concurrency input_len output_len num_prompts"
|
|
if [[ -z "${PHASE1_SCENARIOS:-}" ]]; then
|
|
declare -a PHASE1_SCENARIOS=(
|
|
"1 512 256 32"
|
|
"32 512 256 128"
|
|
"128 512 256 128"
|
|
"1 4000 512 32"
|
|
"32 4000 512 64"
|
|
)
|
|
fi
|
|
PHASE1_MAX_MODEL_LEN=32768
|
|
PHASE1_MAX_NUM_SEQS=256
|
|
PHASE1_MAX_RUNNING=256
|
|
|
|
# Phase 2: long context (max_model_len 210k to leave room for output + padding).
|
|
if [[ -z "${PHASE2_SCENARIOS:-}" ]]; then
|
|
declare -a PHASE2_SCENARIOS=(
|
|
"1 32768 1024 8"
|
|
"1 65536 1024 4"
|
|
"1 131072 1024 2"
|
|
"1 200000 1024 1"
|
|
)
|
|
fi
|
|
PHASE2_MAX_MODEL_LEN=210000
|
|
PHASE2_MAX_NUM_SEQS=2
|
|
PHASE2_MAX_RUNNING=2
|
|
|
|
# ShareGPT dataset test (uses the same max_model_len as Phase 1).
|
|
SHAREGPT_DATASET="${SHAREGPT_DATASET:-/data/user1/yy/datasets/ShareGPT_filtered_chat.json}"
|
|
SHAREGPT_CONTEXT_LEN=32768
|
|
if [[ -z "${SHAREGPT_SCENARIOS:-}" ]]; then
|
|
declare -a SHAREGPT_SCENARIOS=(
|
|
"1 32768 256 32"
|
|
"32 32768 256 128"
|
|
)
|
|
fi
|
|
|
|
# Benchmark client always runs from the sglang env so that the same
|
|
# sglang.bench_serving version is used for both backends.
|
|
VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"
|
|
|
|
# Server start scripts bundled with this experiment.
|
|
SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh"
|
|
VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh"
|