19d336de08
- New experiments/dsv4_h200_max_context_length/ - start_sglang.sh / start_vllm.sh accept target length as argument - run_bench.sh tests a ladder of input lengths (default 64k -> 1M) with --random-range-ratio 1.0 for exact length - extract_metrics.py + parse_results.py produce results.json + report.md - README.md documents usage and control variables - Root README updated with entry and quick command
49 lines
1.4 KiB
Bash
49 lines
1.4 KiB
Bash
# Common configuration for the max context length exploration on H200.
|
|
# All values can be overridden via environment variables.
|
|
|
|
EXPERIMENT="dsv4_h200_max_context_length"
|
|
MODEL_NAME="DeepSeek-V4-Flash"
|
|
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
|
SERVED_MODEL_NAME="deepseek-v4-flash"
|
|
|
|
# Ports must differ from other running experiments.
|
|
SGLANG_PORT="${SGLANG_PORT:-30011}"
|
|
VLLM_PORT="${VLLM_PORT:-30012}"
|
|
|
|
# Virtual environments.
|
|
VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}"
|
|
VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}"
|
|
VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"
|
|
|
|
# Hardware: use all 8 H200 cards for both backends.
|
|
export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
|
|
TP=8
|
|
|
|
# We want exact input lengths, so fix the random range ratio to 1.0.
|
|
OUTPUT_LEN="${OUTPUT_LEN:-1}"
|
|
NUM_PROMPTS="${NUM_PROMPTS:-1}"
|
|
MAX_CONCURRENCY="${MAX_CONCURRENCY:-1}"
|
|
REQUEST_RATE="${REQUEST_RATE:-10000}"
|
|
|
|
# Padding added to target length when setting server max context/model length.
|
|
CONTEXT_PAD="${CONTEXT_PAD:-1024}"
|
|
|
|
# Candidate input lengths (tokens). Override via CANDIDATE_LENS env var.
|
|
if [[ -z "${CANDIDATE_LENS:-}" ]]; then
|
|
declare -a CANDIDATE_LENS=(
|
|
65536
|
|
131072
|
|
262144
|
|
393216
|
|
524288
|
|
655360
|
|
786432
|
|
917504
|
|
1048576
|
|
)
|
|
fi
|
|
|
|
# Server start scripts bundled with this experiment.
|
|
SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh"
|
|
VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh"
|