33 lines
939 B
Bash
33 lines
939 B
Bash
# OOM probe for 256k input + 4k output on DeepSeek-V4-Flash (H200).
|
|
# Tests whether SGLang and vLLM can serve this context length without OOM
|
|
# at concurrency=2 before adding it to the full long-context matrix.
|
|
|
|
EXPERIMENT="dsv4_h200_256k_4k_probe"
|
|
MODEL_NAME="DeepSeek-V4-Flash"
|
|
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
|
|
SERVED_MODEL_NAME="deepseek-v4-flash"
|
|
|
|
SGLANG_PORT="${SGLANG_PORT:-30006}"
|
|
VLLM_PORT="${VLLM_PORT:-30005}"
|
|
|
|
VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}"
|
|
VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}"
|
|
|
|
export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
|
|
TP=8
|
|
|
|
# 256k input + 4k output + padding -> use 270000.
|
|
MAX_MODEL_LEN=270000
|
|
MAX_NUM_SEQS=8
|
|
MAX_RUNNING=8
|
|
|
|
# "concurrency input_len output_len num_prompts"
|
|
declare -a SCENARIOS=(
|
|
"2 262144 4096 4"
|
|
)
|
|
|
|
VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"
|
|
|
|
SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh"
|
|
VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh"
|