a1da95ef44
- P800 INT8 cannot sustain >1 concurrent long-context request without OOM; set all scenarios to concurrency=1. - Keep max_running=2 because max_running=1 triggers req_to_token_pool allocation failures in this image. - Reduce 64k max_context_len to 72000 (65536 + 4096 + pad). - Make run_bench.sh resilient to container death: detect dead container and skip remaining scenarios in the group instead of crashing the orchestrator. - Update README to document the concurrency limitation.
42 lines
1.5 KiB
Bash
42 lines
1.5 KiB
Bash
# Long-context matrix for SGLang-XPU on DeepSeek-V4-Flash-INT8 (P800, TP=8).
|
|
# Input lengths: 64k, 128k (P800 INT8 max input length is ~131072 tokens).
|
|
# Output lengths: 256, 1k, 4k
|
|
# Concurrency: P800 INT8 can only sustain 1 concurrent long-context request
|
|
# without OOM, so all scenarios use concurrency=1. max_running is kept at 2
|
|
# because max_running=1 triggers req_to_token_pool allocation failures.
|
|
|
|
EXPERIMENT="dsv4_p800_long_context_matrix"
|
|
MODEL_NAME="DeepSeek-V4-Flash-INT8"
|
|
MODEL_PATH="/data1/models/DeepSeek-V4-Flash-INT8"
|
|
SERVED_MODEL_NAME="deepseek-v4-flash-int8"
|
|
|
|
PORT="${PORT:-30013}"
|
|
|
|
# Padding added to the longest sequence length when setting server context length.
|
|
CONTEXT_PAD="${CONTEXT_PAD:-1024}"
|
|
|
|
# Per-context server settings. Each group is run with its own server start.
|
|
# Format: "input_label max_context_len max_running"
|
|
declare -a CONTEXT_GROUPS=(
|
|
"64k 72000 2"
|
|
"128k 140000 2"
|
|
)
|
|
|
|
# Scenarios per group: "concurrency input_len output_len num_prompts"
|
|
declare -a SCENARIOS_64K=(
|
|
"1 65536 256 10"
|
|
"1 65536 1024 10"
|
|
"1 65536 4096 10"
|
|
)
|
|
|
|
declare -a SCENARIOS_128K=(
|
|
"1 131072 256 5"
|
|
"1 131072 1024 5"
|
|
"1 131072 4096 5"
|
|
)
|
|
|
|
CONTAINER_NAME="${CONTAINER_NAME:-sglang-dsv4-flash}"
|
|
CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}"
|
|
DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}"
|
|
PATCH_ROOT="${PATCH_ROOT:-${ROOT_DIR}/platforms/patches/kunlun_p800}"
|