f6a45f1ed8
- Add experiments/dsv4_p800_long_context_matrix for 64k/128k/256k context grid on DeepSeek-V4-Flash-INT8 with Kunlun P800. - Change bench_client_docker.sh to use --concurrency instead of --max-concurrency to match the user's long-context benchmark script. - Add SGLANG_HEALTH_CHECK_TIMEOUT override to server_docker.sh for slow long-context server startups. - Tune dsv4_p800_256k_4k_probe with lower max-running-requests and mem-fraction-static to avoid OOM on P800.
61 lines
1.7 KiB
Bash
61 lines
1.7 KiB
Bash
# Long-context matrix for SGLang on DeepSeek-V4-Flash-INT8 (Kunlun P800, TP=8).
|
|
# Input lengths: 64k, 128k, 256k
|
|
# Output lengths: 256, 1k, 4k
|
|
# Concurrency: 25 for 64k; 10, 5, 2 for 128k and 256k
|
|
|
|
EXPERIMENT="${EXPERIMENT:-dsv4_p800_long_context_matrix}"
|
|
MODEL_NAME="${MODEL_NAME:-DeepSeek-V4-Flash-INT8}"
|
|
MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}"
|
|
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-deepseek-v4-flash-int8}"
|
|
PORT="${PORT:-30000}"
|
|
BACKEND="${BACKEND:-sglang}"
|
|
ENGINE="${ENGINE:-sglang-xpu}"
|
|
DATASET="${DATASET:-random}"
|
|
DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
|
|
|
# Server launch mode for P800 SGLang.
|
|
# - "w8a8_int8" : DeepSeek-V4-Flash-INT8 checkpoint with EAGLE speculative decoding
|
|
# - "w8a8_int8_baseline": DeepSeek-V4-Flash-INT8 checkpoint without speculative decoding
|
|
SERVER_MODE="${SERVER_MODE:-w8a8_int8_baseline}"
|
|
|
|
# Per-context server settings. Each group is run with its own server start.
|
|
# Format: "input_label context_length max_running"
|
|
declare -a CONTEXT_GROUPS=(
|
|
"64k 80000 25"
|
|
"128k 140000 10"
|
|
"256k 270000 10"
|
|
)
|
|
|
|
# Scenarios per group: "concurrency input_len output_len num_prompts"
|
|
declare -a SCENARIOS_64K=(
|
|
"25 65536 256 50"
|
|
"25 65536 1024 50"
|
|
"25 65536 4096 50"
|
|
)
|
|
|
|
declare -a SCENARIOS_128K=(
|
|
"10 131072 256 20"
|
|
"5 131072 256 10"
|
|
"2 131072 256 4"
|
|
"10 131072 1024 20"
|
|
"5 131072 1024 10"
|
|
"2 131072 1024 4"
|
|
"10 131072 4096 20"
|
|
"5 131072 4096 10"
|
|
"2 131072 4096 4"
|
|
)
|
|
|
|
declare -a SCENARIOS_256K=(
|
|
"10 262144 256 20"
|
|
"5 262144 256 10"
|
|
"2 262144 256 4"
|
|
"10 262144 1024 20"
|
|
"5 262144 1024 10"
|
|
"2 262144 1024 4"
|
|
"10 262144 4096 20"
|
|
"5 262144 4096 10"
|
|
"2 262144 4096 4"
|
|
)
|
|
|
|
WARMUP="${WARMUP:-0}"
|