f6a45f1ed8
- Add experiments/dsv4_p800_long_context_matrix for 64k/128k/256k context grid on DeepSeek-V4-Flash-INT8 with Kunlun P800. - Change bench_client_docker.sh to use --concurrency instead of --max-concurrency to match the user's long-context benchmark script. - Add SGLANG_HEALTH_CHECK_TIMEOUT override to server_docker.sh for slow long-context server startups. - Tune dsv4_p800_256k_4k_probe with lower max-running-requests and mem-fraction-static to avoid OOM on P800.
35 lines
1.5 KiB
Bash
35 lines
1.5 KiB
Bash
# OOM probe for 256k input + 4k output on DeepSeek-V4-Flash-INT8 (Kunlun P800).
|
|
# Tests whether the P800 SGLang deployment can serve this context length
|
|
# without OOM at concurrency=2 before adding it to the full long-context matrix.
|
|
|
|
EXPERIMENT="${EXPERIMENT:-dsv4_p800_256k_4k_probe}"
|
|
MODEL_NAME="${MODEL_NAME:-DeepSeek-V4-Flash-INT8}"
|
|
MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}"
|
|
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-deepseek-v4-flash-int8}"
|
|
PORT="${PORT:-30000}"
|
|
BACKEND="${BACKEND:-sglang}"
|
|
ENGINE="${ENGINE:-sglang-xpu}"
|
|
DATASET="${DATASET:-random}"
|
|
DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
|
|
|
# Server launch mode for P800 SGLang.
|
|
# - "w8a8_int8" : default tuned config with EAGLE speculative decoding
|
|
# - "w8a8_int8_baseline": baseline config without speculative decoding,
|
|
# matching the long-context P800 tuning.
|
|
SERVER_MODE="${SERVER_MODE:-w8a8_int8_baseline}"
|
|
|
|
# Extra SGLang launch arguments for the 256k-context probe.
|
|
# 262144 input + 4096 output + padding -> use 270000.
|
|
# max-running-requests=2 matches the probe concurrency; mem-fraction-static
|
|
# limits the KV cache reservation to avoid OOM on P800.
|
|
SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:---context-length 270000 --max-running-requests 2 --mem-fraction-static 0.85}"
|
|
|
|
# Scenario list: "concurrency input_len output_len" (one per line).
|
|
SCENARIOS=(
|
|
"2 262144 4096"
|
|
)
|
|
|
|
# Number of prompts for the probe scenario.
|
|
NUM_PROMPTS="${NUM_PROMPTS:-4}"
|
|
WARMUP="${WARMUP:-0}"
|