be94b8e7ef
- Add experiments/dsv4_p800_256k_4k_probe for testing 256k input + 4k output on DeepSeek-V4-Flash-INT8 with Kunlun P800. - Add SGLANG_EXTRA_LAUNCH_ARGS support to scripts/common/server_docker.sh so probe can inject --context-length 270000. - Add w8a8_int8_baseline server mode without EAGLE speculative decoding, matching the long-context P800 tuning. - Switch dsv4_p800_sglang and the new probe to use w8a8_int8_baseline by default; original tuned mode remains available via SERVER_MODE=w8a8_int8.
33 lines
1.3 KiB
Bash
33 lines
1.3 KiB
Bash
# OOM probe for 256k input + 4k output on DeepSeek-V4-Flash-INT8 (Kunlun P800).
|
|
# Tests whether the P800 SGLang deployment can serve this context length
|
|
# without OOM at concurrency=2 before adding it to the full long-context matrix.
|
|
|
|
EXPERIMENT="${EXPERIMENT:-dsv4_p800_256k_4k_probe}"
|
|
MODEL_NAME="${MODEL_NAME:-DeepSeek-V4-Flash-INT8}"
|
|
MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}"
|
|
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-deepseek-v4-flash-int8}"
|
|
PORT="${PORT:-30000}"
|
|
BACKEND="${BACKEND:-sglang}"
|
|
ENGINE="${ENGINE:-sglang-xpu}"
|
|
DATASET="${DATASET:-random}"
|
|
DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
|
|
|
# Server launch mode for P800 SGLang.
|
|
# - "w8a8_int8" : default tuned config with EAGLE speculative decoding
|
|
# - "w8a8_int8_baseline": baseline config without speculative decoding,
|
|
# matching the long-context P800 tuning.
|
|
SERVER_MODE="${SERVER_MODE:-w8a8_int8_baseline}"
|
|
|
|
# Extra SGLang launch arguments for the 256k-context probe.
|
|
# 262144 input + 4096 output + padding -> use 270000.
|
|
SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:---context-length 270000 --max-running-requests 4}"
|
|
|
|
# Scenario list: "concurrency input_len output_len" (one per line).
|
|
SCENARIOS=(
|
|
"2 262144 4096"
|
|
)
|
|
|
|
# Number of prompts for the probe scenario.
|
|
NUM_PROMPTS="${NUM_PROMPTS:-4}"
|
|
WARMUP="${WARMUP:-0}"
|