0fdd3749c7
- New experiments/p800/dsv4_p800_sglang_tp_dp_matrix: TP8/DP1, TP4/DP2, TP2/DP4 matrix with smoke results; TP2/DP4 documents the weight-loading OOM root cause (274 GiB INT8 weights sharded only across TP group). - Launch args drop --ep-size/--chunked-prefill-size/--max-prefill-tokens/ --max-running-requests; experts fall back to TP sharding. - Move dsv4_p800_256k_4k_probe under experiments/p800/. - scripts/common: jq-free parsing fixes in adaptive_bench_lib.sh and parse_backend.py. - .gitignore: cover raw_outputs under nested platform experiment layout.
86 lines
3.6 KiB
Bash
86 lines
3.6 KiB
Bash
# TP×DP matrix experiment for DeepSeek-V4-Flash-INT8 on Kunlun P800 (8 XPUs)
|
||
# using SGLang-XPU. Tests SGLang with three parallel configurations:
|
||
# TP=2, DP=4 -> 2 XPUs per replica, 4 replicas
|
||
# TP=4, DP=2 -> 4 XPUs per replica, 2 replicas
|
||
# TP=8, DP=1 -> 8 XPUs, no data parallelism
|
||
|
||
EXPERIMENT="dsv4_p800_sglang_tp_dp_matrix"
|
||
MODEL_NAME="DeepSeek-V4-Flash-INT8"
|
||
MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}"
|
||
SERVED_MODEL_NAME="deepseek-v4-flash-int8"
|
||
|
||
SGLANG_PORT="${SGLANG_PORT:-30014}"
|
||
|
||
# Dedicated container name so this experiment never touches the
|
||
# `sglang-dsv4-flash` container used by the other P800 experiments.
|
||
CONTAINER_NAME="sglang-dsv4-flash-tpdp"
|
||
|
||
# Python interpreter inside the P800 SGLang container (used to run the
|
||
# benchmark client via `docker exec`).
|
||
CONTAINER_PYTHON="${CONTAINER_PYTHON:-/root/miniconda/envs/python310_torch25_cuda/bin/python}"
|
||
|
||
export XPU_VISIBLE_DEVICES="${XPU_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||
|
||
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
|
||
# directory under this experiment so the benchmark is self-contained.
|
||
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||
|
||
# Parallel configurations to test. Format: "TP DP"
|
||
# Note: TP=2/DP=4 fails at startup with a weight-loading OOM. Root cause:
|
||
# expert weights are sharded only across the TP group (no --ep-size is
|
||
# passed, so experts fall back to TP sharding), and DP replicas do not
|
||
# share expert shards. The INT8 model holds ~264 GiB of routed-expert
|
||
# weights, so TP=2 needs ~137 GiB per XPU (>96 GiB). TP=2/DP=4
|
||
# is kept in the default list so the matrix records the failure and moves on.
|
||
# Override with a space-separated list of comma pairs, e.g.
|
||
# PARALLEL_CONFIGS_STR="8,1" to run only TP=8/DP=1.
|
||
if [[ -n "${PARALLEL_CONFIGS_STR:-}" ]]; then
|
||
declare -a PARALLEL_CONFIGS=()
|
||
for pair in $PARALLEL_CONFIGS_STR; do
|
||
PARALLEL_CONFIGS+=("${pair//,/ }")
|
||
done
|
||
else
|
||
declare -a PARALLEL_CONFIGS=(
|
||
"2 4"
|
||
"4 2"
|
||
"8 1"
|
||
)
|
||
fi
|
||
|
||
# SGLang server settings. P800 INT8 supports at most ~131k input tokens, so
|
||
# the context length is capped well below the H20 1M setting.
|
||
CONTEXT_LENGTH="${CONTEXT_LENGTH:-140000}"
|
||
MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.8}"
|
||
|
||
DOCKER_IMAGE="${DOCKER_IMAGE:-iregistry.baidu-int.com/xpu/sglang-p800-pd-disagg-0510:20260511_4202}"
|
||
|
||
# Random dataset mounted into the container at /workspace/dummy_sharegpt.json.
|
||
# DATASET_PATH is the host copy (used by the adaptive search for sanity
|
||
# checks); the benchmark client always references CONTAINER_DATASET_PATH.
|
||
PATCH_ROOT="${PATCH_ROOT:-/data1/yy/sskj/platforms/patches/kunlun_p800}"
|
||
DATASET_PATH="${DATASET_PATH:-${PATCH_ROOT}/dummy_sharegpt.json}"
|
||
CONTAINER_DATASET_PATH="${CONTAINER_DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
||
|
||
# Matrix and concurrency rules are defined in matrix.json by default.
|
||
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||
|
||
# Sampling density for concurrency (kept for parity with the H20 experiment;
|
||
# generate_scenarios.py currently samples the low/high endpoints only).
|
||
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
||
|
||
# Per-scenario timeout to avoid hangs (seconds).
|
||
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||
|
||
# XPU memory sampling interval (seconds).
|
||
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||
|
||
# Dry-run mode: if 1, only log the server args and scenario plan without
|
||
# starting any server or sending requests.
|
||
DRY_RUN="${DRY_RUN:-0}"
|
||
|
||
# Per-config scenario limit for quick smoke tests. 0 = run all generated
|
||
# scenarios.
|
||
GRID_LIMIT="${GRID_LIMIT:-0}"
|