Add P800 long-context matrix and tune benchmark/health settings
- Add experiments/dsv4_p800_long_context_matrix for 64k/128k/256k context grid on DeepSeek-V4-Flash-INT8 with Kunlun P800. - Change bench_client_docker.sh to use --concurrency instead of --max-concurrency to match the user's long-context benchmark script. - Add SGLANG_HEALTH_CHECK_TIMEOUT override to server_docker.sh for slow long-context server startups. - Tune dsv4_p800_256k_4k_probe with lower max-running-requests and mem-fraction-static to avoid OOM on P800.
This commit is contained in:
yy-fighting
parent
539b3bc539
commit
f6a45f1ed8
@ -20,7 +20,9 @@ SERVER_MODE="${SERVER_MODE:-w8a8_int8_baseline}"
|
||||
|
||||
# Extra SGLang launch arguments for the 256k-context probe.
|
||||
# 262144 input + 4096 output + padding -> use 270000.
|
||||
SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:---context-length 270000 --max-running-requests 4}"
|
||||
# max-running-requests=2 matches the probe concurrency; mem-fraction-static
|
||||
# limits the KV cache reservation to avoid OOM on P800.
|
||||
SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:---context-length 270000 --max-running-requests 2 --mem-fraction-static 0.85}"
|
||||
|
||||
# Scenario list: "concurrency input_len output_len" (one per line).
|
||||
SCENARIOS=(
|
||||
|
||||
@ -65,7 +65,7 @@ data["config"] = {
|
||||
"context_length": 270000,
|
||||
"server_mode": "w8a8_int8_baseline",
|
||||
"server_start_script": f"experiments/{data['metadata']['experiment']}/start_server.sh",
|
||||
"extra_launch_args": "--context-length 270000 --max-running-requests 4",
|
||||
"extra_launch_args": "--context-length 270000 --max-running-requests 2 --mem-fraction-static 0.85",
|
||||
"scenarios": ["2 262144 4096"],
|
||||
"num_prompts": 4
|
||||
}
|
||||
|
||||
52
experiments/dsv4_p800_long_context_matrix/README.md
Normal file
52
experiments/dsv4_p800_long_context_matrix/README.md
Normal file
@ -0,0 +1,52 @@
|
||||
# DSV4 P800 Long-Context Matrix
|
||||
|
||||
Kunlun P800 XPU + SGLang + `DeepSeek-V4-Flash-INT8` long-context matrix benchmark.
|
||||
|
||||
## What it does
|
||||
|
||||
Runs a grid of input/output lengths to measure SGLang long-context serving
|
||||
performance on P800:
|
||||
|
||||
- Input lengths: 64k, 128k, 256k
|
||||
- Output lengths: 256, 1k, 4k
|
||||
- Concurrency: 25 for 64k; 10, 5, 2 for 128k and 256k
|
||||
|
||||
Each context group (64k / 128k / 256k) is started with its own server instance
|
||||
using the appropriate `--context-length` and `--max-running-requests`.
|
||||
|
||||
## Quick Start
|
||||
|
||||
```bash
|
||||
# Run the full matrix (starts server per group, runs all scenarios, stops server)
|
||||
PLATFORM=kunlun_p800 bash experiments/dsv4_p800_long_context_matrix/run_bench.sh
|
||||
```
|
||||
|
||||
Results land in `experiments/dsv4_p800_long_context_matrix/results/<RUN_ID>/`.
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `config.env` or override via environment variables:
|
||||
|
||||
```bash
|
||||
# Run with the tuned EAGLE speculative-decoding config instead of the baseline
|
||||
SERVER_MODE=w8a8_int8 PLATFORM=kunlun_p800 \
|
||||
bash experiments/dsv4_p800_long_context_matrix/run_bench.sh
|
||||
|
||||
# Reduce concurrency for the 256k group if OOM occurs
|
||||
CONTEXT_GROUPS=("64k 80000 25" "128k 140000 10" "256k 270000 5") \
|
||||
PLATFORM=kunlun_p800 \
|
||||
bash experiments/dsv4_p800_long_context_matrix/run_bench.sh
|
||||
```
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|---|---|
|
||||
| `config.env` | Experiment-level configuration (model, scenarios, context groups) |
|
||||
| `start_server.sh` | Start the P800 SGLang Docker container for a context group |
|
||||
| `run_bench.sh` | Orchestrator: server per group → warmup → scenarios → parse |
|
||||
|
||||
## Platform
|
||||
|
||||
This experiment targets `platforms/kunlun_p800.env` and uses
|
||||
`scripts/common/server_docker.sh` / `scripts/common/bench_client_docker.sh`.
|
||||
60
experiments/dsv4_p800_long_context_matrix/config.env
Normal file
60
experiments/dsv4_p800_long_context_matrix/config.env
Normal file
@ -0,0 +1,60 @@
|
||||
# Long-context matrix for SGLang on DeepSeek-V4-Flash-INT8 (Kunlun P800, TP=8).
|
||||
# Input lengths: 64k, 128k, 256k
|
||||
# Output lengths: 256, 1k, 4k
|
||||
# Concurrency: 25 for 64k; 10, 5, 2 for 128k and 256k
|
||||
|
||||
EXPERIMENT="${EXPERIMENT:-dsv4_p800_long_context_matrix}"
|
||||
MODEL_NAME="${MODEL_NAME:-DeepSeek-V4-Flash-INT8}"
|
||||
MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}"
|
||||
SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-deepseek-v4-flash-int8}"
|
||||
PORT="${PORT:-30000}"
|
||||
BACKEND="${BACKEND:-sglang}"
|
||||
ENGINE="${ENGINE:-sglang-xpu}"
|
||||
DATASET="${DATASET:-random}"
|
||||
DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}"
|
||||
|
||||
# Server launch mode for P800 SGLang.
|
||||
# - "w8a8_int8" : DeepSeek-V4-Flash-INT8 checkpoint with EAGLE speculative decoding
|
||||
# - "w8a8_int8_baseline": DeepSeek-V4-Flash-INT8 checkpoint without speculative decoding
|
||||
SERVER_MODE="${SERVER_MODE:-w8a8_int8_baseline}"
|
||||
|
||||
# Per-context server settings. Each group is run with its own server start.
|
||||
# Format: "input_label context_length max_running"
|
||||
declare -a CONTEXT_GROUPS=(
|
||||
"64k 80000 25"
|
||||
"128k 140000 10"
|
||||
"256k 270000 10"
|
||||
)
|
||||
|
||||
# Scenarios per group: "concurrency input_len output_len num_prompts"
|
||||
declare -a SCENARIOS_64K=(
|
||||
"25 65536 256 50"
|
||||
"25 65536 1024 50"
|
||||
"25 65536 4096 50"
|
||||
)
|
||||
|
||||
declare -a SCENARIOS_128K=(
|
||||
"10 131072 256 20"
|
||||
"5 131072 256 10"
|
||||
"2 131072 256 4"
|
||||
"10 131072 1024 20"
|
||||
"5 131072 1024 10"
|
||||
"2 131072 1024 4"
|
||||
"10 131072 4096 20"
|
||||
"5 131072 4096 10"
|
||||
"2 131072 4096 4"
|
||||
)
|
||||
|
||||
declare -a SCENARIOS_256K=(
|
||||
"10 262144 256 20"
|
||||
"5 262144 256 10"
|
||||
"2 262144 256 4"
|
||||
"10 262144 1024 20"
|
||||
"5 262144 1024 10"
|
||||
"2 262144 1024 4"
|
||||
"10 262144 4096 20"
|
||||
"5 262144 4096 10"
|
||||
"2 262144 4096 4"
|
||||
)
|
||||
|
||||
WARMUP="${WARMUP:-0}"
|
||||
255
experiments/dsv4_p800_long_context_matrix/run_bench.sh
Executable file
255
experiments/dsv4_p800_long_context_matrix/run_bench.sh
Executable file
@ -0,0 +1,255 @@
|
||||
#!/usr/bin/env bash
|
||||
# Long-context matrix for SGLang on DeepSeek-V4-Flash-INT8 (Kunlun P800).
|
||||
set -Eeuo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/bench_client_docker.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
|
||||
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
||||
RESULT_BASE="${SCRIPT_DIR}/results"
|
||||
|
||||
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
|
||||
mkdir -p "$log_dir_global"
|
||||
log_init "${log_dir_global}/orchestrator.log"
|
||||
|
||||
log "experiment=${EXPERIMENT_NAME}"
|
||||
log "run_id=${RUN_ID}"
|
||||
log "platform=${PLATFORM}"
|
||||
log "hardware=${HARDWARE}"
|
||||
log "model=${MODEL_PATH}"
|
||||
log "server_mode=${SERVER_MODE}"
|
||||
log "groups=${#CONTEXT_GROUPS[@]}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
is_server_healthy() {
|
||||
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1
|
||||
}
|
||||
|
||||
start_server() {
|
||||
local context_length="$1"
|
||||
local max_running="$2"
|
||||
local extra_args="--context-length ${context_length} --max-running-requests ${max_running} --mem-fraction-static 0.85"
|
||||
export SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:-${extra_args}}"
|
||||
|
||||
log "starting server (context_length=${context_length}, max_running=${max_running}, mode=${SERVER_MODE})"
|
||||
docker_server_start "${MODEL_PATH}" "${PORT}" "${log_dir_global}/server_${context_length}.outer.log" "${SERVER_MODE}" "${SGLANG_EXTRA_LAUNCH_ARGS}"
|
||||
}
|
||||
|
||||
stop_server() {
|
||||
docker_server_stop
|
||||
}
|
||||
|
||||
run_warmup() {
|
||||
local raw_dir="$1"
|
||||
local phase_log_dir="$2"
|
||||
local input_len="$3"
|
||||
local output_len="${4:-256}"
|
||||
|
||||
log "warming up (input=${input_len}, output=${output_len}, num=1)"
|
||||
|
||||
local tmp_warmup_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_warmup_1_${input_len}_${output_len}.jsonl"
|
||||
local host_warmup_file="${raw_dir}/${BACKEND}_warmup_$(date '+%m%d')_1_${input_len}_${output_len}.jsonl"
|
||||
local detail_log="${phase_log_dir}/warmup_${input_len}_${output_len}.log"
|
||||
|
||||
if WARMUP=0 run_random_case \
|
||||
"$BACKEND" "$PORT" "$MODEL_PATH" "$tmp_warmup_file" \
|
||||
1 "$input_len" "$output_len" 1 "${DATASET_PATH:-}" \
|
||||
> "$detail_log" 2>&1; then
|
||||
docker cp "${CONTAINER_NAME}:${tmp_warmup_file}" "$host_warmup_file" 2>/dev/null || true
|
||||
log "warmup completed"
|
||||
else
|
||||
log "WARNING: warmup failed; continuing with group"
|
||||
fi
|
||||
}
|
||||
|
||||
scenario_already_completed() {
|
||||
local output_file="$1"
|
||||
local expected="$2"
|
||||
[[ -s "$output_file" ]] || return 1
|
||||
local completed
|
||||
completed="$(python3 -c "
|
||||
import json, sys
|
||||
path = sys.argv[1]
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
data = json.loads(line)
|
||||
print(data.get('completed', 0))
|
||||
break
|
||||
except Exception:
|
||||
print(0)
|
||||
" "$output_file")"
|
||||
[[ "${completed:-0}" -ge "$expected" ]]
|
||||
}
|
||||
|
||||
run_group() {
|
||||
local group_label="$1"
|
||||
local context_length="$2"
|
||||
local max_running="$3"
|
||||
local -n scenarios_ref="$4"
|
||||
|
||||
local result_root="${RESULT_BASE}/${RUN_ID}/${BACKEND}"
|
||||
local raw_dir="${result_root}/raw_outputs"
|
||||
local phase_log_dir="${result_root}/logs"
|
||||
|
||||
mkdir -p "$raw_dir" "$phase_log_dir"
|
||||
|
||||
log "===== ${BACKEND} ${group_label} START (context_length=${context_length}) ====="
|
||||
|
||||
stop_server
|
||||
if ! start_server "$context_length" "$max_running"; then
|
||||
log "ERROR: server failed to start for ${group_label}; skipping group"
|
||||
stop_server
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Warmup with the shortest output for this input length to save time.
|
||||
local warmup_input="${scenarios_ref[0]}"
|
||||
local warmup_input_len
|
||||
warmup_input_len="$(echo "$warmup_input" | awk '{print $2}')"
|
||||
run_warmup "$raw_dir" "$phase_log_dir" "$warmup_input_len" 256
|
||||
|
||||
for scenario in "${scenarios_ref[@]}"; do
|
||||
# Allow SCENARIOS to be passed as a string like "(25 65536 256 50)" from env.
|
||||
scenario="${scenario//[()]/}"
|
||||
read -r concurrency input_len output_len num_prompts <<< "$scenario"
|
||||
|
||||
tmp_output_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
||||
output_file="${raw_dir}/${BACKEND}_${group_label}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
||||
detail_log="${phase_log_dir}/${BACKEND}_${group_label}_c${concurrency}_i${input_len}_o${output_len}.log"
|
||||
|
||||
if scenario_already_completed "$output_file" "$num_prompts"; then
|
||||
log "skipping already-completed ${BACKEND} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len}"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "running ${BACKEND} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
|
||||
|
||||
if run_random_case \
|
||||
"$BACKEND" "$PORT" "$MODEL_PATH" "$tmp_output_file" \
|
||||
"$concurrency" "$input_len" "$output_len" "$num_prompts" \
|
||||
"${DATASET_PATH:-}" \
|
||||
> "$detail_log" 2>&1; then
|
||||
docker cp "${CONTAINER_NAME}:${tmp_output_file}" "$output_file" || {
|
||||
log "ERROR: failed to copy ${tmp_output_file} from container"
|
||||
continue
|
||||
}
|
||||
log "finished ${BACKEND} ${group_label} scenario: output=${output_file}"
|
||||
else
|
||||
log "ERROR: ${BACKEND} ${group_label} scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
|
||||
continue
|
||||
fi
|
||||
done
|
||||
|
||||
stop_server
|
||||
log "===== ${BACKEND} ${group_label} DONE ====="
|
||||
}
|
||||
|
||||
parse_backend() {
|
||||
local result_root="${RESULT_BASE}/${RUN_ID}/${BACKEND}"
|
||||
log "parsing ${BACKEND} results in ${result_root}"
|
||||
python3 "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$BACKEND" \
|
||||
>> "${result_root}/logs/parse.log" 2>&1 || {
|
||||
log "WARNING: parser failed for ${BACKEND}; see ${result_root}/logs/parse.log"
|
||||
}
|
||||
}
|
||||
|
||||
write_backend_metadata() {
|
||||
local result_root="${RESULT_BASE}/${RUN_ID}/${BACKEND}"
|
||||
ensure_result_root "$result_root"
|
||||
local meta_json="${result_root}/results.json"
|
||||
|
||||
write_metadata_json \
|
||||
"$meta_json" \
|
||||
"${EXPERIMENT_NAME}_${BACKEND}" \
|
||||
"$RUN_ID" \
|
||||
"$MODEL_PATH" \
|
||||
"$BACKEND" \
|
||||
"$ENGINE" \
|
||||
"$HARDWARE" \
|
||||
"$ACCELERATOR" \
|
||||
"$CHIP" \
|
||||
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
||||
"" \
|
||||
"Kunlun P800 long-context matrix ${BACKEND} TP=8 for DeepSeek-V4-Flash-INT8"
|
||||
|
||||
# Build groups JSON from CONTEXT_GROUPS.
|
||||
local groups_json="["
|
||||
local first=1
|
||||
for group in "${CONTEXT_GROUPS[@]}"; do
|
||||
read -r label context_length max_running <<< "$group"
|
||||
if [[ "$first" -eq 1 ]]; then
|
||||
first=0
|
||||
else
|
||||
groups_json+=","
|
||||
fi
|
||||
groups_json+="{\"label\":\"${label}\",\"context_length\":${context_length},\"max_running\":${max_running}}"
|
||||
done
|
||||
groups_json+="]"
|
||||
|
||||
# Embed config.
|
||||
python3 - "$meta_json" "$groups_json" <<'PY' >/dev/null
|
||||
import json
|
||||
import sys
|
||||
|
||||
path = sys.argv[1]
|
||||
groups = json.loads(sys.argv[2])
|
||||
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
|
||||
data["config"] = {
|
||||
"tp": 8,
|
||||
"xpu_visible_devices": "0,1,2,3,4,5,6,7",
|
||||
"backend": data["metadata"]["backend"],
|
||||
"engine": data["metadata"]["engine"],
|
||||
"server_mode": "w8a8_int8_baseline",
|
||||
"server_start_script": f"experiments/{data['metadata']['experiment']}/start_server.sh",
|
||||
"groups": groups
|
||||
}
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
PY
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Cleanup any leftovers.
|
||||
stop_server
|
||||
|
||||
write_backend_metadata
|
||||
|
||||
for group in "${CONTEXT_GROUPS[@]}"; do
|
||||
read -r group_label context_length max_running <<< "$group"
|
||||
|
||||
# Resolve scenario array for this group.
|
||||
scenario_array_name="SCENARIOS_${group_label^^}"
|
||||
if [[ -z "${!scenario_array_name:-}" ]]; then
|
||||
log "WARNING: no scenarios defined for group ${group_label}"
|
||||
continue
|
||||
fi
|
||||
|
||||
run_group "$group_label" "$context_length" "$max_running" "$scenario_array_name"
|
||||
done
|
||||
|
||||
parse_backend
|
||||
|
||||
log "all results saved to ${RESULT_BASE}/${RUN_ID}"
|
||||
28
experiments/dsv4_p800_long_context_matrix/start_server.sh
Executable file
28
experiments/dsv4_p800_long_context_matrix/start_server.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start SGLang server for the P800 long-context matrix.
|
||||
# Usage: start_server.sh <context_length> <max_running>
|
||||
set -Eeuo pipefail
|
||||
|
||||
CONTEXT_LENGTH="${1:-80000}"
|
||||
MAX_RUNNING="${2:-64}"
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh"
|
||||
# shellcheck source=/dev/null
|
||||
source "${SCRIPT_DIR}/config.env"
|
||||
|
||||
log_init "${RESULT_ROOT:-/tmp/${EXPERIMENT_NAME}}/logs/start_server.log"
|
||||
log "starting server for ${EXPERIMENT_NAME}"
|
||||
log "model: ${MODEL_PATH}"
|
||||
log "port: ${PORT}"
|
||||
log "server mode: ${SERVER_MODE}"
|
||||
log "context length: ${CONTEXT_LENGTH}"
|
||||
log "max running requests: ${MAX_RUNNING}"
|
||||
|
||||
extra_args="--context-length ${CONTEXT_LENGTH} --max-running-requests ${MAX_RUNNING} --mem-fraction-static 0.85"
|
||||
export SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:-${extra_args}}"
|
||||
|
||||
docker_server_start "${MODEL_PATH}" "${PORT}" "${LOG_DIR}/server.outer.log" "${SERVER_MODE}" "${SGLANG_EXTRA_LAUNCH_ARGS}"
|
||||
@ -63,7 +63,7 @@ run_random_case() {
|
||||
--random-input-len "$input_len"
|
||||
--random-output-len "$output_len"
|
||||
--num-prompts "$num_prompts"
|
||||
--max-concurrency "$concurrency"
|
||||
--concurrency "$concurrency"
|
||||
--warmup-requests "$warmup"
|
||||
--output-file "$output_file"
|
||||
--output-details
|
||||
|
||||
@ -206,11 +206,12 @@ EOF
|
||||
bash -c "echo '${server_cmd_b64}' | base64 -d | bash" \
|
||||
>> "${server_log}" 2>&1
|
||||
|
||||
log "container ${container} started, waiting for health"
|
||||
if health_check 127.0.0.1 "$port" 600; then
|
||||
local health_wait="${SGLANG_HEALTH_CHECK_TIMEOUT:-600}"
|
||||
log "container ${container} started, waiting for health (timeout=${health_wait}s)"
|
||||
if health_check 127.0.0.1 "$port" "$health_wait"; then
|
||||
log "container ${container} is healthy"
|
||||
else
|
||||
log "ERROR: container ${container} failed health check"
|
||||
log "ERROR: container ${container} failed health check after ${health_wait}s"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user