diff --git a/experiments/dsv4_p800_256k_4k_probe/config.env b/experiments/dsv4_p800_256k_4k_probe/config.env index de8af0a..d03041a 100644 --- a/experiments/dsv4_p800_256k_4k_probe/config.env +++ b/experiments/dsv4_p800_256k_4k_probe/config.env @@ -20,7 +20,9 @@ SERVER_MODE="${SERVER_MODE:-w8a8_int8_baseline}" # Extra SGLang launch arguments for the 256k-context probe. # 262144 input + 4096 output + padding -> use 270000. -SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:---context-length 270000 --max-running-requests 4}" +# max-running-requests=2 matches the probe concurrency; mem-fraction-static +# limits the KV cache reservation to avoid OOM on P800. +SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:---context-length 270000 --max-running-requests 2 --mem-fraction-static 0.85}" # Scenario list: "concurrency input_len output_len" (one per line). SCENARIOS=( diff --git a/experiments/dsv4_p800_256k_4k_probe/run_bench.sh b/experiments/dsv4_p800_256k_4k_probe/run_bench.sh index c37a0fa..fdf10e5 100755 --- a/experiments/dsv4_p800_256k_4k_probe/run_bench.sh +++ b/experiments/dsv4_p800_256k_4k_probe/run_bench.sh @@ -65,7 +65,7 @@ data["config"] = { "context_length": 270000, "server_mode": "w8a8_int8_baseline", "server_start_script": f"experiments/{data['metadata']['experiment']}/start_server.sh", - "extra_launch_args": "--context-length 270000 --max-running-requests 4", + "extra_launch_args": "--context-length 270000 --max-running-requests 2 --mem-fraction-static 0.85", "scenarios": ["2 262144 4096"], "num_prompts": 4 } diff --git a/experiments/dsv4_p800_long_context_matrix/README.md b/experiments/dsv4_p800_long_context_matrix/README.md new file mode 100644 index 0000000..d5b7c20 --- /dev/null +++ b/experiments/dsv4_p800_long_context_matrix/README.md @@ -0,0 +1,52 @@ +# DSV4 P800 Long-Context Matrix + +Kunlun P800 XPU + SGLang + `DeepSeek-V4-Flash-INT8` long-context matrix benchmark. + +## What it does + +Runs a grid of input/output lengths to measure SGLang long-context serving +performance on P800: + +- Input lengths: 64k, 128k, 256k +- Output lengths: 256, 1k, 4k +- Concurrency: 25 for 64k; 10, 5, 2 for 128k and 256k + +Each context group (64k / 128k / 256k) is started with its own server instance +using the appropriate `--context-length` and `--max-running-requests`. + +## Quick Start + +```bash +# Run the full matrix (starts server per group, runs all scenarios, stops server) +PLATFORM=kunlun_p800 bash experiments/dsv4_p800_long_context_matrix/run_bench.sh +``` + +Results land in `experiments/dsv4_p800_long_context_matrix/results//`. + +## Configuration + +Edit `config.env` or override via environment variables: + +```bash +# Run with the tuned EAGLE speculative-decoding config instead of the baseline +SERVER_MODE=w8a8_int8 PLATFORM=kunlun_p800 \ +bash experiments/dsv4_p800_long_context_matrix/run_bench.sh + +# Reduce concurrency for the 256k group if OOM occurs +CONTEXT_GROUPS=("64k 80000 25" "128k 140000 10" "256k 270000 5") \ +PLATFORM=kunlun_p800 \ +bash experiments/dsv4_p800_long_context_matrix/run_bench.sh +``` + +## Files + +| File | Purpose | +|---|---| +| `config.env` | Experiment-level configuration (model, scenarios, context groups) | +| `start_server.sh` | Start the P800 SGLang Docker container for a context group | +| `run_bench.sh` | Orchestrator: server per group → warmup → scenarios → parse | + +## Platform + +This experiment targets `platforms/kunlun_p800.env` and uses +`scripts/common/server_docker.sh` / `scripts/common/bench_client_docker.sh`. diff --git a/experiments/dsv4_p800_long_context_matrix/config.env b/experiments/dsv4_p800_long_context_matrix/config.env new file mode 100644 index 0000000..f836732 --- /dev/null +++ b/experiments/dsv4_p800_long_context_matrix/config.env @@ -0,0 +1,60 @@ +# Long-context matrix for SGLang on DeepSeek-V4-Flash-INT8 (Kunlun P800, TP=8). +# Input lengths: 64k, 128k, 256k +# Output lengths: 256, 1k, 4k +# Concurrency: 25 for 64k; 10, 5, 2 for 128k and 256k + +EXPERIMENT="${EXPERIMENT:-dsv4_p800_long_context_matrix}" +MODEL_NAME="${MODEL_NAME:-DeepSeek-V4-Flash-INT8}" +MODEL_PATH="${MODEL_PATH:-/data1/models/DeepSeek-V4-Flash-INT8}" +SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-deepseek-v4-flash-int8}" +PORT="${PORT:-30000}" +BACKEND="${BACKEND:-sglang}" +ENGINE="${ENGINE:-sglang-xpu}" +DATASET="${DATASET:-random}" +DATASET_PATH="${DATASET_PATH:-/workspace/dummy_sharegpt.json}" + +# Server launch mode for P800 SGLang. +# - "w8a8_int8" : DeepSeek-V4-Flash-INT8 checkpoint with EAGLE speculative decoding +# - "w8a8_int8_baseline": DeepSeek-V4-Flash-INT8 checkpoint without speculative decoding +SERVER_MODE="${SERVER_MODE:-w8a8_int8_baseline}" + +# Per-context server settings. Each group is run with its own server start. +# Format: "input_label context_length max_running" +declare -a CONTEXT_GROUPS=( + "64k 80000 25" + "128k 140000 10" + "256k 270000 10" +) + +# Scenarios per group: "concurrency input_len output_len num_prompts" +declare -a SCENARIOS_64K=( + "25 65536 256 50" + "25 65536 1024 50" + "25 65536 4096 50" +) + +declare -a SCENARIOS_128K=( + "10 131072 256 20" + "5 131072 256 10" + "2 131072 256 4" + "10 131072 1024 20" + "5 131072 1024 10" + "2 131072 1024 4" + "10 131072 4096 20" + "5 131072 4096 10" + "2 131072 4096 4" +) + +declare -a SCENARIOS_256K=( + "10 262144 256 20" + "5 262144 256 10" + "2 262144 256 4" + "10 262144 1024 20" + "5 262144 1024 10" + "2 262144 1024 4" + "10 262144 4096 20" + "5 262144 4096 10" + "2 262144 4096 4" +) + +WARMUP="${WARMUP:-0}" diff --git a/experiments/dsv4_p800_long_context_matrix/run_bench.sh b/experiments/dsv4_p800_long_context_matrix/run_bench.sh new file mode 100755 index 0000000..95df3d3 --- /dev/null +++ b/experiments/dsv4_p800_long_context_matrix/run_bench.sh @@ -0,0 +1,255 @@ +#!/usr/bin/env bash +# Long-context matrix for SGLang on DeepSeek-V4-Flash-INT8 (Kunlun P800). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/bench_client_docker.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "platform=${PLATFORM}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "server_mode=${SERVER_MODE}" +log "groups=${#CONTEXT_GROUPS[@]}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1 +} + +start_server() { + local context_length="$1" + local max_running="$2" + local extra_args="--context-length ${context_length} --max-running-requests ${max_running} --mem-fraction-static 0.85" + export SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:-${extra_args}}" + + log "starting server (context_length=${context_length}, max_running=${max_running}, mode=${SERVER_MODE})" + docker_server_start "${MODEL_PATH}" "${PORT}" "${log_dir_global}/server_${context_length}.outer.log" "${SERVER_MODE}" "${SGLANG_EXTRA_LAUNCH_ARGS}" +} + +stop_server() { + docker_server_stop +} + +run_warmup() { + local raw_dir="$1" + local phase_log_dir="$2" + local input_len="$3" + local output_len="${4:-256}" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" + + local tmp_warmup_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_warmup_1_${input_len}_${output_len}.jsonl" + local host_warmup_file="${raw_dir}/${BACKEND}_warmup_$(date '+%m%d')_1_${input_len}_${output_len}.jsonl" + local detail_log="${phase_log_dir}/warmup_${input_len}_${output_len}.log" + + if WARMUP=0 run_random_case \ + "$BACKEND" "$PORT" "$MODEL_PATH" "$tmp_warmup_file" \ + 1 "$input_len" "$output_len" 1 "${DATASET_PATH:-}" \ + > "$detail_log" 2>&1; then + docker cp "${CONTAINER_NAME}:${tmp_warmup_file}" "$host_warmup_file" 2>/dev/null || true + log "warmup completed" + else + log "WARNING: warmup failed; continuing with group" + fi +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$(python3 -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +run_group() { + local group_label="$1" + local context_length="$2" + local max_running="$3" + local -n scenarios_ref="$4" + + local result_root="${RESULT_BASE}/${RUN_ID}/${BACKEND}" + local raw_dir="${result_root}/raw_outputs" + local phase_log_dir="${result_root}/logs" + + mkdir -p "$raw_dir" "$phase_log_dir" + + log "===== ${BACKEND} ${group_label} START (context_length=${context_length}) =====" + + stop_server + if ! start_server "$context_length" "$max_running"; then + log "ERROR: server failed to start for ${group_label}; skipping group" + stop_server + return 1 + fi + + # Warmup with the shortest output for this input length to save time. + local warmup_input="${scenarios_ref[0]}" + local warmup_input_len + warmup_input_len="$(echo "$warmup_input" | awk '{print $2}')" + run_warmup "$raw_dir" "$phase_log_dir" "$warmup_input_len" 256 + + for scenario in "${scenarios_ref[@]}"; do + # Allow SCENARIOS to be passed as a string like "(25 65536 256 50)" from env. + scenario="${scenario//[()]/}" + read -r concurrency input_len output_len num_prompts <<< "$scenario" + + tmp_output_file="/tmp/bench_outputs/${CHIP}_${ENGINE}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + output_file="${raw_dir}/${BACKEND}_${group_label}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${phase_log_dir}/${BACKEND}_${group_label}_c${concurrency}_i${input_len}_o${output_len}.log" + + if scenario_already_completed "$output_file" "$num_prompts"; then + log "skipping already-completed ${BACKEND} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len}" + continue + fi + + log "running ${BACKEND} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" + + if run_random_case \ + "$BACKEND" "$PORT" "$MODEL_PATH" "$tmp_output_file" \ + "$concurrency" "$input_len" "$output_len" "$num_prompts" \ + "${DATASET_PATH:-}" \ + > "$detail_log" 2>&1; then + docker cp "${CONTAINER_NAME}:${tmp_output_file}" "$output_file" || { + log "ERROR: failed to copy ${tmp_output_file} from container" + continue + } + log "finished ${BACKEND} ${group_label} scenario: output=${output_file}" + else + log "ERROR: ${BACKEND} ${group_label} scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + continue + fi + done + + stop_server + log "===== ${BACKEND} ${group_label} DONE =====" +} + +parse_backend() { + local result_root="${RESULT_BASE}/${RUN_ID}/${BACKEND}" + log "parsing ${BACKEND} results in ${result_root}" + python3 "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$BACKEND" \ + >> "${result_root}/logs/parse.log" 2>&1 || { + log "WARNING: parser failed for ${BACKEND}; see ${result_root}/logs/parse.log" + } +} + +write_backend_metadata() { + local result_root="${RESULT_BASE}/${RUN_ID}/${BACKEND}" + ensure_result_root "$result_root" + local meta_json="${result_root}/results.json" + + write_metadata_json \ + "$meta_json" \ + "${EXPERIMENT_NAME}_${BACKEND}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$BACKEND" \ + "$ENGINE" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "" \ + "Kunlun P800 long-context matrix ${BACKEND} TP=8 for DeepSeek-V4-Flash-INT8" + + # Build groups JSON from CONTEXT_GROUPS. + local groups_json="[" + local first=1 + for group in "${CONTEXT_GROUPS[@]}"; do + read -r label context_length max_running <<< "$group" + if [[ "$first" -eq 1 ]]; then + first=0 + else + groups_json+="," + fi + groups_json+="{\"label\":\"${label}\",\"context_length\":${context_length},\"max_running\":${max_running}}" + done + groups_json+="]" + + # Embed config. + python3 - "$meta_json" "$groups_json" <<'PY' >/dev/null +import json +import sys + +path = sys.argv[1] +groups = json.loads(sys.argv[2]) + +with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + +data["config"] = { + "tp": 8, + "xpu_visible_devices": "0,1,2,3,4,5,6,7", + "backend": data["metadata"]["backend"], + "engine": data["metadata"]["engine"], + "server_mode": "w8a8_int8_baseline", + "server_start_script": f"experiments/{data['metadata']['experiment']}/start_server.sh", + "groups": groups +} +with open(path, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) +PY +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +stop_server + +write_backend_metadata + +for group in "${CONTEXT_GROUPS[@]}"; do + read -r group_label context_length max_running <<< "$group" + + # Resolve scenario array for this group. + scenario_array_name="SCENARIOS_${group_label^^}" + if [[ -z "${!scenario_array_name:-}" ]]; then + log "WARNING: no scenarios defined for group ${group_label}" + continue + fi + + run_group "$group_label" "$context_length" "$max_running" "$scenario_array_name" +done + +parse_backend + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/dsv4_p800_long_context_matrix/start_server.sh b/experiments/dsv4_p800_long_context_matrix/start_server.sh new file mode 100755 index 0000000..5b5bcf6 --- /dev/null +++ b/experiments/dsv4_p800_long_context_matrix/start_server.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Start SGLang server for the P800 long-context matrix. +# Usage: start_server.sh +set -Eeuo pipefail + +CONTEXT_LENGTH="${1:-80000}" +MAX_RUNNING="${2:-64}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/server_docker.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +log_init "${RESULT_ROOT:-/tmp/${EXPERIMENT_NAME}}/logs/start_server.log" +log "starting server for ${EXPERIMENT_NAME}" +log "model: ${MODEL_PATH}" +log "port: ${PORT}" +log "server mode: ${SERVER_MODE}" +log "context length: ${CONTEXT_LENGTH}" +log "max running requests: ${MAX_RUNNING}" + +extra_args="--context-length ${CONTEXT_LENGTH} --max-running-requests ${MAX_RUNNING} --mem-fraction-static 0.85" +export SGLANG_EXTRA_LAUNCH_ARGS="${SGLANG_EXTRA_LAUNCH_ARGS:-${extra_args}}" + +docker_server_start "${MODEL_PATH}" "${PORT}" "${LOG_DIR}/server.outer.log" "${SERVER_MODE}" "${SGLANG_EXTRA_LAUNCH_ARGS}" diff --git a/scripts/common/bench_client_docker.sh b/scripts/common/bench_client_docker.sh index 504412d..6b3de60 100755 --- a/scripts/common/bench_client_docker.sh +++ b/scripts/common/bench_client_docker.sh @@ -63,7 +63,7 @@ run_random_case() { --random-input-len "$input_len" --random-output-len "$output_len" --num-prompts "$num_prompts" - --max-concurrency "$concurrency" + --concurrency "$concurrency" --warmup-requests "$warmup" --output-file "$output_file" --output-details diff --git a/scripts/common/server_docker.sh b/scripts/common/server_docker.sh index 521b85a..3d82312 100755 --- a/scripts/common/server_docker.sh +++ b/scripts/common/server_docker.sh @@ -206,11 +206,12 @@ EOF bash -c "echo '${server_cmd_b64}' | base64 -d | bash" \ >> "${server_log}" 2>&1 - log "container ${container} started, waiting for health" - if health_check 127.0.0.1 "$port" 600; then + local health_wait="${SGLANG_HEALTH_CHECK_TIMEOUT:-600}" + log "container ${container} started, waiting for health (timeout=${health_wait}s)" + if health_check 127.0.0.1 "$port" "$health_wait"; then log "container ${container} is healthy" else - log "ERROR: container ${container} failed health check" + log "ERROR: container ${container} failed health check after ${health_wait}s" return 1 fi }