a4e38b9e33
Move all experiments under hardware-specific folders: - experiments/h200/ : H200 GPU experiments (15 dirs) - experiments/h20/ : H20 GPU experiments (2 dirs) - experiments/p800/ : Kunlun P800 experiments (3 dirs) - experiments/pro6000/ : RTX 6000D experiments (2 dirs) This improves discoverability and keeps hardware-specific configs isolated from each other.
342 lines
11 KiB
Bash
Executable File
342 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Long-context matrix: SGLang vs vLLM on DeepSeek-V4-Flash (H200).
|
|
set -Eeuo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
|
# shellcheck source=/dev/null
|
|
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
|
# shellcheck source=/dev/null
|
|
source "${SCRIPT_DIR}/config.env"
|
|
|
|
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
|
RESULT_BASE="${SCRIPT_DIR}/results"
|
|
# Comma-separated list of backends to run. Default: sglang,vllm,dspark.
|
|
BACKENDS="${BACKENDS:-sglang,vllm,dspark}"
|
|
|
|
backend_enabled() {
|
|
local backend="$1"
|
|
[[ ",${BACKENDS}," == *",${backend},"* ]]
|
|
}
|
|
|
|
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
|
|
mkdir -p "$log_dir_global"
|
|
log_init "${log_dir_global}/orchestrator.log"
|
|
|
|
log "experiment=${EXPERIMENT_NAME}"
|
|
log "run_id=${RUN_ID}"
|
|
log "platform=${PLATFORM}"
|
|
log "hardware=${HARDWARE}"
|
|
log "model=${MODEL_PATH}"
|
|
log "groups=${#CONTEXT_GROUPS[@]}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
is_server_healthy() {
|
|
local port="$1"
|
|
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
|
|
}
|
|
|
|
stop_server() {
|
|
local backend="$1"
|
|
local pid_file="/data/user1/yy/dsv4_h200_long_context_matrix_${backend}.pid"
|
|
if [[ -f "$pid_file" ]]; then
|
|
local pid
|
|
pid="$(cat "$pid_file")"
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
log "stopping ${backend} server pid=${pid}"
|
|
kill "$pid" 2>/dev/null || true
|
|
sleep 5
|
|
kill -9 "$pid" 2>/dev/null || true
|
|
fi
|
|
rm -f "$pid_file"
|
|
fi
|
|
# Fallback cleanup.
|
|
if [[ "$backend" == "sglang" ]]; then
|
|
pkill -9 -f "sglang serve.*DeepSeek-V4-Flash" 2>/dev/null || true
|
|
elif [[ "$backend" == "dspark" ]]; then
|
|
pkill -9 -f "vllm serve.*DeepSeek-V4-Flash-DSpark" 2>/dev/null || true
|
|
else
|
|
pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true
|
|
fi
|
|
sleep 2
|
|
}
|
|
|
|
start_server() {
|
|
local backend="$1"
|
|
local max_model_len="$2"
|
|
local max_slots="$3"
|
|
local start_script port
|
|
if [[ "$backend" == "sglang" ]]; then
|
|
start_script="${SGLANG_START_SCRIPT}"
|
|
port="$SGLANG_PORT"
|
|
elif [[ "$backend" == "dspark" ]]; then
|
|
start_script="${DSPARK_START_SCRIPT}"
|
|
port="$DSPARK_PORT"
|
|
else
|
|
start_script="${VLLM_START_SCRIPT}"
|
|
port="$VLLM_PORT"
|
|
fi
|
|
|
|
log "starting ${backend} server (max_model_len=${max_model_len}, slots=${max_slots}) with ${start_script}"
|
|
bash "${start_script}" "$max_model_len" "$max_slots" >> "${log_dir_global}/${backend}_${max_model_len}.server.outer.log" 2>&1
|
|
|
|
if ! is_server_healthy "$port"; then
|
|
log "error: ${backend} server failed to become healthy"
|
|
return 1
|
|
fi
|
|
log "${backend} server is healthy on port ${port}"
|
|
}
|
|
|
|
run_warmup() {
|
|
local backend="$1"
|
|
local input_len="$2"
|
|
local output_len="${3:-256}"
|
|
local port
|
|
if [[ "$backend" == "sglang" ]]; then
|
|
port="$SGLANG_PORT"
|
|
elif [[ "$backend" == "dspark" ]]; then
|
|
port="$DSPARK_PORT"
|
|
else
|
|
port="$VLLM_PORT"
|
|
fi
|
|
|
|
log "warming up ${backend} (input=${input_len}, output=${output_len}, num=1)"
|
|
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \
|
|
--backend vllm \
|
|
--port "$port" \
|
|
--input-len "$input_len" \
|
|
--output-len "$output_len" \
|
|
--num 1 \
|
|
--env-python "${VENV_CLIENT}/bin/python" \
|
|
>> "${log_dir_global}/${backend}.warmup.log" 2>&1
|
|
log "warmup for ${backend} completed"
|
|
}
|
|
|
|
scenario_already_completed() {
|
|
local output_file="$1"
|
|
local expected="$2"
|
|
[[ -s "$output_file" ]] || return 1
|
|
local completed
|
|
completed="$(${VENV_CLIENT}/bin/python -c "
|
|
import json, sys
|
|
path = sys.argv[1]
|
|
try:
|
|
with open(path, 'r', encoding='utf-8') as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
data = json.loads(line)
|
|
print(data.get('completed', 0))
|
|
break
|
|
except Exception:
|
|
print(0)
|
|
" "$output_file")"
|
|
[[ "${completed:-0}" -ge "$expected" ]]
|
|
}
|
|
|
|
run_group() {
|
|
local backend="$1"
|
|
local group_label="$2"
|
|
local max_model_len="$3"
|
|
local max_slots="$4"
|
|
local -n scenarios_ref="$5"
|
|
|
|
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
|
|
local raw_dir="${result_root}/raw_outputs"
|
|
local phase_log_dir="${result_root}/logs"
|
|
|
|
mkdir -p "$raw_dir" "$phase_log_dir"
|
|
|
|
local port
|
|
if [[ "$backend" == "sglang" ]]; then
|
|
port="$SGLANG_PORT"
|
|
elif [[ "$backend" == "dspark" ]]; then
|
|
port="$DSPARK_PORT"
|
|
else
|
|
port="$VLLM_PORT"
|
|
fi
|
|
|
|
log "===== ${backend} ${group_label} START (max_model_len=${max_model_len}) ====="
|
|
|
|
stop_server "$backend"
|
|
if ! start_server "$backend" "$max_model_len" "$max_slots"; then
|
|
log "ERROR: ${backend} failed to start for ${group_label}; skipping group"
|
|
stop_server "$backend"
|
|
return 1
|
|
fi
|
|
|
|
# Warmup with the shortest output for this input length to save time.
|
|
local warmup_input="${scenarios_ref[0]}"
|
|
local warmup_input_len
|
|
warmup_input_len="$(echo "$warmup_input" | awk '{print $2}')"
|
|
run_warmup "$backend" "$warmup_input_len" 256
|
|
|
|
# Bench client backend: DSpark is served by vLLM API, so use vllm client.
|
|
local client_backend="$backend"
|
|
if [[ "$backend" == "dspark" ]]; then
|
|
client_backend="vllm"
|
|
fi
|
|
|
|
# Output filename prefix: DSpark raw files must start with "vllm_" so that
|
|
# scripts/common/parse_backend.py (which only knows sglang/vllm) can parse them.
|
|
local file_prefix="${backend}"
|
|
if [[ "$backend" == "dspark" ]]; then
|
|
file_prefix="vllm_dspark"
|
|
fi
|
|
|
|
for scenario in "${scenarios_ref[@]}"; do
|
|
read -r concurrency input_len output_len num_prompts <<< "$scenario"
|
|
output_file="${raw_dir}/${file_prefix}_${group_label}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
|
detail_log="${phase_log_dir}/${backend}_${group_label}_c${concurrency}_i${input_len}_o${output_len}.log"
|
|
|
|
if scenario_already_completed "$output_file" "$num_prompts"; then
|
|
log "skipping already-completed ${backend} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len}"
|
|
continue
|
|
fi
|
|
|
|
log "running ${backend} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
|
|
|
|
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
|
|
--backend "$client_backend" \
|
|
--host 127.0.0.1 \
|
|
--port "$port" \
|
|
--dataset-name random \
|
|
--random-input-len "$input_len" \
|
|
--random-output-len "$output_len" \
|
|
--num-prompts "$num_prompts" \
|
|
--max-concurrency "$concurrency" \
|
|
--request-rate 10000 \
|
|
--output-file "$output_file" \
|
|
--output-details \
|
|
> "$detail_log" 2>&1 || {
|
|
log "ERROR: ${backend} ${group_label} scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
|
|
continue
|
|
}
|
|
|
|
log "finished ${backend} ${group_label} scenario: output=${output_file}"
|
|
done
|
|
|
|
stop_server "$backend"
|
|
log "===== ${backend} ${group_label} DONE ====="
|
|
}
|
|
|
|
parse_backend() {
|
|
local backend="$1"
|
|
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
|
|
# parse_backend.py only knows sglang/vllm; DSpark raw files use vllm client format.
|
|
local parser_backend="$backend"
|
|
if [[ "$backend" == "dspark" ]]; then
|
|
parser_backend="vllm"
|
|
fi
|
|
log "parsing ${backend} results in ${result_root} (parser_backend=${parser_backend})"
|
|
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$parser_backend" \
|
|
>> "${result_root}/logs/parse.log" 2>&1 || {
|
|
log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log"
|
|
}
|
|
}
|
|
|
|
write_backend_metadata() {
|
|
local backend="$1"
|
|
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
|
|
ensure_result_root "$result_root"
|
|
local meta_json="${result_root}/results.json"
|
|
|
|
local env_path model_path engine_name
|
|
if [[ "$backend" == "sglang" ]]; then
|
|
env_path="$VENV_SGLANG"
|
|
model_path="$MODEL_PATH"
|
|
engine_name="sglang"
|
|
elif [[ "$backend" == "dspark" ]]; then
|
|
env_path="$VENV_VLLM_DSPARK"
|
|
model_path="$DSPARK_MODEL_PATH"
|
|
engine_name="vllm-dspark"
|
|
else
|
|
env_path="$VENV_VLLM"
|
|
model_path="$MODEL_PATH"
|
|
engine_name="vllm"
|
|
fi
|
|
|
|
write_metadata_json \
|
|
"$meta_json" \
|
|
"${EXPERIMENT_NAME}_${backend}" \
|
|
"$RUN_ID" \
|
|
"$model_path" \
|
|
"${backend}" \
|
|
"$engine_name" \
|
|
"$HARDWARE" \
|
|
"$ACCELERATOR" \
|
|
"$CHIP" \
|
|
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
|
"$env_path" \
|
|
"H200 long-context matrix ${backend} TP=8 for DeepSeek-V4-Flash"
|
|
|
|
# Embed config.
|
|
jq --arg backend "$backend" \
|
|
'.config = {
|
|
"tp": 8,
|
|
"cuda_visible_devices": "0,1,2,3,4,5,6,7",
|
|
"backend": $backend,
|
|
"server_start_script": "experiments/dsv4_h200_long_context_matrix/start_\($backend).sh",
|
|
"groups": [
|
|
{"label": "64k", "max_model_len": 80000, "max_slots": 25},
|
|
{"label": "128k", "max_model_len": 140000, "max_slots": 10},
|
|
{"label": "256k", "max_model_len": 270000, "max_slots": 10}
|
|
]
|
|
}' "$meta_json" > "${meta_json}.tmp" && mv "${meta_json}.tmp" "$meta_json"
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Cleanup any leftovers.
|
|
for backend in sglang vllm dspark; do
|
|
backend_enabled "$backend" && stop_server "$backend"
|
|
done
|
|
|
|
for backend in sglang vllm dspark; do
|
|
backend_enabled "$backend" && write_backend_metadata "$backend"
|
|
done
|
|
|
|
for group in "${CONTEXT_GROUPS[@]}"; do
|
|
read -r group_label max_model_len max_slots <<< "$group"
|
|
|
|
# Resolve scenario array for this group.
|
|
scenario_array_name="SCENARIOS_${group_label^^}"
|
|
if [[ -z "${!scenario_array_name:-}" ]]; then
|
|
log "WARNING: no scenarios defined for group ${group_label}"
|
|
continue
|
|
fi
|
|
|
|
for backend in sglang vllm dspark; do
|
|
backend_enabled "$backend" && run_group "$backend" "$group_label" "$max_model_len" "$max_slots" "$scenario_array_name"
|
|
done
|
|
done
|
|
|
|
# Parse and compare.
|
|
for backend in sglang vllm dspark; do
|
|
backend_enabled "$backend" && parse_backend "$backend"
|
|
done
|
|
|
|
if backend_enabled sglang && backend_enabled vllm && backend_enabled dspark; then
|
|
log "generating three-way comparison report"
|
|
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \
|
|
--sglang "${RESULT_BASE}/${RUN_ID}/sglang" \
|
|
--vllm "${RESULT_BASE}/${RUN_ID}/vllm" \
|
|
--dspark "${RESULT_BASE}/${RUN_ID}/dspark" \
|
|
--output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
|
|
>> "${log_dir_global}/compare.log" 2>&1 || {
|
|
log "WARNING: comparison script failed; see ${log_dir_global}/compare.log"
|
|
}
|
|
else
|
|
log "skipping three-way comparison (not all backends enabled: BACKENDS=${BACKENDS})"
|
|
fi
|
|
|
|
log "all results saved to ${RESULT_BASE}/${RUN_ID}"
|