162168da3c
- sglang_vs_vllm/run_bench.sh: add phase1/phase2 server_args per backend - dspark_vs_default/run_bench.sh: add server_args for dspark/default - READMEs document where to find the recorded server args
256 lines
7.8 KiB
Bash
Executable File
256 lines
7.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# vLLM+DSpark vs vLLM default controlled comparison on H200.
|
|
set -Eeuo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
|
|
|
# shellcheck source=/dev/null
|
|
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
|
|
# shellcheck source=/dev/null
|
|
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
|
|
# shellcheck source=/dev/null
|
|
source "${SCRIPT_DIR}/config.env"
|
|
|
|
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
|
RESULT_BASE="${SCRIPT_DIR}/results"
|
|
|
|
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
|
|
mkdir -p "$log_dir_global"
|
|
log_init "${log_dir_global}/orchestrator.log"
|
|
|
|
log "experiment=${EXPERIMENT_NAME}"
|
|
log "run_id=${RUN_ID}"
|
|
log "platform=${PLATFORM}"
|
|
log "hardware=${HARDWARE}"
|
|
log "dspark_model=${DSPARK_MODEL_PATH}"
|
|
log "default_model=${DEFAULT_MODEL_PATH}"
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Helpers
|
|
# ---------------------------------------------------------------------------
|
|
|
|
is_server_healthy() {
|
|
local port="$1"
|
|
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
|
|
}
|
|
|
|
stop_server() {
|
|
local backend="$1"
|
|
local pid_file="/data/user1/yy/${EXPERIMENT_NAME}_${backend}.pid"
|
|
if [[ -f "$pid_file" ]]; then
|
|
local pid
|
|
pid="$(cat "$pid_file")"
|
|
if kill -0 "$pid" 2>/dev/null; then
|
|
log "stopping ${backend} server pid=${pid}"
|
|
kill "$pid" 2>/dev/null || true
|
|
sleep 5
|
|
kill -9 "$pid" 2>/dev/null || true
|
|
fi
|
|
rm -f "$pid_file"
|
|
fi
|
|
sleep 2
|
|
}
|
|
|
|
start_server() {
|
|
local backend="$1"
|
|
local start_script port
|
|
if [[ "$backend" == "dspark" ]]; then
|
|
start_script="${DSPARK_START_SCRIPT}"
|
|
port="$DSPARK_PORT"
|
|
else
|
|
start_script="${DEFAULT_START_SCRIPT}"
|
|
port="$DEFAULT_PORT"
|
|
fi
|
|
|
|
log "starting ${backend} server with ${start_script}"
|
|
bash "${start_script}" >> "${log_dir_global}/${backend}.server.outer.log" 2>&1
|
|
|
|
if ! is_server_healthy "$port"; then
|
|
log "error: ${backend} server failed to become healthy"
|
|
return 1
|
|
fi
|
|
log "${backend} server is healthy on port ${port}"
|
|
}
|
|
|
|
run_warmup() {
|
|
local backend="$1"
|
|
local port
|
|
if [[ "$backend" == "dspark" ]]; then
|
|
port="$DSPARK_PORT"
|
|
else
|
|
port="$DEFAULT_PORT"
|
|
fi
|
|
|
|
log "warming up ${backend} (input=4000, output=512, num=2)"
|
|
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
|
|
--backend vllm \
|
|
--host 127.0.0.1 \
|
|
--port "$port" \
|
|
--dataset-name random \
|
|
--random-input-len 4000 \
|
|
--random-output-len 512 \
|
|
--num-prompts 2 \
|
|
--max-concurrency 1 \
|
|
--request-rate 10000 \
|
|
--output-file /tmp/${EXPERIMENT_NAME}_${backend}_warmup.jsonl \
|
|
--output-details \
|
|
> "${log_dir_global}/${backend}.warmup.log" 2>&1 || true
|
|
log "warmup for ${backend} completed"
|
|
}
|
|
|
|
run_benchmark() {
|
|
local backend="$1"
|
|
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
|
|
local raw_dir="${result_root}/raw_outputs"
|
|
local bench_log_dir="${result_root}/logs"
|
|
|
|
mkdir -p "$raw_dir" "$bench_log_dir"
|
|
|
|
local port
|
|
if [[ "$backend" == "dspark" ]]; then
|
|
port="$DSPARK_PORT"
|
|
else
|
|
port="$DEFAULT_PORT"
|
|
fi
|
|
|
|
log "===== ${backend} BENCHMARK START ====="
|
|
|
|
stop_server "$backend"
|
|
start_server "$backend"
|
|
run_warmup "$backend"
|
|
|
|
for scenario in "${SCENARIOS[@]}"; do
|
|
read -r concurrency input_len output_len num_prompts <<< "$scenario"
|
|
output_file="${raw_dir}/${backend}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
|
detail_log="${bench_log_dir}/${backend}_c${concurrency}_i${input_len}_o${output_len}.log"
|
|
|
|
log "running ${backend} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
|
|
|
|
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
|
|
--backend vllm \
|
|
--host 127.0.0.1 \
|
|
--port "$port" \
|
|
--dataset-name random \
|
|
--random-input-len "$input_len" \
|
|
--random-output-len "$output_len" \
|
|
--num-prompts "$num_prompts" \
|
|
--max-concurrency "$concurrency" \
|
|
--request-rate 10000 \
|
|
--output-file "$output_file" \
|
|
--output-details \
|
|
> "$detail_log" 2>&1 || {
|
|
log "ERROR: ${backend} scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
|
|
continue
|
|
}
|
|
|
|
log "finished ${backend} scenario: output=${output_file}"
|
|
done
|
|
|
|
stop_server "$backend"
|
|
log "===== ${backend} BENCHMARK DONE ====="
|
|
}
|
|
|
|
parse_backend() {
|
|
local backend="$1"
|
|
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
|
|
log "parsing ${backend} results in ${result_root}"
|
|
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_backend.py" "$result_root" --backend "$backend" \
|
|
>> "${result_root}/logs/parse.log" 2>&1 || {
|
|
log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log"
|
|
}
|
|
}
|
|
|
|
write_backend_metadata() {
|
|
local backend="$1"
|
|
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
|
|
ensure_result_root "$result_root"
|
|
local meta_json="${result_root}/results.json"
|
|
|
|
local model_path env_path engine
|
|
if [[ "$backend" == "dspark" ]]; then
|
|
model_path="$DSPARK_MODEL_PATH"
|
|
env_path="$VENV_VLLM_DSPARK"
|
|
engine="vllm-dspark"
|
|
else
|
|
model_path="$DEFAULT_MODEL_PATH"
|
|
env_path="$VENV_VLLM"
|
|
engine="vllm-default"
|
|
fi
|
|
|
|
# Capture exact server launch args for reproducibility.
|
|
local server_args
|
|
if [[ "$backend" == "dspark" ]]; then
|
|
server_args="vllm serve $model_path --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len $MAX_MODEL_LEN --max-num-seqs $MAX_NUM_SEQS --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --spec-method dspark --spec-model $model_path --spec-tokens 5 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port $DSPARK_PORT"
|
|
else
|
|
server_args="vllm serve $model_path --trust-remote-code --tensor-parallel-size 8 --kv-cache-dtype fp8 --max-model-len $MAX_MODEL_LEN --max-num-seqs $MAX_NUM_SEQS --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port $DEFAULT_PORT"
|
|
fi
|
|
|
|
write_metadata_json \
|
|
"$meta_json" \
|
|
"${EXPERIMENT_NAME}_${backend}" \
|
|
"$RUN_ID" \
|
|
"$model_path" \
|
|
"vllm" \
|
|
"$engine" \
|
|
"$HARDWARE" \
|
|
"$ACCELERATOR" \
|
|
"$CHIP" \
|
|
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
|
"$env_path" \
|
|
"H200 ${engine} TP=8 benchmark for TTFT comparison"
|
|
|
|
# Embed config and server args.
|
|
"${VENV_CLIENT}/bin/python" - "$meta_json" "$backend" "$server_args" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
path, backend, server_args = sys.argv[1], sys.argv[2], sys.argv[3]
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
|
|
data["config"] = {
|
|
"tp": 8,
|
|
"cuda_visible_devices": "0,1,2,3,4,5,6,7",
|
|
"max_model_len": 32768,
|
|
"max_num_seqs": 256,
|
|
"backend": backend,
|
|
"server_start_script": f"experiments/dsv4_h200_vllm_dspark_vs_default/start_{backend}.sh",
|
|
"server_args": server_args,
|
|
}
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
json.dump(data, f, indent=2, ensure_ascii=False)
|
|
PY
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# Cleanup.
|
|
stop_server dspark
|
|
stop_server default
|
|
|
|
write_backend_metadata dspark
|
|
write_backend_metadata default
|
|
|
|
# Run DSpark.
|
|
run_benchmark dspark
|
|
parse_backend dspark
|
|
|
|
# Run default vLLM.
|
|
run_benchmark default
|
|
parse_backend default
|
|
|
|
# Generate comparison.
|
|
log "generating comparison report"
|
|
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \
|
|
--dspark "${RESULT_BASE}/${RUN_ID}/dspark" \
|
|
--default "${RESULT_BASE}/${RUN_ID}/default" \
|
|
--output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
|
|
>> "${log_dir_global}/compare.log" 2>&1 || {
|
|
log "WARNING: comparison script failed; see ${log_dir_global}/compare.log"
|
|
}
|
|
|
|
log "all results saved to ${RESULT_BASE}/${RUN_ID}"
|