From a127cf7ef00589da0a60a381aa52275edd781a94 Mon Sep 17 00:00:00 2001 From: yy-fighting Date: Wed, 8 Jul 2026 10:48:43 +0000 Subject: [PATCH] refactor(common): centralize parse/compare/warmup; add TEMPLATE and EXPERIMENT_GUIDE --- docs/EXPERIMENT_GUIDE.md | 178 ++++++++++++ experiments/TEMPLATE/README.md | 45 +++ experiments/TEMPLATE/config.env | 33 +++ experiments/TEMPLATE/run_bench.sh | 265 ++++++++++++++++++ experiments/TEMPLATE/start_sglang.sh | 62 ++++ experiments/TEMPLATE/start_vllm.sh | 65 +++++ .../dsv4_h200_256k_4k_probe/run_bench.sh | 6 +- .../dsv4_h200_64k_sglang_vs_vllm/run_bench.sh | 6 +- .../dsv4_h200_sglang_vs_vllm/run_bench.sh | 6 +- scripts/common/compare.py | 88 ++++++ scripts/common/parse_backend.py | 218 ++++++++++++++ scripts/common/warmup.py | 76 +++++ 12 files changed, 1039 insertions(+), 9 deletions(-) create mode 100644 docs/EXPERIMENT_GUIDE.md create mode 100644 experiments/TEMPLATE/README.md create mode 100644 experiments/TEMPLATE/config.env create mode 100755 experiments/TEMPLATE/run_bench.sh create mode 100755 experiments/TEMPLATE/start_sglang.sh create mode 100755 experiments/TEMPLATE/start_vllm.sh create mode 100755 scripts/common/compare.py create mode 100755 scripts/common/parse_backend.py create mode 100755 scripts/common/warmup.py diff --git a/docs/EXPERIMENT_GUIDE.md b/docs/EXPERIMENT_GUIDE.md new file mode 100644 index 0000000..9609f5e --- /dev/null +++ b/docs/EXPERIMENT_GUIDE.md @@ -0,0 +1,178 @@ +# 实验规范指南 + +本仓库用于统一记录和复现不同芯片、不同 backend（SGLang / vLLM）下的推理测速实验。 + +## 1. 目录结构 + +``` +experiments// +├── config.env # 实验参数唯一来源 +├── run_bench.sh # 编排入口 +├── start_sglang.sh # SGLang server 启动脚本 +├── start_vllm.sh # vLLM server 启动脚本 +└── results// + ├── comparison.md + ├── sglang/ + │ ├── results.json + │ ├── report.md + │ ├── raw_outputs/ # .gitignore 忽略 + │ └── logs/ # .gitignore 忽略 + └── vllm/ + ├── results.json + ├── report.md + ├── raw_outputs/ # .gitignore 忽略 + └── logs/ # .gitignore 忽略 +``` + +通用工具集中放在 `scripts/common/`，不要复制到每个实验： + +- `scripts/common/lib.sh`：日志、metadata、JSON 工具 +- `scripts/common/platform.sh`：平台自动检测 +- `scripts/common/warmup.py`：server 预热 +- `scripts/common/parse_backend.py`：解析 raw jsonl -> results.json + report.md +- `scripts/common/compare.py`：生成 SGLang vs vLLM 对比表 + +## 2. 新增一个实验 + +最快方式： + +```bash +cp -r experiments/TEMPLATE experiments/ +# 修改 config.env、start_*.sh +bash experiments//run_bench.sh +``` + +### 2.1 config.env 必备字段 + +```bash +EXPERIMENT="" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" + +SGLANG_PORT="${SGLANG_PORT:-30006}" +VLLM_PORT="${VLLM_PORT:-30005}" + +VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" +VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +TP="${TP:-8}" + +MAX_MODEL_LEN=... +MAX_NUM_SEQS=... +MAX_RUNNING=... + +# 场景："concurrency input_len output_len num_prompts" +declare -a SCENARIOS=( + "1 512 256 32" +) + +VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" +SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh" +VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh" +``` + +### 2.2 可复现性要求 + +每个 `results.json` 必须记录： + +- `metadata.model`：模型路径 +- `metadata.hardware` / `metadata.accelerator` / `metadata.chip` +- `metadata.env`：使用的虚拟环境路径 +- `metadata.git_commit` / `metadata.git_dirty` +- `config.server_args`：完整的 server 启动命令 +- `config.cuda_visible_devices` + +这些通过 `scripts/common/lib.sh` 中的 `write_metadata_json` 和 `jq` 注入。 + +## 3. 平台与硬件适配 + +### 3.1 芯片级默认配置 + +芯片相关默认值放到 `platforms/.env`，例如： + +- `platforms/nvidia_h200.env` +- `platforms/kunlun_p800.env` + +实验级覆盖通过 `config.env` 实现。 + +### 3.2 跨芯片注意事项 + +| 项 | H200 | P800 | 备注 | +|---|---|---|---| +| TP 大小 | 8 | 8/16 | 由 `TP` 控制 | +| KV cache dtype | fp8 | 可能不同 | 在 `start_vllm.sh` 调整 | +| MLA backend | flashinfer_mla | 可能不支持 | 在 `start_sglang.sh`/`start_vllm.sh` 调整 | +| 最大上下文 | 受显存限制 | 受显存限制 | 通过探针测试确定 | +| 设备变量 | `CUDA_VISIBLE_DEVICES` | 可能不同 | 在 `config.env` 设置 | + +如果某芯片不支持某个 feature，在 `start_*.sh` 里用条件判断，不要把条件写进通用脚本。 + +## 4. SLO 标准 + +默认使用 S2 层级： + +- TTFT P95 < 3000 ms +- TPOT mean < 50 ms + +`scripts/common/parse_backend.py` 和 `scripts/common/compare.py` 默认按这个标准打标。如果需要其他 tier，调用 `compare.py` 时传入 `--ttft-limit` 和 `--tpot-limit`。 + +## 5. 断点续测 + +`run_bench.sh` 必须实现 `scenario_already_completed()` 检查： + +- 如果某 scenario 的 `raw_outputs/*.jsonl` 中 `completed` 数量已达标，直接跳过 +- 重新执行脚本即可从中断处继续 + +## 6. Git 提交规范 + +### 6.1 必须提交 + +- 实验代码：`config.env`、`run_bench.sh`、`start_*.sh` +- 最终结果：`comparison.md`、`sglang/results.json`、`sglang/report.md`、`vllm/results.json`、`vllm/report.md` + +### 6.2 不要提交 + +- `raw_outputs/` +- `logs/` +- 中间 server 日志 + +这些已在 `.gitignore` 中忽略。 + +### 6.3 提交示例 + +```bash +git add experiments//config.env \ + experiments//run_bench.sh \ + experiments//start_*.sh \ + experiments//results//comparison.md \ + experiments//results//sglang/results.json \ + experiments//results//sglang/report.md \ + experiments//results//vllm/results.json \ + experiments//results//vllm/report.md + +git commit -m "results: run " +git push origin main +``` + +## 7. 命名约定 + +- 实验目录：`{model}_{chip}_{scenario}_{backend_vs_backend}`，例如 `dsv4_h200_sglang_vs_vllm` +- scenario 名称：`c{concurrency}_i{input_len}_o{output_len}` +- raw 输出：`{backend}_{label}_{MMDD}_{concurrency}_{input_len}_{output_len}.jsonl` +- run_id：`YYYYMMDD-HHMMSS` + +## 8. 常见问题 + +### 8.1 新芯片上 server 起不来 + +1. 检查 `platforms/.env` 是否已定义 +2. 检查 `start_*.sh` 中的 backend-specific 参数是否被该芯片支持 +3. 先用最小场景（input=512, output=256, concurrency=1）验证通路 + +### 8.2 长上下文 OOM + +1. 降低并发 +2. 降低输出长度 +3. 调整 `mem-fraction-static` / `gpu-memory-utilization` +4. 记录能跑通的最大组合，更新矩阵 diff --git a/experiments/TEMPLATE/README.md b/experiments/TEMPLATE/README.md new file mode 100644 index 0000000..c6ee833 --- /dev/null +++ b/experiments/TEMPLATE/README.md @@ -0,0 +1,45 @@ +# TEMPLATE：SGLang vs vLLM 实验模板 + +这是一个最小可运行的实验模板。要新增一个实验： + +```bash +cp -r experiments/TEMPLATE experiments/ +# 修改 config.env 里的参数 +# 按需调整 start_sglang.sh / start_vllm.sh 的启动参数 +bash experiments//run_bench.sh +``` + +## 目录约定 + +- `config.env`：实验参数唯一来源 +- `start_sglang.sh` / `start_vllm.sh`：server 启动脚本 +- `run_bench.sh`：编排入口 +- `results//`：自动生成的结果目录 + - `sglang/`, `vllm/`：各自 backend 的结果 + - `comparison.md`：横向对比表 + +## 通用工具位置 + +不要在本目录下复制解析脚本，直接引用： + +- 预热：`../../scripts/common/warmup.py` +- 结果解析：`../../scripts/common/parse_backend.py` +- 对比报告：`../../scripts/common/compare.py` +- 公共函数：`../../scripts/common/lib.sh`, `../../scripts/common/platform.sh` + +## 提交规范 + +只提交最终文件： + +```bash +git add experiments//config.env \ + experiments//run_bench.sh \ + experiments//start_*.sh \ + experiments//results//comparison.md \ + experiments//results//sglang/results.json \ + experiments//results//sglang/report.md \ + experiments//results//vllm/results.json \ + experiments//results//vllm/report.md +``` + +`raw_outputs/` 和 `logs/` 已在 `.gitignore` 中忽略。 diff --git a/experiments/TEMPLATE/config.env b/experiments/TEMPLATE/config.env new file mode 100644 index 0000000..8d6df3c --- /dev/null +++ b/experiments/TEMPLATE/config.env @@ -0,0 +1,33 @@ +# Template configuration for a SGLang vs vLLM benchmark experiment. +# Copy this directory to experiments// and adjust the values. + +EXPERIMENT="TEMPLATE" +MODEL_NAME="DeepSeek-V4-Flash" +MODEL_PATH="/data/models/DeepSeek-V4-Flash" +SERVED_MODEL_NAME="deepseek-v4-flash" + +SGLANG_PORT="${SGLANG_PORT:-30006}" +VLLM_PORT="${VLLM_PORT:-30005}" + +VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}" +VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}" + +# Hardware: override in platforms/.env or set explicitly. +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +TP="${TP:-8}" + +# Server limits. +MAX_MODEL_LEN="${MAX_MODEL_LEN:-32768}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" +MAX_RUNNING="${MAX_RUNNING:-256}" + +# Scenarios: "concurrency input_len output_len num_prompts". +declare -a SCENARIOS=( + "1 512 256 32" + "32 512 256 128" +) + +VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}" + +SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh" +VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh" diff --git a/experiments/TEMPLATE/run_bench.sh b/experiments/TEMPLATE/run_bench.sh new file mode 100755 index 0000000..04a6fdd --- /dev/null +++ b/experiments/TEMPLATE/run_bench.sh @@ -0,0 +1,265 @@ +#!/usr/bin/env bash +# Template orchestrator for a SGLang vs vLLM benchmark experiment. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME}" +log "run_id=${RUN_ID}" +log "platform=${PLATFORM}" +log "hardware=${HARDWARE}" +log "model=${MODEL_PATH}" +log "max_model_len=${MAX_MODEL_LEN}" +log "scenarios=${#SCENARIOS[@]}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + local port="$1" + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1 +} + +stop_server() { + local backend="$1" + local pid_file="/data/user1/yy/${EXPERIMENT_NAME}_${backend}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping ${backend} server pid=${pid}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + # Fallback cleanup. + if [[ "$backend" == "sglang" ]]; then + pkill -9 -f "sglang serve.*${MODEL_NAME}" 2>/dev/null || true + else + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + fi + sleep 2 +} + +start_server() { + local backend="$1" + local start_script + if [[ "$backend" == "sglang" ]]; then + start_script="${SGLANG_START_SCRIPT}" + local port="$SGLANG_PORT" + else + start_script="${VLLM_START_SCRIPT}" + local port="$VLLM_PORT" + fi + + log "starting ${backend} server with ${start_script}" + bash "${start_script}" >> "${log_dir_global}/${backend}.server.outer.log" 2>&1 + + if ! is_server_healthy "$port"; then + log "error: ${backend} server failed to become healthy" + return 1 + fi + log "${backend} server is healthy on port ${port}" +} + +run_warmup() { + local backend="$1" + local port + if [[ "$backend" == "sglang" ]]; then + port="$SGLANG_PORT" + else + port="$VLLM_PORT" + fi + + log "warming up ${backend}" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ + --backend "$backend" \ + --port "$port" \ + --input-len 4000 \ + --output-len 512 \ + --num 2 \ + --env-python "${VENV_CLIENT}/bin/python" \ + >> "${log_dir_global}/${backend}.warmup.log" 2>&1 + log "warmup for ${backend} completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$(${VENV_CLIENT}/bin/python -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +run_benchmark() { + local backend="$1" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + local raw_dir="${result_root}/raw_outputs" + local phase_log_dir="${result_root}/logs" + + mkdir -p "$raw_dir" "$phase_log_dir" + + local port + if [[ "$backend" == "sglang" ]]; then + port="$SGLANG_PORT" + else + port="$VLLM_PORT" + fi + + log "===== ${backend} START (max_model_len=${MAX_MODEL_LEN}) =====" + + stop_server "$backend" + start_server "$backend" + run_warmup "$backend" + + for scenario in "${SCENARIOS[@]}"; do + read -r concurrency input_len output_len num_prompts <<< "$scenario" + output_file="${raw_dir}/${backend}_phase1_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl" + detail_log="${phase_log_dir}/${backend}_phase1_c${concurrency}_i${input_len}_o${output_len}.log" + + if scenario_already_completed "$output_file" "$num_prompts"; then + log "skipping already-completed ${backend} scenario: c=${concurrency} i=${input_len} o=${output_len}" + continue + fi + + log "running ${backend} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}" + + "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \ + --backend "$backend" \ + --host 127.0.0.1 \ + --port "$port" \ + --dataset-name random \ + --random-input-len "$input_len" \ + --random-output-len "$output_len" \ + --num-prompts "$num_prompts" \ + --max-concurrency "$concurrency" \ + --request-rate 10000 \ + --output-file "$output_file" \ + --output-details \ + > "$detail_log" 2>&1 || { + log "ERROR: ${backend} scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}" + continue + } + + log "finished ${backend} scenario: output=${output_file}" + done + + stop_server "$backend" + log "===== ${backend} DONE =====" +} + +parse_backend() { + local backend="$1" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + log "parsing ${backend} results in ${result_root}" + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$backend" \ + >> "${result_root}/logs/parse.log" 2>&1 || { + log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log" + } +} + +write_backend_metadata() { + local backend="$1" + local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" + ensure_result_root "$result_root" + local meta_json="${result_root}/results.json" + + local env_path + if [[ "$backend" == "sglang" ]]; then + env_path="$VENV_SGLANG" + else + env_path="$VENV_VLLM" + fi + + # Capture the exact server launch args for reproducibility. + local server_args + if [[ "$backend" == "sglang" ]]; then + server_args="sglang serve --trust-remote-code --model-path $MODEL_PATH --tp $TP --moe-runner-backend marlin --context-length $MAX_MODEL_LEN --max-running-requests $MAX_RUNNING --mem-fraction-static 0.88 --host 0.0.0.0 --port $SGLANG_PORT" + else + server_args="vllm serve $MODEL_PATH --trust-remote-code --tensor-parallel-size $TP --kv-cache-dtype fp8 --max-model-len $MAX_MODEL_LEN --max-num-seqs $MAX_NUM_SEQS --block-size 256 --gpu-memory-utilization 0.90 --tokenizer-mode deepseek_v4 --reasoning-parser deepseek_v4 --no-disable-hybrid-kv-cache-manager --disable-uvicorn-access-log --port $VLLM_PORT" + fi + + write_metadata_json \ + "$meta_json" \ + "${EXPERIMENT_NAME}_${backend}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "$backend" \ + "$backend" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$env_path" \ + "${HARDWARE} ${backend} TP=${TP} benchmark for ${MODEL_NAME}" + + jq --arg backend "$backend" \ + --arg server_args "$server_args" \ + '.config = { + "tp": ($TP | tonumber), + "cuda_visible_devices": $CUDA_VISIBLE_DEVICES, + "max_model_len": ($MAX_MODEL_LEN | tonumber), + "backend": $backend, + "server_start_script": "experiments/\($EXPERIMENT_NAME)/start_\($backend).sh", + "server_args": $server_args + }' "$meta_json" > "${meta_json}.tmp" && mv "${meta_json}.tmp" "$meta_json" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +stop_server sglang +stop_server vllm + +write_backend_metadata sglang +write_backend_metadata vllm + +run_benchmark sglang +parse_backend sglang + +run_benchmark vllm +parse_backend vllm + +log "generating comparison report" +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/compare.py" \ + --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \ + --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/TEMPLATE/start_sglang.sh b/experiments/TEMPLATE/start_sglang.sh new file mode 100755 index 0000000..0bd63aa --- /dev/null +++ b/experiments/TEMPLATE/start_sglang.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Start SGLang server for this experiment. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs + +export PATH="${VENV_SGLANG}/bin:$PATH" +export PYTHONUNBUFFERED=1 +export SGLANG_LOG_LEVEL=info +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/${EXPERIMENT}_sglang_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/${EXPERIMENT}_sglang.pid" + +rm -f "$PID_FILE" + +echo "=== Starting SGLang server (TP=$TP, max_model_len=$MAX_MODEL_LEN) ===" +echo "Model: $MODEL_PATH" +echo "Port: $SGLANG_PORT" +echo "Log: $LOG" + +nohup sglang serve \ + --trust-remote-code \ + --model-path "$MODEL_PATH" \ + --tp "$TP" \ + --moe-runner-backend marlin \ + --context-length "$MAX_MODEL_LEN" \ + --max-running-requests "$MAX_RUNNING" \ + --mem-fraction-static 0.88 \ + --host 0.0.0.0 \ + --port "$SGLANG_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then + echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: SGLang server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: SGLang server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/TEMPLATE/start_vllm.sh b/experiments/TEMPLATE/start_vllm.sh new file mode 100755 index 0000000..555b88b --- /dev/null +++ b/experiments/TEMPLATE/start_vllm.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# Start vLLM server for this experiment. +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +cd /data/user1/yy +mkdir -p logs + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR=/data/user1/yy/tmp +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="/data/user1/yy/logs/${EXPERIMENT}_vllm_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="/data/user1/yy/${EXPERIMENT}_vllm.pid" + +rm -f "$PID_FILE" + +echo "=== Starting vLLM server (TP=$TP, max_model_len=$MAX_MODEL_LEN) ===" +echo "Model: $MODEL_PATH" +echo "Port: $VLLM_PORT" +echo "Log: $LOG" + +nohup vllm serve "$MODEL_PATH" \ + --trust-remote-code \ + --tensor-parallel-size "$TP" \ + --kv-cache-dtype fp8 \ + --max-model-len "$MAX_MODEL_LEN" \ + --max-num-seqs "$MAX_NUM_SEQS" \ + --block-size 256 \ + --gpu-memory-utilization 0.90 \ + --tokenizer-mode deepseek_v4 \ + --reasoning-parser deepseek_v4 \ + --no-disable-hybrid-kv-cache-manager \ + --disable-uvicorn-access-log \ + --port "$VLLM_PORT" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/dsv4_h200_256k_4k_probe/run_bench.sh b/experiments/dsv4_h200_256k_4k_probe/run_bench.sh index c05cbc6..5b6b8ef 100644 --- a/experiments/dsv4_h200_256k_4k_probe/run_bench.sh +++ b/experiments/dsv4_h200_256k_4k_probe/run_bench.sh @@ -95,7 +95,7 @@ run_warmup() { num=1 log "warming up ${backend} (input=${input_len}, output=${output_len}, num=${num})" - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/warmup.py" \ + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ --backend "$backend" \ --port "$port" \ --input-len "$input_len" \ @@ -189,7 +189,7 @@ parse_backend() { local backend="$1" local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" log "parsing ${backend} results in ${result_root}" - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_backend.py" "$result_root" --backend "$backend" \ + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$backend" \ >> "${result_root}/logs/parse.log" 2>&1 || { log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log" } @@ -264,7 +264,7 @@ parse_backend vllm # Generate comparison. log "generating comparison report" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \ +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/compare.py" \ --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \ --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \ --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ diff --git a/experiments/dsv4_h200_64k_sglang_vs_vllm/run_bench.sh b/experiments/dsv4_h200_64k_sglang_vs_vllm/run_bench.sh index eaf41b5..532b5fb 100644 --- a/experiments/dsv4_h200_64k_sglang_vs_vllm/run_bench.sh +++ b/experiments/dsv4_h200_64k_sglang_vs_vllm/run_bench.sh @@ -95,7 +95,7 @@ run_warmup() { num=1 log "warming up ${backend} (input=${input_len}, output=${output_len}, num=${num})" - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/warmup.py" \ + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ --backend "$backend" \ --port "$port" \ --input-len "$input_len" \ @@ -189,7 +189,7 @@ parse_backend() { local backend="$1" local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" log "parsing ${backend} results in ${result_root}" - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_backend.py" "$result_root" --backend "$backend" \ + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$backend" \ >> "${result_root}/logs/parse.log" 2>&1 || { log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log" } @@ -264,7 +264,7 @@ parse_backend vllm # Generate comparison. log "generating comparison report" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \ +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/compare.py" \ --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \ --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \ --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ diff --git a/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh index 56d738d..4299eed 100755 --- a/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh +++ b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh @@ -100,7 +100,7 @@ run_warmup() { fi log "warming up ${backend} (phase=${phase}, input=${input_len}, output=${output_len}, num=${num})" - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/warmup.py" \ + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/warmup.py" \ --backend "$backend" \ --port "$port" \ --input-len "$input_len" \ @@ -264,7 +264,7 @@ parse_backend() { local backend="$1" local result_root="${RESULT_BASE}/${RUN_ID}/${backend}" log "parsing ${backend} results in ${result_root}" - "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_backend.py" "$result_root" --backend "$backend" \ + "${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend "$backend" \ >> "${result_root}/logs/parse.log" 2>&1 || { log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log" } @@ -348,7 +348,7 @@ parse_backend vllm # Generate comparison. log "generating comparison report" -"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \ +"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/../../scripts/common/compare.py" \ --sglang "${RESULT_BASE}/${RUN_ID}/sglang" \ --vllm "${RESULT_BASE}/${RUN_ID}/vllm" \ --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ diff --git a/scripts/common/compare.py b/scripts/common/compare.py new file mode 100755 index 0000000..a29dfec --- /dev/null +++ b/scripts/common/compare.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Generate a side-by-side comparison of SGLang and vLLM results. + +Usage: + python3 compare.py --sglang --vllm \ + [--output comparison.md] +""" +import argparse +import json +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "✅" + if ttft_ok or tpot_ok: + return "⚠️" + return "❌" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--sglang", type=Path, required=True) + parser.add_argument("--vllm", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + sglang_data = load_result(args.sglang) + vllm_data = load_result(args.vllm) + + model = sglang_data.get("metadata", {}).get("model", "unknown") + hardware = sglang_data.get("metadata", {}).get("hardware", "unknown") + + by_scenario = defaultdict(dict) + for data in (sglang_data, vllm_data): + backend = data["metadata"]["engine"] + for s in data.get("scenarios", []): + key = s["name"] + by_scenario[key][backend] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# SGLang vs vLLM ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + f.write("## Side-by-side results\n\n") + f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n") + f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for scenario_name in sorted(by_scenario.keys()): + for backend in ("sglang", "vllm"): + s = by_scenario[scenario_name].get(backend) + if s is None: + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/scripts/common/parse_backend.py b/scripts/common/parse_backend.py new file mode 100755 index 0000000..f07e6c6 --- /dev/null +++ b/scripts/common/parse_backend.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Parse raw sglang.bench_serving JSONL outputs for one backend. + +Reads JSONL files like {backend}_{label}_MMDD_concurrency_inputlen_outputlen.jsonl +and updates results.json + report.md in the given result root. + +Usage: + python3 parse_backend.py [--backend sglang|vllm] +""" +import argparse +import json +from pathlib import Path + + +def parse_jsonl(path: Path) -> dict | None: + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + try: + return json.loads(line) + except json.JSONDecodeError: + continue + return None + + +def compute_metrics(data: dict) -> dict: + completed = data.get("completed", 0) + total = len(data.get("input_lens", [])) + failed = total - completed if total > 0 else 0 + duration_s = data.get("duration", 0.0) + + return { + "success": completed, + "failed": failed, + "duration_s": duration_s, + "request_throughput": data.get("request_throughput", 0.0), + "input_token_throughput": data.get("input_throughput", 0.0), + "output_token_throughput": data.get("output_throughput", 0.0), + "total_token_throughput": data.get("total_throughput", 0.0), + "total_input_tokens": data.get("total_input_tokens", 0), + "total_output_tokens": data.get("total_output_tokens", 0), + "e2e_ms": { + "mean": data.get("mean_e2e_latency_ms", 0.0), + "p50": data.get("median_e2e_latency_ms", 0.0), + "p90": data.get("p90_e2e_latency_ms", 0.0), + "p95": data.get("p95_e2e_latency_ms", 0.0), + "p99": data.get("p99_e2e_latency_ms", 0.0), + }, + "ttft_ms": { + "mean": data.get("mean_ttft_ms", 0.0), + "p50": data.get("median_ttft_ms", 0.0), + "p90": data.get("p90_ttft_ms", 0.0), + "p95": data.get("p95_ttft_ms", 0.0), + "p99": data.get("p99_ttft_ms", 0.0), + }, + "tpot_ms": { + "mean": data.get("mean_tpot_ms", 0.0), + "p50": data.get("median_tpot_ms", 0.0), + "p90": data.get("p90_tpot_ms", 0.0), + "p95": data.get("p95_tpot_ms", 0.0), + "p99": data.get("p99_tpot_ms", 0.0), + }, + "itl_ms": { + "mean": data.get("mean_itl_ms", 0.0), + "p50": data.get("median_itl_ms", 0.0), + "p90": data.get("p90_itl_ms", 0.0), + "p95": data.get("p95_itl_ms", 0.0), + "p99": data.get("p99_itl_ms", 0.0), + }, + } + + +def scenario_name(concurrency: int, input_len: int, output_len: int) -> str: + return f"c{concurrency}_i{input_len}_o{output_len}" + + +def slo_status(metrics: dict, ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> dict: + """Check SLO: TTFT P95 < limit, TPOT mean < limit. Defaults to S2 tier.""" + ttft_ok = metrics["ttft_ms"]["p95"] < ttft_limit_ms + tpot_ok = metrics["tpot_ms"]["mean"] < tpot_limit_ms + if ttft_ok and tpot_ok: + mark = "✅" + elif ttft_ok or tpot_ok: + mark = "⚠️" + else: + mark = "❌" + return { + "ttft_p95_ok": ttft_ok, + "tpot_mean_ok": tpot_ok, + "overall": mark, + } + + +def generate_report(result_root: Path, backend: str, scenarios: list[dict], metadata: dict | None) -> None: + report_path = result_root / "report.md" + model = metadata.get("model", "unknown") if metadata else "unknown" + hardware = metadata.get("hardware", "unknown") if metadata else "unknown" + + with open(report_path, "w", encoding="utf-8") as f: + f.write(f"# {hardware} {backend.upper()} Benchmark Report\n\n") + f.write(f"- Result root: `{result_root}`\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Backend: {backend.upper()}\n") + f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n") + + f.write("## Results\n\n") + f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n") + f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n") + + for s in scenarios: + cfg = s["config"] + m = s["metrics"] + slo = s.get("slo_status", {}).get("overall", "") + f.write( + f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | " + f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | " + f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['total_token_throughput']:.2f} | " + f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n" + ) + f.write("\n") + f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("result_root", type=Path) + parser.add_argument("--backend", default=None, choices=["sglang", "vllm"]) + args = parser.parse_args() + + result_root = args.result_root + raw_dir = result_root / "raw_outputs" + results_json = result_root / "results.json" + + if not raw_dir.exists(): + raise SystemExit(f"raw_outputs directory not found: {raw_dir}") + + backend = args.backend + if backend is None: + for p in raw_dir.iterdir(): + if p.name.startswith("sglang_"): + backend = "sglang" + break + if p.name.startswith("vllm_"): + backend = "vllm" + break + if backend is None: + raise SystemExit("Could not infer backend from raw outputs") + + metadata = None + if results_json.exists(): + with open(results_json, "r", encoding="utf-8") as f: + try: + metadata = json.load(f).get("metadata") + except json.JSONDecodeError: + pass + + scenarios = [] + for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")): + parts = jsonl_path.stem.split("_") + if len(parts) < 5: + continue + + label = parts[1] + if label == "sharegpt": + phase = "sharegpt" + dataset = "sharegpt" + else: + phase = label + dataset = "random" + + try: + concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1]) + except ValueError: + continue + + data = parse_jsonl(jsonl_path) + if data is None: + continue + + metrics = compute_metrics(data) + scenario = { + "name": scenario_name(concurrency, input_len, output_len), + "config": { + "phase": phase, + "concurrency": concurrency, + "input_len": input_len, + "output_len": output_len, + "dataset": dataset, + "num_prompts": metrics["success"] + metrics["failed"], + }, + "metrics": metrics, + "slo_status": slo_status(metrics), + "raw_file": str(jsonl_path), + } + scenarios.append(scenario) + + if not scenarios: + print("No benchmark outputs found to parse") + return + + if results_json.exists(): + with open(results_json, "r", encoding="utf-8") as f: + data = json.load(f) + data["scenarios"] = scenarios + with open(results_json, "w", encoding="utf-8") as f: + json.dump(data, f, indent=2, ensure_ascii=False) + + generate_report(result_root, backend, scenarios, metadata) + print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md") + + +if __name__ == "__main__": + main() diff --git a/scripts/common/warmup.py b/scripts/common/warmup.py new file mode 100755 index 0000000..75dbe5f --- /dev/null +++ b/scripts/common/warmup.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Send a small number of warmup requests to a running backend. + +Uses sglang.bench_serving with a single request so that the same code path +(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real +benchmark begins. Discards the output. +""" +import argparse +import subprocess +import sys +import tempfile +from pathlib import Path + + +def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None: + with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp: + cmd = [ + str(env_python), + "-m", + "sglang.bench_serving", + "--backend", + backend, + "--host", + host, + "--port", + str(port), + "--dataset-name", + "random", + "--random-input-len", + str(input_len), + "--random-output-len", + str(output_len), + "--num-prompts", + str(num), + "--max-concurrency", + "1", + "--request-rate", + "10000", + "--output-file", + tmp.name, + "--output-details", + ] + print(f"[warmup] {' '.join(cmd)}", flush=True) + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + print("[warmup] FAILED", file=sys.stderr) + print(result.stdout, file=sys.stderr) + print(result.stderr, file=sys.stderr) + sys.exit(1) + print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}") + + +def main(): + parser = argparse.ArgumentParser(description="Warmup a serving backend.") + parser.add_argument("--backend", required=True, choices=["sglang", "vllm"]) + parser.add_argument("--port", type=int, required=True) + parser.add_argument("--input-len", type=int, required=True) + parser.add_argument("--output-len", type=int, required=True) + parser.add_argument("--num", type=int, default=1) + parser.add_argument("--host", default="127.0.0.1") + parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python") + args = parser.parse_args() + + run_warmup( + backend=args.backend, + host=args.host, + port=args.port, + input_len=args.input_len, + output_len=args.output_len, + num=args.num, + env_python=Path(args.env_python), + ) + + +if __name__ == "__main__": + main()