qqtang/sskj
1
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(long-context-matrix): add SGLang vs vLLM long-context matrix experiment

Browse Source
This commit is contained in:
yy-fighting 2026-07-08 10:09:32 +00:00
parent cd2ba29d49
commit 868ebc6cdf
7 changed files with 870 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

85
experiments/dsv4_h200_long_context_matrix/compare.py Executable file
View File

@ -0,0 +1,85 @@
#!/usr/bin/env python3
"""Generate a side-by-side comparison of SGLang and vLLM results.
Usage:
python3 compare.py --sglang <sglang_result_root> --vllm <vllm_result_root> \
[--output comparison.md]
"""
import argparse
import json
from collections import defaultdict
from pathlib import Path
def load_result(result_root: Path) -> dict:
path = result_root / "results.json"
with open(path, "r", encoding="utf-8") as f:
return json.load(f)
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float) -> str:
# S2 tier targets from scripts/SLO_STANDARDS.md.
ttft_ok = ttft_p95_ms < 3000.0
tpot_ok = tpot_mean_ms < 50.0
if ttft_ok and tpot_ok:
return "✅"
if ttft_ok or tpot_ok:
return "⚠️"
return "❌"
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--sglang", type=Path, required=True)
parser.add_argument("--vllm", type=Path, required=True)
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
args = parser.parse_args()
sglang_data = load_result(args.sglang)
vllm_data = load_result(args.vllm)
by_scenario = defaultdict(dict)
for data in (sglang_data, vllm_data):
backend = data["metadata"]["engine"]
for s in data.get("scenarios", []):
key = s["name"]
by_scenario[key][backend] = s
with open(args.output, "w", encoding="utf-8") as f:
f.write("# SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8)\n\n")
f.write("## Summary\n\n")
f.write("- Model: `/data/models/DeepSeek-V4-Flash`\n")
f.write("- Hardware: 8x NVIDIA H200 143GB\n")
f.write("- Tensor Parallelism: 8\n")
f.write("- Benchmark client: `sglang.bench_serving`\n")
f.write("- SLO reference: S2 tier (TTFT P95 < 3s, TPOT < 50ms)\n\n")
f.write("## Side-by-side results\n\n")
f.write("| Scenario | Backend | Conc | Input | Output | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | SLO |\n")
f.write("|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
for scenario_name in sorted(by_scenario.keys()):
for backend in ("sglang", "vllm"):
s = by_scenario[scenario_name].get(backend)
if s is None:
continue
cfg = s["config"]
m = s["metrics"]
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"])
f.write(
f"| {scenario_name} | {backend} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
f"{m['e2e_ms']['p99']:.2f} | {status} |\n"
)
f.write("\n## Notes\n\n")
f.write("- SLO check uses TTFT P95 and TPOT mean (the same criteria as `scripts/SLO_STANDARDS.md`).\n")
f.write("- A ⚠️ indicates one of the two metrics is out of target; ❌ indicates both are out.\n")
print(f"Wrote comparison to {args.output}")
if __name__ == "__main__":
main()

62
experiments/dsv4_h200_long_context_matrix/config.env Normal file
View File

@ -0,0 +1,62 @@
# Long-context matrix for SGLang vs vLLM on DeepSeek-V4-Flash (H200, TP=8).
# Input lengths: 64k, 128k, 256k
# Output lengths: 256, 1k, 4k
# Concurrency: 25 for 64k; 10, 5, 2 for 128k and 256k
EXPERIMENT="dsv4_h200_long_context_matrix"
MODEL_NAME="DeepSeek-V4-Flash"
MODEL_PATH="/data/models/DeepSeek-V4-Flash"
SERVED_MODEL_NAME="deepseek-v4-flash"
SGLANG_PORT="${SGLANG_PORT:-30006}"
VLLM_PORT="${VLLM_PORT:-30005}"
VENV_SGLANG="${VENV_SGLANG:-/data/user1/yy/envs/sglang}"
VENV_VLLM="${VENV_VLLM:-/data/user1/yy/envs/vllm}"
export CUDA_VISIBLE_DEVICES="0,1,2,3,4,5,6,7"
TP=8
# Per-context server settings. Each group is run with its own server start.
# Format: "input_label max_model_len max_concurrency"
declare -a CONTEXT_GROUPS=(
"64k 80000 25"
"128k 140000 10"
"256k 270000 10"
)
# Scenarios per group: "concurrency input_len output_len num_prompts"
declare -a SCENARIOS_64K=(
"25 65536 256 50"
"25 65536 1024 50"
"25 65536 4096 50"
)
declare -a SCENARIOS_128K=(
"10 131072 256 20"
"5 131072 256 10"
"2 131072 256 4"
"10 131072 1024 20"
"5 131072 1024 10"
"2 131072 1024 4"
"10 131072 4096 20"
"5 131072 4096 10"
"2 131072 4096 4"
)
declare -a SCENARIOS_256K=(
"10 262144 256 20"
"5 262144 256 10"
"2 262144 256 4"
"10 262144 1024 20"
"5 262144 1024 10"
"2 262144 1024 4"
"10 262144 4096 20"
"5 262144 4096 10"
"2 262144 4096 4"
)
VENV_CLIENT="${VENV_CLIENT:-$VENV_SGLANG}"
SGLANG_START_SCRIPT="${SCRIPT_DIR:-.}/start_sglang.sh"
VLLM_START_SCRIPT="${SCRIPT_DIR:-.}/start_vllm.sh"

219
experiments/dsv4_h200_long_context_matrix/parse_backend.py Executable file
View File

@ -0,0 +1,219 @@
#!/usr/bin/env python3
"""Parse raw sglang.bench_serving JSONL outputs for one backend.
Reads JSONL files like {sglang|vllm}_phase1_MMDD_concurrency_inputlen_outputlen.jsonl
and generates results.json + report.md in the given result root.
Usage:
python3 parse_backend.py <result_root> [--backend sglang|vllm]
"""
import argparse
import json
import re
import sys
from pathlib import Path
def parse_jsonl(path: Path) -> dict | None:
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
return json.loads(line)
except json.JSONDecodeError:
continue
return None
def compute_metrics(data: dict) -> dict:
completed = data.get("completed", 0)
total = len(data.get("input_lens", []))
failed = total - completed if total > 0 else 0
duration_s = data.get("duration", 0.0)
return {
"success": completed,
"failed": failed,
"duration_s": duration_s,
"request_throughput": data.get("request_throughput", 0.0),
"input_token_throughput": data.get("input_throughput", 0.0),
"output_token_throughput": data.get("output_throughput", 0.0),
"total_token_throughput": data.get("total_throughput", 0.0),
"total_input_tokens": data.get("total_input_tokens", 0),
"total_output_tokens": data.get("total_output_tokens", 0),
"e2e_ms": {
"mean": data.get("mean_e2e_latency_ms", 0.0),
"p50": data.get("median_e2e_latency_ms", 0.0),
"p90": data.get("p90_e2e_latency_ms", 0.0),
"p95": data.get("p95_e2e_latency_ms", 0.0),
"p99": data.get("p99_e2e_latency_ms", 0.0),
},
"ttft_ms": {
"mean": data.get("mean_ttft_ms", 0.0),
"p50": data.get("median_ttft_ms", 0.0),
"p90": data.get("p90_ttft_ms", 0.0),
"p95": data.get("p95_ttft_ms", 0.0),
"p99": data.get("p99_ttft_ms", 0.0),
},
"tpot_ms": {
"mean": data.get("mean_tpot_ms", 0.0),
"p50": data.get("median_tpot_ms", 0.0),
"p90": data.get("p90_tpot_ms", 0.0),
"p95": data.get("p95_tpot_ms", 0.0),
"p99": data.get("p99_tpot_ms", 0.0),
},
"itl_ms": {
"mean": data.get("mean_itl_ms", 0.0),
"p50": data.get("median_itl_ms", 0.0),
"p90": data.get("p90_itl_ms", 0.0),
"p95": data.get("p95_itl_ms", 0.0),
"p99": data.get("p99_itl_ms", 0.0),
},
}
def scenario_name(concurrency: int, input_len: int, output_len: int) -> str:
return f"c{concurrency}_i{input_len}_o{output_len}"
def slo_status(metrics: dict) -> dict:
"""Check S2-tier SLO: TTFT P95 < 3s, TPOT mean < 50ms."""
ttft_ok = metrics["ttft_ms"]["p95"] < 3000.0
tpot_ok = metrics["tpot_ms"]["mean"] < 50.0
if ttft_ok and tpot_ok:
mark = "✅"
elif ttft_ok or tpot_ok:
mark = "⚠️"
else:
mark = "❌"
return {
"ttft_p95_ok": ttft_ok,
"tpot_mean_ok": tpot_ok,
"overall": mark,
}
def append_scenario(results_json: Path, scenario: dict) -> None:
with open(results_json, "r", encoding="utf-8") as f:
data = json.load(f)
data["scenarios"].append(scenario)
with open(results_json, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def generate_report(result_root: Path, backend: str, scenarios: list[dict]) -> None:
report_path = result_root / "report.md"
with open(report_path, "w", encoding="utf-8") as f:
f.write(f"# H200 {backend.upper()} Comparison Benchmark Report\n\n")
f.write(f"- Result root: `{result_root}`\n")
f.write(f"- Model: `/data/models/DeepSeek-V4-Flash`\n")
f.write(f"- Backend: {backend.upper()} (TP=8)\n")
f.write(f"- Benchmark client: `sglang.bench_serving --backend {backend}`\n\n")
f.write("## Results\n\n")
f.write("| Scenario | Phase | Concurrency | Input | Output | Duration(s) | Success | Req/s | In tok/s | Out tok/s | Total tok/s | Mean TTFT(ms) | P95 TTFT(ms) | P99 TTFT(ms) | Mean TPOT(ms) | P95 TPOT(ms) | P99 TPOT(ms) | Mean E2E(ms) | P95 E2E(ms) | P99 E2E(ms) | SLO |\n")
f.write("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
for s in scenarios:
cfg = s["config"]
m = s["metrics"]
slo = s.get("slo_status", {}).get("overall", "")
f.write(
f"| {s['name']} | {cfg['phase']} | {cfg['concurrency']} | {cfg['input_len']} | {cfg['output_len']} | "
f"{m['duration_s']:.2f} | {m['success']} | {m['request_throughput']:.2f} | "
f"{m['input_token_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
f"{m['total_token_throughput']:.2f} | "
f"{m['ttft_ms']['mean']:.2f} | {m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
f"{m['e2e_ms']['mean']:.2f} | {m['e2e_ms']['p95']:.2f} | {m['e2e_ms']['p99']:.2f} | {slo} |\n"
)
f.write("\n")
f.write("SLO: S2 tier — TTFT P95 < 3000ms, TPOT mean < 50ms. ✅ pass, ⚠️ partial, ❌ fail.\n\n")
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("result_root", type=Path)
parser.add_argument("--backend", default=None, choices=["sglang", "vllm"])
args = parser.parse_args()
result_root = args.result_root
raw_dir = result_root / "raw_outputs"
results_json = result_root / "results.json"
if not raw_dir.exists():
raise SystemExit(f"raw_outputs directory not found: {raw_dir}")
backend = args.backend
if backend is None:
# Infer from filenames if not provided.
for p in raw_dir.iterdir():
if p.name.startswith("sglang_"):
backend = "sglang"
break
if p.name.startswith("vllm_"):
backend = "vllm"
break
if backend is None:
raise SystemExit("Could not infer backend from raw outputs")
scenarios = []
for jsonl_path in sorted(raw_dir.glob(f"{backend}_*.jsonl")):
parts = jsonl_path.stem.split("_")
if len(parts) < 5:
continue
label = parts[1]
if label == "sharegpt":
phase = "sharegpt"
dataset = "sharegpt"
else:
phase = label
dataset = "random"
# The last three numeric tokens are: concurrency, input_len/output_ctx, output_len.
try:
concurrency, input_len, output_len = int(parts[-3]), int(parts[-2]), int(parts[-1])
except ValueError:
continue
data = parse_jsonl(jsonl_path)
if data is None:
continue
metrics = compute_metrics(data)
scenario = {
"name": scenario_name(concurrency, input_len, output_len),
"config": {
"phase": phase,
"concurrency": concurrency,
"input_len": input_len,
"output_len": output_len,
"dataset": dataset,
"num_prompts": metrics["success"] + metrics["failed"],
},
"metrics": metrics,
"slo_status": slo_status(metrics),
"raw_file": str(jsonl_path),
}
scenarios.append(scenario)
if not scenarios:
print("No benchmark outputs found to parse")
return
if results_json.exists():
with open(results_json, "r", encoding="utf-8") as f:
data = json.load(f)
data["scenarios"] = scenarios
with open(results_json, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
generate_report(result_root, backend, scenarios)
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
if __name__ == "__main__":
main()

293
experiments/dsv4_h200_long_context_matrix/run_bench.sh Executable file
View File

@ -0,0 +1,293 @@
#!/usr/bin/env bash
# Long-context matrix: SGLang vs vLLM on DeepSeek-V4-Flash (H200).
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
RESULT_BASE="${SCRIPT_DIR}/results"
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
mkdir -p "$log_dir_global"
log_init "${log_dir_global}/orchestrator.log"
log "experiment=${EXPERIMENT_NAME}"
log "run_id=${RUN_ID}"
log "platform=${PLATFORM}"
log "hardware=${HARDWARE}"
log "model=${MODEL_PATH}"
log "groups=${#CONTEXT_GROUPS[@]}"
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
is_server_healthy() {
local port="$1"
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${port}/health" >/dev/null 2>&1
}
stop_server() {
local backend="$1"
local pid_file="/data/user1/yy/dsv4_h200_long_context_matrix_${backend}.pid"
if [[ -f "$pid_file" ]]; then
local pid
pid="$(cat "$pid_file")"
if kill -0 "$pid" 2>/dev/null; then
log "stopping ${backend} server pid=${pid}"
kill "$pid" 2>/dev/null || true
sleep 5
kill -9 "$pid" 2>/dev/null || true
fi
rm -f "$pid_file"
fi
# Fallback cleanup.
if [[ "$backend" == "sglang" ]]; then
pkill -9 -f "sglang serve.*DeepSeek-V4-Flash" 2>/dev/null || true
else
pkill -9 -f "vllm serve.*DeepSeek-V4-Flash" 2>/dev/null || true
fi
sleep 2
}
start_server() {
local backend="$1"
local max_model_len="$2"
local max_slots="$3"
local start_script
if [[ "$backend" == "sglang" ]]; then
start_script="${SGLANG_START_SCRIPT}"
local port="$SGLANG_PORT"
else
start_script="${VLLM_START_SCRIPT}"
local port="$VLLM_PORT"
fi
log "starting ${backend} server (max_model_len=${max_model_len}, slots=${max_slots}) with ${start_script}"
bash "${start_script}" "$max_model_len" "$max_slots" >> "${log_dir_global}/${backend}_${max_model_len}.server.outer.log" 2>&1
if ! is_server_healthy "$port"; then
log "error: ${backend} server failed to become healthy"
return 1
fi
log "${backend} server is healthy on port ${port}"
}
run_warmup() {
local backend="$1"
local input_len="$2"
local output_len="${3:-256}"
local port
if [[ "$backend" == "sglang" ]]; then
port="$SGLANG_PORT"
else
port="$VLLM_PORT"
fi
log "warming up ${backend} (input=${input_len}, output=${output_len}, num=1)"
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/warmup.py" \
--backend "$backend" \
--port "$port" \
--input-len "$input_len" \
--output-len "$output_len" \
--num 1 \
--env-python "${VENV_CLIENT}/bin/python" \
>> "${log_dir_global}/${backend}.warmup.log" 2>&1
log "warmup for ${backend} completed"
}
scenario_already_completed() {
local output_file="$1"
local expected="$2"
[[ -s "$output_file" ]] || return 1
local completed
completed="$(${VENV_CLIENT}/bin/python -c "
import json, sys
path = sys.argv[1]
try:
with open(path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line:
data = json.loads(line)
print(data.get('completed', 0))
break
except Exception:
print(0)
" "$output_file")"
[[ "${completed:-0}" -ge "$expected" ]]
}
run_group() {
local backend="$1"
local group_label="$2"
local max_model_len="$3"
local max_slots="$4"
local -n scenarios_ref="$5"
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
local raw_dir="${result_root}/raw_outputs"
local phase_log_dir="${result_root}/logs"
mkdir -p "$raw_dir" "$phase_log_dir"
local port
if [[ "$backend" == "sglang" ]]; then
port="$SGLANG_PORT"
else
port="$VLLM_PORT"
fi
log "===== ${backend} ${group_label} START (max_model_len=${max_model_len}) ====="
stop_server "$backend"
if ! start_server "$backend" "$max_model_len" "$max_slots"; then
log "ERROR: ${backend} failed to start for ${group_label}; skipping group"
stop_server "$backend"
return 1
fi
# Warmup with the shortest output for this input length to save time.
local warmup_input="${scenarios_ref[0]}"
local warmup_input_len
warmup_input_len="$(echo "$warmup_input" | awk '{print $2}')"
run_warmup "$backend" "$warmup_input_len" 256
for scenario in "${scenarios_ref[@]}"; do
read -r concurrency input_len output_len num_prompts <<< "$scenario"
output_file="${raw_dir}/${backend}_${group_label}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
detail_log="${phase_log_dir}/${backend}_${group_label}_c${concurrency}_i${input_len}_o${output_len}.log"
if scenario_already_completed "$output_file" "$num_prompts"; then
log "skipping already-completed ${backend} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len}"
continue
fi
log "running ${backend} ${group_label} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
--backend "$backend" \
--host 127.0.0.1 \
--port "$port" \
--dataset-name random \
--random-input-len "$input_len" \
--random-output-len "$output_len" \
--num-prompts "$num_prompts" \
--max-concurrency "$concurrency" \
--request-rate 10000 \
--output-file "$output_file" \
--output-details \
> "$detail_log" 2>&1 || {
log "ERROR: ${backend} ${group_label} scenario c=${concurrency} i=${input_len} o=${output_len} failed; see ${detail_log}"
continue
}
log "finished ${backend} ${group_label} scenario: output=${output_file}"
done
stop_server "$backend"
log "===== ${backend} ${group_label} DONE ====="
}
parse_backend() {
local backend="$1"
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
log "parsing ${backend} results in ${result_root}"
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/parse_backend.py" "$result_root" --backend "$backend" \
>> "${result_root}/logs/parse.log" 2>&1 || {
log "WARNING: parser failed for ${backend}; see ${result_root}/logs/parse.log"
}
}
write_backend_metadata() {
local backend="$1"
local result_root="${RESULT_BASE}/${RUN_ID}/${backend}"
ensure_result_root "$result_root"
local meta_json="${result_root}/results.json"
local env_path
if [[ "$backend" == "sglang" ]]; then
env_path="$VENV_SGLANG"
else
env_path="$VENV_VLLM"
fi
write_metadata_json \
"$meta_json" \
"${EXPERIMENT_NAME}_${backend}" \
"$RUN_ID" \
"$MODEL_PATH" \
"$backend" \
"$backend" \
"$HARDWARE" \
"$ACCELERATOR" \
"$CHIP" \
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
"$env_path" \
"H200 long-context matrix ${backend} TP=8 for DeepSeek-V4-Flash"
# Embed config.
jq --arg backend "$backend" \
'.config = {
"tp": 8,
"cuda_visible_devices": "0,1,2,3,4,5,6,7",
"backend": $backend,
"server_start_script": "experiments/dsv4_h200_long_context_matrix/start_\($backend).sh",
"groups": [
{"label": "64k", "max_model_len": 80000, "max_slots": 25},
{"label": "128k", "max_model_len": 140000, "max_slots": 10},
{"label": "256k", "max_model_len": 270000, "max_slots": 10}
]
}' "$meta_json" > "${meta_json}.tmp" && mv "${meta_json}.tmp" "$meta_json"
}
# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------
# Cleanup any leftovers.
stop_server sglang
stop_server vllm
write_backend_metadata sglang
write_backend_metadata vllm
for group in "${CONTEXT_GROUPS[@]}"; do
read -r group_label max_model_len max_slots <<< "$group"
# Resolve scenario array for this group.
scenario_array_name="SCENARIOS_${group_label^^}"
if [[ -z "${!scenario_array_name:-}" ]]; then
log "WARNING: no scenarios defined for group ${group_label}"
continue
fi
# Run SGLang for this group.
run_group sglang "$group_label" "$max_model_len" "$max_slots" "$scenario_array_name"
# Run vLLM for this group.
run_group vllm "$group_label" "$max_model_len" "$max_slots" "$scenario_array_name"
done
# Parse and compare.
parse_backend sglang
parse_backend vllm
log "generating comparison report"
"${VENV_CLIENT}/bin/python" "${SCRIPT_DIR}/compare.py" \
--sglang "${RESULT_BASE}/${RUN_ID}/sglang" \
--vllm "${RESULT_BASE}/${RUN_ID}/vllm" \
--output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
>> "${log_dir_global}/compare.log" 2>&1 || {
log "WARNING: comparison script failed; see ${log_dir_global}/compare.log"
}
log "all results saved to ${RESULT_BASE}/${RUN_ID}"

66
experiments/dsv4_h200_long_context_matrix/start_sglang.sh Executable file
View File

@ -0,0 +1,66 @@
#!/bin/bash
# Start SGLang server for the long-context matrix.
# Usage: start_sglang.sh <max_model_len> <max_running>
set -e
MAX_MODEL_LEN="${1:-80000}"
MAX_RUNNING="${2:-64}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
cd /data/user1/yy
mkdir -p logs
export PATH="${VENV_SGLANG}/bin:$PATH"
export PYTHONUNBUFFERED=1
export SGLANG_LOG_LEVEL=info
export TMPDIR=/data/user1/yy/tmp
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
LOG="/data/user1/yy/logs/dsv4_h200_long_context_matrix_sglang_${MAX_MODEL_LEN}_$(date +%Y%m%d_%H%M%S).log"
PID_FILE="/data/user1/yy/dsv4_h200_long_context_matrix_sglang.pid"
rm -f "$PID_FILE"
echo "=== Starting SGLang server (TP=$TP, max_model_len=$MAX_MODEL_LEN, max_running=$MAX_RUNNING) ==="
echo "Model: $MODEL_PATH"
echo "Port: $SGLANG_PORT"
echo "Log: $LOG"
nohup sglang serve \
--trust-remote-code \
--model-path "$MODEL_PATH" \
--tp "$TP" \
--moe-runner-backend marlin \
--context-length "$MAX_MODEL_LEN" \
--max-running-requests "$MAX_RUNNING" \
--mem-fraction-static 0.88 \
--host 0.0.0.0 \
--port "$SGLANG_PORT" \
> "$LOG" 2>&1 &
PID=$!
echo $PID > "$PID_FILE"
echo "PID: $PID"
echo "Waiting for health..."
for i in $(seq 1 240); do
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${SGLANG_PORT}/health" >/dev/null 2>&1; then
echo "SGLang server is ready at http://127.0.0.1:${SGLANG_PORT}"
echo "Log: $LOG"
exit 0
fi
if ! kill -0 $PID 2>/dev/null; then
echo "ERROR: SGLang server exited early"
tail -200 "$LOG"
exit 1
fi
echo "Waiting... ($i/240)"
sleep 5
done
echo "ERROR: SGLang server not healthy after 240 retries"
tail -200 "$LOG"
exit 1

69
experiments/dsv4_h200_long_context_matrix/start_vllm.sh Executable file
View File

@ -0,0 +1,69 @@
#!/bin/bash
# Start vLLM server for the long-context matrix.
# Usage: start_vllm.sh <max_model_len> <max_num_seqs>
set -e
MAX_MODEL_LEN="${1:-80000}"
MAX_NUM_SEQS="${2:-64}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env"
cd /data/user1/yy
mkdir -p logs
VENV="${VENV_VLLM}"
export PATH="$VENV/bin:$PATH"
export PYTHONUNBUFFERED=1
export TMPDIR=/data/user1/yy/tmp
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
LOG="/data/user1/yy/logs/dsv4_h200_long_context_matrix_vllm_${MAX_MODEL_LEN}_$(date +%Y%m%d_%H%M%S).log"
PID_FILE="/data/user1/yy/dsv4_h200_long_context_matrix_vllm.pid"
rm -f "$PID_FILE"
echo "=== Starting vLLM server (TP=$TP, max_model_len=$MAX_MODEL_LEN, max_num_seqs=$MAX_NUM_SEQS) ==="
echo "Model: $MODEL_PATH"
echo "Port: $VLLM_PORT"
echo "Log: $LOG"
nohup vllm serve "$MODEL_PATH" \
--trust-remote-code \
--tensor-parallel-size "$TP" \
--kv-cache-dtype fp8 \
--max-model-len "$MAX_MODEL_LEN" \
--max-num-seqs "$MAX_NUM_SEQS" \
--block-size 256 \
--gpu-memory-utilization 0.90 \
--tokenizer-mode deepseek_v4 \
--reasoning-parser deepseek_v4 \
--no-disable-hybrid-kv-cache-manager \
--disable-uvicorn-access-log \
--port "$VLLM_PORT" \
> "$LOG" 2>&1 &
PID=$!
echo $PID > "$PID_FILE"
echo "PID: $PID"
echo "Waiting for health..."
for i in $(seq 1 240); do
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then
echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}"
echo "Log: $LOG"
exit 0
fi
if ! kill -0 $PID 2>/dev/null; then
echo "ERROR: vLLM server exited early"
tail -200 "$LOG"
exit 1
fi
echo "Waiting... ($i/240)"
sleep 5
done
echo "ERROR: vLLM server not healthy after 240 retries"
tail -200 "$LOG"
exit 1

76
experiments/dsv4_h200_long_context_matrix/warmup.py Executable file
View File

@ -0,0 +1,76 @@
#!/usr/bin/env python3
"""Send a small number of warmup requests to a running backend.
Uses sglang.bench_serving with a single request so that the same code path
(prefill / decode kernels, CUDA graphs, etc.) is exercised before the real
benchmark begins. Discards the output.
"""
import argparse
import subprocess
import sys
import tempfile
from pathlib import Path
def run_warmup(backend: str, host: str, port: int, input_len: int, output_len: int, num: int, env_python: Path) -> None:
with tempfile.NamedTemporaryFile(mode="w", suffix=".jsonl", delete=True) as tmp:
cmd = [
str(env_python),
"-m",
"sglang.bench_serving",
"--backend",
backend,
"--host",
host,
"--port",
str(port),
"--dataset-name",
"random",
"--random-input-len",
str(input_len),
"--random-output-len",
str(output_len),
"--num-prompts",
str(num),
"--max-concurrency",
"1",
"--request-rate",
"10000",
"--output-file",
tmp.name,
"--output-details",
]
print(f"[warmup] {' '.join(cmd)}", flush=True)
result = subprocess.run(cmd, capture_output=True, text=True)
if result.returncode != 0:
print("[warmup] FAILED", file=sys.stderr)
print(result.stdout, file=sys.stderr)
print(result.stderr, file=sys.stderr)
sys.exit(1)
print(f"[warmup] OK: backend={backend} port={port} input={input_len} output={output_len} num={num}")
def main():
parser = argparse.ArgumentParser(description="Warmup a serving backend.")
parser.add_argument("--backend", required=True, choices=["sglang", "vllm"])
parser.add_argument("--port", type=int, required=True)
parser.add_argument("--input-len", type=int, required=True)
parser.add_argument("--output-len", type=int, required=True)
parser.add_argument("--num", type=int, default=1)
parser.add_argument("--host", default="127.0.0.1")
parser.add_argument("--env-python", default="/data/user1/yy/envs/sglang/bin/python")
args = parser.parse_args()
run_warmup(
backend=args.backend,
host=args.host,
port=args.port,
input_len=args.input_len,
output_len=args.output_len,
num=args.num,
env_python=Path(args.env_python),
)
if __name__ == "__main__":
main()