feat: support resuming interrupted SGLang vs vLLM runs

- run_bench.sh: skip scenarios whose raw jsonl already exists and has expected completed count
- parse_backend.py: replace scenarios list instead of appending, so resume+reparse stays clean
- README.md: document RUN_ID-based resume
This commit is contained in:
yy-fighting 2026-07-08 08:42:21 +00:00
parent fd0474099a
commit 5eb605b7a2
3 changed files with 47 additions and 2 deletions

View File

@ -47,6 +47,16 @@
bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
```
### 断点续跑
如果实验中途因为脚本/机器原因中断,可以复用同一个 `RUN_ID` 继续跑,已经完成的 scenario 会自动跳过:
```bash
RUN_ID=20260708-XXXXXX bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
```
判断标准是 `raw_outputs/*.jsonl` 已存在且 `completed` 数量达到预期。
结果保存在 `experiments/dsv4_h200_sglang_vs_vllm/results/<RUN_ID>/`
```

View File

@ -207,8 +207,11 @@ def main() -> None:
return
if results_json.exists():
for s in scenarios:
append_scenario(results_json, s)
with open(results_json, "r", encoding="utf-8") as f:
data = json.load(f)
data["scenarios"] = scenarios
with open(results_json, "w", encoding="utf-8") as f:
json.dump(data, f, indent=2, ensure_ascii=False)
generate_report(result_root, backend, scenarios)
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")

View File

@ -111,6 +111,28 @@ run_warmup() {
log "warmup for ${backend} ${phase} completed"
}
scenario_already_completed() {
local output_file="$1"
local expected="$2"
[[ -s "$output_file" ]] || return 1
local completed
completed="$(${VENV_CLIENT}/bin/python -c "
import json, sys
path = sys.argv[1]
try:
with open(path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip()
if line:
data = json.loads(line)
print(data.get('completed', 0))
break
except Exception:
print(0)
" "$output_file")"
[[ "${completed:-0}" -ge "$expected" ]]
}
run_phase() {
local backend="$1"
local phase="$2"
@ -148,6 +170,11 @@ run_phase() {
output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_i${input_len}_o${output_len}.log"
if scenario_already_completed "$output_file" "$num_prompts"; then
log "skipping already-completed ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len}"
continue
fi
log "running ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
@ -201,6 +228,11 @@ run_sharegpt() {
output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${context_len}_${output_len}.jsonl"
detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_ctx${context_len}_o${output_len}.log"
if scenario_already_completed "$output_file" "$num_prompts"; then
log "skipping already-completed ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len}"
continue
fi
log "running ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len} n=${num_prompts}"
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \