From 5eb605b7a241b6a41c2dd685a17d55d2bfdbd5f7 Mon Sep 17 00:00:00 2001
From: yy-fighting <yy-fighting@noreply.gitee.com>
Date: Wed, 8 Jul 2026 08:42:21 +0000
Subject: [PATCH] feat: support resuming interrupted SGLang vs vLLM runs

- run_bench.sh: skip scenarios whose raw jsonl already exists and has expected completed count
- parse_backend.py: replace scenarios list instead of appending, so resume+reparse stays clean
- README.md: document RUN_ID-based resume
---
 .../dsv4_h200_sglang_vs_vllm/README.md        | 10 ++++++
 .../dsv4_h200_sglang_vs_vllm/parse_backend.py |  7 ++--
 .../dsv4_h200_sglang_vs_vllm/run_bench.sh     | 32 +++++++++++++++++++
 3 files changed, 47 insertions(+), 2 deletions(-)

diff --git a/experiments/dsv4_h200_sglang_vs_vllm/README.md b/experiments/dsv4_h200_sglang_vs_vllm/README.md
index 1b594c6..f4c5faa 100644
--- a/experiments/dsv4_h200_sglang_vs_vllm/README.md
+++ b/experiments/dsv4_h200_sglang_vs_vllm/README.md
@@ -47,6 +47,16 @@
 bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
 ```
 
+### 断点续跑
+
+如果实验中途因为脚本/机器原因中断，可以复用同一个 `RUN_ID` 继续跑，已经完成的 scenario 会自动跳过：
+
+```bash
+RUN_ID=20260708-XXXXXX bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
+```
+
+判断标准是 `raw_outputs/*.jsonl` 已存在且 `completed` 数量达到预期。
+
 结果保存在 `experiments/dsv4_h200_sglang_vs_vllm/results/<RUN_ID>/`：
 
 ```
diff --git a/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py b/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py
index 8ae2322..d0c195f 100755
--- a/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py
+++ b/experiments/dsv4_h200_sglang_vs_vllm/parse_backend.py
@@ -207,8 +207,11 @@ def main() -> None:
         return
 
     if results_json.exists():
-        for s in scenarios:
-            append_scenario(results_json, s)
+        with open(results_json, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        data["scenarios"] = scenarios
+        with open(results_json, "w", encoding="utf-8") as f:
+            json.dump(data, f, indent=2, ensure_ascii=False)
 
     generate_report(result_root, backend, scenarios)
     print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
diff --git a/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
index ab26636..e69764f 100755
--- a/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
+++ b/experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
@@ -111,6 +111,28 @@ run_warmup() {
   log "warmup for ${backend} ${phase} completed"
 }
 
+scenario_already_completed() {
+  local output_file="$1"
+  local expected="$2"
+  [[ -s "$output_file" ]] || return 1
+  local completed
+  completed="$(${VENV_CLIENT}/bin/python -c "
+import json, sys
+path = sys.argv[1]
+try:
+    with open(path, 'r', encoding='utf-8') as f:
+        for line in f:
+            line = line.strip()
+            if line:
+                data = json.loads(line)
+                print(data.get('completed', 0))
+                break
+except Exception:
+    print(0)
+" "$output_file")"
+  [[ "${completed:-0}" -ge "$expected" ]]
+}
+
 run_phase() {
   local backend="$1"
   local phase="$2"
@@ -148,6 +170,11 @@ run_phase() {
     output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
     detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_i${input_len}_o${output_len}.log"
 
+    if scenario_already_completed "$output_file" "$num_prompts"; then
+      log "skipping already-completed ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len}"
+      continue
+    fi
+
     log "running ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
 
     "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
@@ -201,6 +228,11 @@ run_sharegpt() {
     output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${context_len}_${output_len}.jsonl"
     detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_ctx${context_len}_o${output_len}.log"
 
+    if scenario_already_completed "$output_file" "$num_prompts"; then
+      log "skipping already-completed ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len}"
+      continue
+    fi
+
     log "running ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len} n=${num_prompts}"
 
     "${VENV_CLIENT}/bin/python" -m sglang.bench_serving \