feat: support resuming interrupted SGLang vs vLLM runs
- run_bench.sh: skip scenarios whose raw jsonl already exists and has expected completed count - parse_backend.py: replace scenarios list instead of appending, so resume+reparse stays clean - README.md: document RUN_ID-based resume
This commit is contained in:
parent
fd0474099a
commit
5eb605b7a2
@ -47,6 +47,16 @@
|
||||
bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
|
||||
```
|
||||
|
||||
### 断点续跑
|
||||
|
||||
如果实验中途因为脚本/机器原因中断,可以复用同一个 `RUN_ID` 继续跑,已经完成的 scenario 会自动跳过:
|
||||
|
||||
```bash
|
||||
RUN_ID=20260708-XXXXXX bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
|
||||
```
|
||||
|
||||
判断标准是 `raw_outputs/*.jsonl` 已存在且 `completed` 数量达到预期。
|
||||
|
||||
结果保存在 `experiments/dsv4_h200_sglang_vs_vllm/results/<RUN_ID>/`:
|
||||
|
||||
```
|
||||
|
||||
@ -207,8 +207,11 @@ def main() -> None:
|
||||
return
|
||||
|
||||
if results_json.exists():
|
||||
for s in scenarios:
|
||||
append_scenario(results_json, s)
|
||||
with open(results_json, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
data["scenarios"] = scenarios
|
||||
with open(results_json, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
generate_report(result_root, backend, scenarios)
|
||||
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
||||
|
||||
@ -111,6 +111,28 @@ run_warmup() {
|
||||
log "warmup for ${backend} ${phase} completed"
|
||||
}
|
||||
|
||||
scenario_already_completed() {
|
||||
local output_file="$1"
|
||||
local expected="$2"
|
||||
[[ -s "$output_file" ]] || return 1
|
||||
local completed
|
||||
completed="$(${VENV_CLIENT}/bin/python -c "
|
||||
import json, sys
|
||||
path = sys.argv[1]
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if line:
|
||||
data = json.loads(line)
|
||||
print(data.get('completed', 0))
|
||||
break
|
||||
except Exception:
|
||||
print(0)
|
||||
" "$output_file")"
|
||||
[[ "${completed:-0}" -ge "$expected" ]]
|
||||
}
|
||||
|
||||
run_phase() {
|
||||
local backend="$1"
|
||||
local phase="$2"
|
||||
@ -148,6 +170,11 @@ run_phase() {
|
||||
output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
||||
detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_i${input_len}_o${output_len}.log"
|
||||
|
||||
if scenario_already_completed "$output_file" "$num_prompts"; then
|
||||
log "skipping already-completed ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len}"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "running ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
|
||||
|
||||
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
|
||||
@ -201,6 +228,11 @@ run_sharegpt() {
|
||||
output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${context_len}_${output_len}.jsonl"
|
||||
detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_ctx${context_len}_o${output_len}.log"
|
||||
|
||||
if scenario_already_completed "$output_file" "$num_prompts"; then
|
||||
log "skipping already-completed ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len}"
|
||||
continue
|
||||
fi
|
||||
|
||||
log "running ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len} n=${num_prompts}"
|
||||
|
||||
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user