feat: support resuming interrupted SGLang vs vLLM runs
- run_bench.sh: skip scenarios whose raw jsonl already exists and has expected completed count - parse_backend.py: replace scenarios list instead of appending, so resume+reparse stays clean - README.md: document RUN_ID-based resume
This commit is contained in:
parent
fd0474099a
commit
5eb605b7a2
@ -47,6 +47,16 @@
|
|||||||
bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
|
bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
|
||||||
```
|
```
|
||||||
|
|
||||||
|
### 断点续跑
|
||||||
|
|
||||||
|
如果实验中途因为脚本/机器原因中断,可以复用同一个 `RUN_ID` 继续跑,已经完成的 scenario 会自动跳过:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
RUN_ID=20260708-XXXXXX bash experiments/dsv4_h200_sglang_vs_vllm/run_bench.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
判断标准是 `raw_outputs/*.jsonl` 已存在且 `completed` 数量达到预期。
|
||||||
|
|
||||||
结果保存在 `experiments/dsv4_h200_sglang_vs_vllm/results/<RUN_ID>/`:
|
结果保存在 `experiments/dsv4_h200_sglang_vs_vllm/results/<RUN_ID>/`:
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|||||||
@ -207,8 +207,11 @@ def main() -> None:
|
|||||||
return
|
return
|
||||||
|
|
||||||
if results_json.exists():
|
if results_json.exists():
|
||||||
for s in scenarios:
|
with open(results_json, "r", encoding="utf-8") as f:
|
||||||
append_scenario(results_json, s)
|
data = json.load(f)
|
||||||
|
data["scenarios"] = scenarios
|
||||||
|
with open(results_json, "w", encoding="utf-8") as f:
|
||||||
|
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||||
|
|
||||||
generate_report(result_root, backend, scenarios)
|
generate_report(result_root, backend, scenarios)
|
||||||
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
print(f"Parsed {len(scenarios)} scenarios into {result_root}/report.md")
|
||||||
|
|||||||
@ -111,6 +111,28 @@ run_warmup() {
|
|||||||
log "warmup for ${backend} ${phase} completed"
|
log "warmup for ${backend} ${phase} completed"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
scenario_already_completed() {
|
||||||
|
local output_file="$1"
|
||||||
|
local expected="$2"
|
||||||
|
[[ -s "$output_file" ]] || return 1
|
||||||
|
local completed
|
||||||
|
completed="$(${VENV_CLIENT}/bin/python -c "
|
||||||
|
import json, sys
|
||||||
|
path = sys.argv[1]
|
||||||
|
try:
|
||||||
|
with open(path, 'r', encoding='utf-8') as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if line:
|
||||||
|
data = json.loads(line)
|
||||||
|
print(data.get('completed', 0))
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
print(0)
|
||||||
|
" "$output_file")"
|
||||||
|
[[ "${completed:-0}" -ge "$expected" ]]
|
||||||
|
}
|
||||||
|
|
||||||
run_phase() {
|
run_phase() {
|
||||||
local backend="$1"
|
local backend="$1"
|
||||||
local phase="$2"
|
local phase="$2"
|
||||||
@ -148,6 +170,11 @@ run_phase() {
|
|||||||
output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${input_len}_${output_len}.jsonl"
|
||||||
detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_i${input_len}_o${output_len}.log"
|
detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_i${input_len}_o${output_len}.log"
|
||||||
|
|
||||||
|
if scenario_already_completed "$output_file" "$num_prompts"; then
|
||||||
|
log "skipping already-completed ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
log "running ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
|
log "running ${backend} ${phase} scenario: c=${concurrency} i=${input_len} o=${output_len} n=${num_prompts}"
|
||||||
|
|
||||||
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
|
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
|
||||||
@ -201,6 +228,11 @@ run_sharegpt() {
|
|||||||
output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${context_len}_${output_len}.jsonl"
|
output_file="${raw_dir}/${backend}_${phase}_$(date '+%m%d')_${concurrency}_${context_len}_${output_len}.jsonl"
|
||||||
detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_ctx${context_len}_o${output_len}.log"
|
detail_log="${phase_log_dir}/${backend}_${phase}_c${concurrency}_ctx${context_len}_o${output_len}.log"
|
||||||
|
|
||||||
|
if scenario_already_completed "$output_file" "$num_prompts"; then
|
||||||
|
log "skipping already-completed ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
log "running ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len} n=${num_prompts}"
|
log "running ${backend} ${phase} scenario: c=${concurrency} ctx=${context_len} o=${output_len} n=${num_prompts}"
|
||||||
|
|
||||||
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
|
"${VENV_CLIENT}/bin/python" -m sglang.bench_serving \
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user