[Fix] Validate per-request benchmark errors correctly

This commit is contained in:
Zhiyi Hong 2026-08-19 11:54:34 +08:00
parent a90c898683
commit 63f2327a90

View File

@ -307,13 +307,17 @@ run_bench() {
--warmup-requests "${WARMUP_REQUESTS}" --output-file "$output" \
--output-details --disable-tqdm \
>"${RESULT_ROOT}/bench/${stem}.log" 2>&1
python3 - "$output" "${NUM_PROMPTS}" <<'PY'
python3 - "$output" "${NUM_PROMPTS}" "${OUTPUT_LEN}" <<'PY'
import json, sys
path, expected = sys.argv[1], int(sys.argv[2])
path, expected, expected_output_len = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
rows = [json.loads(x) for x in open(path, encoding="utf-8") if x.strip()]
assert len(rows) == 1, (path, len(rows))
assert rows[0].get("completed") == expected, rows[0].get("completed")
assert not rows[0].get("errors"), rows[0].get("errors")
errors = rows[0].get("errors") or []
assert all(not error for error in errors), errors
output_lens = rows[0].get("output_lens") or []
assert len(output_lens) == expected, (len(output_lens), expected)
assert all(length == expected_output_len for length in output_lens), output_lens
PY
}