[Fix] Validate per-request benchmark errors correctly
This commit is contained in:
parent
a90c898683
commit
63f2327a90
@ -307,13 +307,17 @@ run_bench() {
|
||||
--warmup-requests "${WARMUP_REQUESTS}" --output-file "$output" \
|
||||
--output-details --disable-tqdm \
|
||||
>"${RESULT_ROOT}/bench/${stem}.log" 2>&1
|
||||
python3 - "$output" "${NUM_PROMPTS}" <<'PY'
|
||||
python3 - "$output" "${NUM_PROMPTS}" "${OUTPUT_LEN}" <<'PY'
|
||||
import json, sys
|
||||
path, expected = sys.argv[1], int(sys.argv[2])
|
||||
path, expected, expected_output_len = sys.argv[1], int(sys.argv[2]), int(sys.argv[3])
|
||||
rows = [json.loads(x) for x in open(path, encoding="utf-8") if x.strip()]
|
||||
assert len(rows) == 1, (path, len(rows))
|
||||
assert rows[0].get("completed") == expected, rows[0].get("completed")
|
||||
assert not rows[0].get("errors"), rows[0].get("errors")
|
||||
errors = rows[0].get("errors") or []
|
||||
assert all(not error for error in errors), errors
|
||||
output_lens = rows[0].get("output_lens") or []
|
||||
assert len(output_lens) == expected, (len(output_lens), expected)
|
||||
assert all(length == expected_output_len for length in output_lens), output_lens
|
||||
PY
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user