Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Summary table: 'cached' instead of a misleading ~0s time

Browse Source 
Tokens are cumulative (they include restored predictions' usage) while
time was this-run wall -- fully replayed benches showed 0s in the same
column, reading as broken. run_info now carries gen_fresh; a bench with
zero fresh generations renders its time cell as 'cached' (tokens keep
showing the true cumulative investment).

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-15 10:01:09 +00:00
parent f4ab4b4416
commit 1fd0dcbdbb
2 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
evalharness/cli.py
View File

@ -943,7 +943,10 @@ def _cmd_eval_run(args) -> int:
# (days old, slower setup) -- that once reported 15.9h for a # (days old, slower setup) -- that once reported 15.9h for a
# one-hour run # one-hour run
_wall = round(_time.time() - t0, 1) _wall = round(_time.time() - t0, 1)
rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary), rows.append({'name': name, 'metric': primary,
'cached': (report.metric_groups.get('run_info', {})
.get('gen_fresh') == 0) if _repeats <= 1 else False,
'value': report.metrics.get(primary),
# repeats evaluate the SAME N problems k times: the # repeats evaluate the SAME N problems k times: the
# count people expect is the generations, not N # count people expect is the generations, not N
'n': report.num_samples * _repeats, 'n': report.num_samples * _repeats,
@ -1045,7 +1048,8 @@ def _print_summary_table(console, rows):
for r in rows: for r in rows:
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR' v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
wall = r.get('wall') or r.get('secs') or 0 wall = r.get('wall') or r.get('secs') or 0
tm = f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s' tm = 'cached' if r.get('cached') else (
f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s')
ti, to = r.get('tok_in', 0), r.get('tok_out', 0) ti, to = r.get('tok_in', 0), r.get('tok_out', 0)
tin = f'{ti:,}' if ti else '—' tin = f'{ti:,}' if ti else '—'
tout = f'{to:,}' if to else '—' tout = f'{to:,}' if to else '—'

9
evalharness/model/runner.py
View File

@ -503,7 +503,7 @@ async def generate_predictions(
# ckpt info (store + per-position keys) so run_eval can read/write # ckpt info (store + per-position keys) so run_eval can read/write
# SCORES bound to these predictions; None when checkpointing is off # SCORES bound to these predictions; None when checkpointing is off
ckpt_info = (ckpt_store, keys) if ckpt_store is not None else None ckpt_info = (ckpt_store, keys) if ckpt_store is not None else None
return preds, usages, total_usage, ckpt_info return preds, usages, total_usage, ckpt_info, len(fresh) - len(failed_samples)
finally: finally:
# reporter lifecycle belongs to the CALLER (CLI reuses one reporter # reporter lifecycle belongs to the CALLER (CLI reuses one reporter
# across benchmarks and closes it after the whole run); only close # across benchmarks and closes it after the whole run); only close
@ -781,7 +781,7 @@ async def run_eval(
try: try:
from .gen_profiles import merge_gen_kwargs from .gen_profiles import merge_gen_kwargs
preds, _usages, usage, ckpt_info = await generate_predictions( preds, _usages, usage, ckpt_info, n_fresh = await generate_predictions(
adapter, list(raw_samples), concurrency, progress=progress, adapter, list(raw_samples), concurrency, progress=progress,
progress_reporter=progress_reporter, progress_reporter=progress_reporter,
status_callback=status_callback, status_callback=status_callback,
@ -831,7 +831,10 @@ async def run_eval(
status_callback('Scoring predictions against the benchmark recipe') status_callback('Scoring predictions against the benchmark recipe')
_meta = {'gen_input_tokens': usage.input_tokens, _meta = {'gen_input_tokens': usage.input_tokens,
'gen_output_tokens': usage.output_tokens, 'gen_output_tokens': usage.output_tokens,
'gen_total_tokens': usage.total_tokens} 'gen_total_tokens': usage.total_tokens,
# fresh=0 means the whole bench replayed from checkpoint: the
# summary table then shows 'cached' instead of a ~0s time
'gen_fresh': n_fresh}
if _records is not None: if _records is not None:
from ..eval.runner import evaluate_cached from ..eval.runner import evaluate_cached