diff --git a/evalharness/cli.py b/evalharness/cli.py index ffe782f..ed8fb81 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -943,7 +943,10 @@ def _cmd_eval_run(args) -> int: # (days old, slower setup) -- that once reported 15.9h for a # one-hour run _wall = round(_time.time() - t0, 1) - rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary), + rows.append({'name': name, 'metric': primary, + 'cached': (report.metric_groups.get('run_info', {}) + .get('gen_fresh') == 0) if _repeats <= 1 else False, + 'value': report.metrics.get(primary), # repeats evaluate the SAME N problems k times: the # count people expect is the generations, not N 'n': report.num_samples * _repeats, @@ -1045,7 +1048,8 @@ def _print_summary_table(console, rows): for r in rows: v = _fmt_score(r.get('value')) if r['ok'] else 'ERR' wall = r.get('wall') or r.get('secs') or 0 - tm = f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s' + tm = 'cached' if r.get('cached') else ( + f'{wall / 3600:.2f}h' if wall >= 3600 else f'{wall:.0f}s') ti, to = r.get('tok_in', 0), r.get('tok_out', 0) tin = f'{ti:,}' if ti else '—' tout = f'{to:,}' if to else '—' diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index b181cff..450a49b 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -503,7 +503,7 @@ async def generate_predictions( # ckpt info (store + per-position keys) so run_eval can read/write # SCORES bound to these predictions; None when checkpointing is off ckpt_info = (ckpt_store, keys) if ckpt_store is not None else None - return preds, usages, total_usage, ckpt_info + return preds, usages, total_usage, ckpt_info, len(fresh) - len(failed_samples) finally: # reporter lifecycle belongs to the CALLER (CLI reuses one reporter # across benchmarks and closes it after the whole run); only close @@ -781,7 +781,7 @@ async def run_eval( try: from .gen_profiles import merge_gen_kwargs - preds, _usages, usage, ckpt_info = await generate_predictions( + preds, _usages, usage, ckpt_info, n_fresh = await generate_predictions( adapter, list(raw_samples), concurrency, progress=progress, progress_reporter=progress_reporter, status_callback=status_callback, @@ -831,7 +831,10 @@ async def run_eval( status_callback('Scoring predictions against the benchmark recipe') _meta = {'gen_input_tokens': usage.input_tokens, 'gen_output_tokens': usage.output_tokens, - 'gen_total_tokens': usage.total_tokens} + 'gen_total_tokens': usage.total_tokens, + # fresh=0 means the whole bench replayed from checkpoint: the + # summary table then shows 'cached' instead of a ~0s time + 'gen_fresh': n_fresh} if _records is not None: from ..eval.runner import evaluate_cached