From 7eb7b267869cb2864ee69006265adfc3eec52fd6 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Mon, 14 Sep 2026 06:15:51 +0000 Subject: [PATCH] Summary time = this run's wall clock; n counts repeat generations time_s/time_h summed per-prediction latency_s, which includes RESTORED predictions' original generation time -- days old and from a slower setup, it once reported 15.9h for a one-hour aime25 run. All rows now report the bench's actual wall clock; token totals stay as the true cost of the predictions used. n for repeats>1 is num_samples x repeats (12 runs over 30 problems is 360 generations, not 30). Co-Authored-By: Claude --- evalharness/cli.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index 72222d9..7d0ffa9 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -926,15 +926,13 @@ def _cmd_eval_run(args) -> int: print(render(report, style=args.style)) all_reports.append(report) primary = next(iter(report.metrics), '') - secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0) - for s in report.samples) groups = {k: v for k, v in report.metric_groups.items() if isinstance(v, dict) and k not in ('run_info',) and not k.startswith('agg_error')} info = report.metric_groups.get('run_info', {}) or {} - if _repeats > 1 and _rep_secs: - # repeats: report the SUM over all runs, not the last one - secs_total = _rep_secs + if _repeats > 1: + # repeats: tokens are the SUM over all runs (real cost of the + # predictions used) info = {**info, 'gen_input_tokens': _rep_tin, 'gen_output_tokens': _rep_tout, 'gen_total_tokens': _rep_tin + _rep_tout} @@ -945,12 +943,19 @@ def _cmd_eval_run(args) -> int: return lats[min(int(len(lats) * q), len(lats) - 1)] if lats else 0.0 fins = [(s.usage or {}).get('finish_reason', '') for s in report.samples] + # time = THIS run's wall clock everywhere: summing per-prediction + # latency_s counts RESTORED predictions' original generation time + # (days old, slower setup) -- that once reported 15.9h for a + # one-hour run + _wall = round(_time.time() - t0, 1) rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary), - 'n': report.num_samples, + # repeats evaluate the SAME N problems k times: the + # count people expect is the generations, not N + 'n': report.num_samples * _repeats, 'extract_fail': report.num_failed_extractions, - 'secs': round(secs_total, 1), - 'wall': round(_time.time() - t0, 1), - 'hours': round(secs_total / 3600, 2), + 'secs': _wall, + 'wall': _wall, + 'hours': round(_wall / 3600, 2), 'tok_in': info.get('gen_input_tokens', 0) or 0, 'tok_out': info.get('gen_output_tokens', 0) or 0, 'tokens': (info.get('gen_input_tokens', 0) or 0)