diff --git a/evalharness/cli.py b/evalharness/cli.py index 72222d9..7d0ffa9 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -926,15 +926,13 @@ def _cmd_eval_run(args) -> int: print(render(report, style=args.style)) all_reports.append(report) primary = next(iter(report.metrics), '') - secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0) - for s in report.samples) groups = {k: v for k, v in report.metric_groups.items() if isinstance(v, dict) and k not in ('run_info',) and not k.startswith('agg_error')} info = report.metric_groups.get('run_info', {}) or {} - if _repeats > 1 and _rep_secs: - # repeats: report the SUM over all runs, not the last one - secs_total = _rep_secs + if _repeats > 1: + # repeats: tokens are the SUM over all runs (real cost of the + # predictions used) info = {**info, 'gen_input_tokens': _rep_tin, 'gen_output_tokens': _rep_tout, 'gen_total_tokens': _rep_tin + _rep_tout} @@ -945,12 +943,19 @@ def _cmd_eval_run(args) -> int: return lats[min(int(len(lats) * q), len(lats) - 1)] if lats else 0.0 fins = [(s.usage or {}).get('finish_reason', '') for s in report.samples] + # time = THIS run's wall clock everywhere: summing per-prediction + # latency_s counts RESTORED predictions' original generation time + # (days old, slower setup) -- that once reported 15.9h for a + # one-hour run + _wall = round(_time.time() - t0, 1) rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary), - 'n': report.num_samples, + # repeats evaluate the SAME N problems k times: the + # count people expect is the generations, not N + 'n': report.num_samples * _repeats, 'extract_fail': report.num_failed_extractions, - 'secs': round(secs_total, 1), - 'wall': round(_time.time() - t0, 1), - 'hours': round(secs_total / 3600, 2), + 'secs': _wall, + 'wall': _wall, + 'hours': round(_wall / 3600, 2), 'tok_in': info.get('gen_input_tokens', 0) or 0, 'tok_out': info.get('gen_output_tokens', 0) or 0, 'tokens': (info.get('gen_input_tokens', 0) or 0)