Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Summary time = this run's wall clock; n counts repeat generations

Browse Source 
time_s/time_h summed per-prediction latency_s, which includes RESTORED
predictions' original generation time -- days old and from a slower
setup, it once reported 15.9h for a one-hour aime25 run. All rows now
report the bench's actual wall clock; token totals stay as the true
cost of the predictions used.

n for repeats>1 is num_samples x repeats (12 runs over 30 problems is
360 generations, not 30).

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-14 06:15:51 +00:00
parent f272146b54
commit 7eb7b26786
1 changed files with 14 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
evalharness/cli.py
View File

@ -926,15 +926,13 @@ def _cmd_eval_run(args) -> int:
print(render(report, style=args.style)) print(render(report, style=args.style))
all_reports.append(report) all_reports.append(report)
primary = next(iter(report.metrics), '') primary = next(iter(report.metrics), '')
secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0)
for s in report.samples)
groups = {k: v for k, v in report.metric_groups.items() groups = {k: v for k, v in report.metric_groups.items()
if isinstance(v, dict) and k not in ('run_info',) if isinstance(v, dict) and k not in ('run_info',)
and not k.startswith('agg_error')} and not k.startswith('agg_error')}
info = report.metric_groups.get('run_info', {}) or {} info = report.metric_groups.get('run_info', {}) or {}
if _repeats > 1 and _rep_secs: if _repeats > 1:
# repeats: report the SUM over all runs, not the last one # repeats: tokens are the SUM over all runs (real cost of the
secs_total = _rep_secs # predictions used)
info = {**info, 'gen_input_tokens': _rep_tin, info = {**info, 'gen_input_tokens': _rep_tin,
'gen_output_tokens': _rep_tout, 'gen_output_tokens': _rep_tout,
'gen_total_tokens': _rep_tin + _rep_tout} 'gen_total_tokens': _rep_tin + _rep_tout}
@ -945,12 +943,19 @@ def _cmd_eval_run(args) -> int:
return lats[min(int(len(lats) * q), len(lats) - 1)] if lats else 0.0 return lats[min(int(len(lats) * q), len(lats) - 1)] if lats else 0.0
fins = [(s.usage or {}).get('finish_reason', '') fins = [(s.usage or {}).get('finish_reason', '')
for s in report.samples] for s in report.samples]
# time = THIS run's wall clock everywhere: summing per-prediction
# latency_s counts RESTORED predictions' original generation time
# (days old, slower setup) -- that once reported 15.9h for a
# one-hour run
_wall = round(_time.time() - t0, 1)
rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary), rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary),
'n': report.num_samples, # repeats evaluate the SAME N problems k times: the
# count people expect is the generations, not N
'n': report.num_samples * _repeats,
'extract_fail': report.num_failed_extractions, 'extract_fail': report.num_failed_extractions,
'secs': round(secs_total, 1), 'secs': _wall,
'wall': round(_time.time() - t0, 1), 'wall': _wall,
'hours': round(secs_total / 3600, 2), 'hours': round(_wall / 3600, 2),
'tok_in': info.get('gen_input_tokens', 0) or 0, 'tok_in': info.get('gen_input_tokens', 0) or 0,
'tok_out': info.get('gen_output_tokens', 0) or 0, 'tok_out': info.get('gen_output_tokens', 0) or 0,
'tokens': (info.get('gen_input_tokens', 0) or 0) 'tokens': (info.get('gen_input_tokens', 0) or 0)