Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Primary metric skips extraction_failure_rate (diagnostics)

Browse Source 
mrcr's summary row displayed 'extraction_failure_rate 0.0%' as its
score because dict insertion order put the diagnostic first. Both
primary-metric picks (row build and repeats mean) now skip it.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-17 08:38:27 +00:00
parent 081249d47c
commit 8519e9ae18
1 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
evalharness/cli.py
View File

@ -939,7 +939,8 @@ def _cmd_eval_run(args) -> int:
'max': round(max(_scores), 4), 'max': round(max(_scores), 4),
'std': round(_var ** 0.5, 4), 'std': round(_var ** 0.5, 4),
} }
_primary = next(iter(report.metrics), '') _primary = next((k for k in report.metrics
if k != 'extraction_failure_rate'), '')
if _primary: if _primary:
report.metrics[f'{_primary}_last_run'] = report.metrics[_primary] report.metrics[f'{_primary}_last_run'] = report.metrics[_primary]
report.metrics[_primary] = _mean report.metrics[_primary] = _mean
@ -980,7 +981,11 @@ def _cmd_eval_run(args) -> int:
if console is None: if console is None:
print(render(report, style=args.style)) print(render(report, style=args.style))
all_reports.append(report) all_reports.append(report)
primary = next(iter(report.metrics), '') # primary metric = first REAL metric; extraction_failure_rate
# is diagnostics (mrcr showed '0.0% extraction_failure_rate' as
# its score in the summary because dict order put it first)
primary = next((k for k in report.metrics
if k != 'extraction_failure_rate'), '')
groups = {k: v for k, v in report.metric_groups.items() groups = {k: v for k, v in report.metric_groups.items()
if isinstance(v, dict) and k not in ('run_info',) if isinstance(v, dict) and k not in ('run_info',)
and not k.startswith('agg_error')} and not k.startswith('agg_error')}