From 8519e9ae18d6ce7940a35b9d67dfab1cbae8e688 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 17 Sep 2026 08:38:27 +0000 Subject: [PATCH] Primary metric skips extraction_failure_rate (diagnostics) mrcr's summary row displayed 'extraction_failure_rate 0.0%' as its score because dict insertion order put the diagnostic first. Both primary-metric picks (row build and repeats mean) now skip it. Co-Authored-By: Claude --- evalharness/cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index a5df6ae..5c995d3 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -939,7 +939,8 @@ def _cmd_eval_run(args) -> int: 'max': round(max(_scores), 4), 'std': round(_var ** 0.5, 4), } - _primary = next(iter(report.metrics), '') + _primary = next((k for k in report.metrics + if k != 'extraction_failure_rate'), '') if _primary: report.metrics[f'{_primary}_last_run'] = report.metrics[_primary] report.metrics[_primary] = _mean @@ -980,7 +981,11 @@ def _cmd_eval_run(args) -> int: if console is None: print(render(report, style=args.style)) all_reports.append(report) - primary = next(iter(report.metrics), '') + # primary metric = first REAL metric; extraction_failure_rate + # is diagnostics (mrcr showed '0.0% extraction_failure_rate' as + # its score in the summary because dict order put it first) + primary = next((k for k in report.metrics + if k != 'extraction_failure_rate'), '') groups = {k: v for k, v in report.metric_groups.items() if isinstance(v, dict) and k not in ('run_info',) and not k.startswith('agg_error')}