From 8c3d32bf2eda024a2dc87bc1f742f09f86746e41 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Fri, 18 Sep 2026 17:15:37 +0000 Subject: [PATCH] swe scorer: return 'resolved' key (recipe metric name, not 'acc') The scorer returned {'acc': ...} while the recipe registers the metric as 'resolved' -- the aggregator looked for 'resolved' in scores, found nothing, and reported 0.0 despite per-sample resolved=1.0 in the checkpoint. Key names now match the recipe registration. Co-Authored-By: Claude --- evalharness/eval/recipes/agent.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py index 0fc2b77..8c657a4 100644 --- a/evalharness/eval/recipes/agent.py +++ b/evalharness/eval/recipes/agent.py @@ -375,8 +375,8 @@ def _swe_official_reward(pred, target, sample, ctx): if have: try: res = _official_swebench_eval(md, patch) - return ({'acc': res.get('resolved', 0.0)}, - {'acc': {'mode': 'official_swebench', + return ({'resolved': res.get('resolved', 0.0)}, + {'resolved': {'mode': 'official_swebench', 'resolved': res.get('resolved'), 'error': res.get('error', ''), 'turns_used': es.get('turns_used')}})