From 9889fb6c0e50de8af95d13438bd62185a70f893a Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Fri, 18 Sep 2026 16:14:00 +0000 Subject: [PATCH] swe scorer: unwrap grader's per-instance return shape get_eval_report returns {instance_id: {resolved: bool, ...}}, not a flat dict. report.get('resolved') was always None -> every sample scored 0 despite the grader judging resolved=true (confirmed by direct grader call on the real test output: astropy-13453 resolved=true). Co-Authored-By: Claude --- evalharness/eval/recipes/agent.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py index cc44f54..0fc2b77 100644 --- a/evalharness/eval/recipes/agent.py +++ b/evalharness/eval/recipes/agent.py @@ -323,8 +323,16 @@ def _official_swebench_eval(instance_meta: dict, patch: str, timeout: int = 1800 test_log_path=log_dir / 'test_output.txt', include_tests_status=True, ) - return {'resolved': float(report.get('resolved', 0.0)), - 'report': {k: v for k, v in report.items() + # get_eval_report returns {instance_id: {resolved: bool, ...}}, + # NOT a flat dict -- report.get('resolved') was always None, so + # every instance scored 0 even when the grader said resolved=true + inner = report.get(test_spec.instance_id) or report + if isinstance(inner, dict): + resolved = float(inner.get('resolved', 0) or 0) + else: + resolved = 0.0 + return {'resolved': resolved, + 'report': {k: v for k, v in (inner if isinstance(inner, dict) else {}).items() if isinstance(v, (int, float, str, bool))}} finally: try: