From 014e8bcd9be84db853393e23be57e5bdb1dcb927 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Fri, 18 Sep 2026 15:28:05 +0000 Subject: [PATCH] swe scorer: metadata from env_state (positional pair misalign fix) sample.metadata vs env_state mismatch confirmed: sample[0] said astropy-12907 while the paired prediction held 13453's test_patch -- make_test_spec built the eval_script for the WRONG bug, scoring real patches against wrong tests (0/10 despite manual verification that the patches are correct). The env_state IS the actual instance the agent ran; it takes priority, sample.metadata fills gaps. Co-Authored-By: Claude --- evalharness/eval/recipes/agent.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py index 117cc8f..cc44f54 100644 --- a/evalharness/eval/recipes/agent.py +++ b/evalharness/eval/recipes/agent.py @@ -340,7 +340,14 @@ def _swe_official_reward(pred, target, sample, ctx): patch = es.get('patch') or pred or '' if not patch or '(no patch produced)' in patch: return ({'acc': 0.0}, {'acc': {'error': 'no patch'}}) - md = dict(sample.metadata or {}) + # metadata from ENV_STATE, not sample.metadata: the runner shuffles + # samples and restores predictions by checkpoint key, so positional + # pairing can misalign -- env_state carries the ACTUAL instance the + # agent ran (verified: sample.metadata said 12907 while env_state + # held 13453's test_patch, building an eval_script for the wrong bug) + md = dict(es) + md.update({k: v for k, v in (sample.metadata or {}).items() + if k not in md or not md.get(k)}) # official make_test_spec needs the raw instance fields for k in ('FAIL_TO_PASS', 'PASS_TO_PASS'): v = md.get(k)