diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py index 117cc8f..cc44f54 100644 --- a/evalharness/eval/recipes/agent.py +++ b/evalharness/eval/recipes/agent.py @@ -340,7 +340,14 @@ def _swe_official_reward(pred, target, sample, ctx): patch = es.get('patch') or pred or '' if not patch or '(no patch produced)' in patch: return ({'acc': 0.0}, {'acc': {'error': 'no patch'}}) - md = dict(sample.metadata or {}) + # metadata from ENV_STATE, not sample.metadata: the runner shuffles + # samples and restores predictions by checkpoint key, so positional + # pairing can misalign -- env_state carries the ACTUAL instance the + # agent ran (verified: sample.metadata said 12907 while env_state + # held 13453's test_patch, building an eval_script for the wrong bug) + md = dict(es) + md.update({k: v for k, v in (sample.metadata or {}).items() + if k not in md or not md.get(k)}) # official make_test_spec needs the raw instance fields for k in ('FAIL_TO_PASS', 'PASS_TO_PASS'): v = md.get(k)