From 565ba117906d0cfc785549d86a3befa322a03a25 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Fri, 18 Sep 2026 08:11:18 +0000 Subject: [PATCH] swe datasets: carry version/patch/hints/created_at in metadata Official make_test_spec indexes MAP_REPO_VERSION_TO_SPECS[repo][version] -- a None version KeyError'd the official scorer (first agentic run scored 0 with 'KeyError: None' hidden in details). The base converter never copied these fields; both variants now carry them, verified with a live make_test_spec call (spec + 2KB eval script + correct image key). Co-Authored-By: Claude --- evalharness/data/datasets/swe_bench_verified.py | 4 ++++ .../data/datasets/swe_bench_verified_agentic.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/evalharness/data/datasets/swe_bench_verified.py b/evalharness/data/datasets/swe_bench_verified.py index 0a074ef..610b67e 100644 --- a/evalharness/data/datasets/swe_bench_verified.py +++ b/evalharness/data/datasets/swe_bench_verified.py @@ -37,6 +37,10 @@ def _record_to_sample(record: dict) -> Sample: 'PASS_TO_PASS': record['PASS_TO_PASS'], 'environment_setup_commit': record.get('environment_setup_commit'), 'difficulty': record.get('difficulty'), + 'version': record.get('version'), # official MAP_REPO_VERSION_TO_SPECS key + 'patch': record.get('patch'), + 'hints_text': record.get('hints_text'), + 'created_at': record.get('created_at'), }, ) diff --git a/evalharness/data/datasets/swe_bench_verified_agentic.py b/evalharness/data/datasets/swe_bench_verified_agentic.py index ef09260..945e428 100644 --- a/evalharness/data/datasets/swe_bench_verified_agentic.py +++ b/evalharness/data/datasets/swe_bench_verified_agentic.py @@ -21,6 +21,18 @@ from ..spec import DatasetSpec ) ) def swe_bench_verified_agentic(): - from .swe_bench_verified import _record_to_sample + def to_sample(record: dict): + from .swe_bench_verified import _record_to_sample - return _record_to_sample + s = _record_to_sample(record) + # official make_test_spec requires these raw fields + md = s.metadata or {} + md.setdefault('version', record.get('version') or '') + md.setdefault('patch', record.get('patch') or '') + md.setdefault('hints_text', record.get('hints_text') or '') + md.setdefault('created_at', record.get('created_at') or '') + md['problem_statement'] = record.get('problem_statement') or s.input_text + s.metadata = md + return s + + return to_sample