565ba11790
Official make_test_spec indexes MAP_REPO_VERSION_TO_SPECS[repo][version] -- a None version KeyError'd the official scorer (first agentic run scored 0 with 'KeyError: None' hidden in details). The base converter never copied these fields; both variants now carry them, verified with a live make_test_spec call (spec + 2KB eval script + correct image key). Co-Authored-By: Claude <noreply@anthropic.com>
39 lines
1.4 KiB
Python
39 lines
1.4 KiB
Python
"""swe_bench_verified_agentic: same data as swe_bench_verified, agentic bench.
|
|
|
|
Separate bench name so both variants coexist (single-turn oracle vs
|
|
multi-turn agent); source/split/fields identical — the difference lives
|
|
in the recipe (env loop + official harness scoring) and config."""
|
|
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='swe_bench_verified_agentic',
|
|
source='princeton-nlp/SWE-bench_Verified',
|
|
split='test',
|
|
task_type='agent',
|
|
tags=['code', 'agent', 'swe'],
|
|
requires=['docker'],
|
|
description='SWE-bench Verified (agentic): multi-turn bash agent in '
|
|
'the per-instance /testbed container.',
|
|
)
|
|
)
|
|
def swe_bench_verified_agentic():
|
|
def to_sample(record: dict):
|
|
from .swe_bench_verified import _record_to_sample
|
|
|
|
s = _record_to_sample(record)
|
|
# official make_test_spec requires these raw fields
|
|
md = s.metadata or {}
|
|
md.setdefault('version', record.get('version') or '')
|
|
md.setdefault('patch', record.get('patch') or '')
|
|
md.setdefault('hints_text', record.get('hints_text') or '')
|
|
md.setdefault('created_at', record.get('created_at') or '')
|
|
md['problem_statement'] = record.get('problem_statement') or s.input_text
|
|
s.metadata = md
|
|
return s
|
|
|
|
return to_sample
|