"""SWE-bench Verified (official source: princeton-nlp/SWE-bench_Verified).""" from ..sample import Sample, SandboxSpec from ..registry import register_dataset from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='swe_bench_verified', source='princeton-nlp/SWE-bench_Verified', # official: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified split='test', gen_config={'temperature': 0.0, 'max_tokens': 32768}, task_type='agent', tags=['code', 'agent', 'swe'], requires=['docker'], description='SWE-bench Verified; per-instance docker image carried in Sample.sandbox.', ) ) def swe_bench_verified(): def to_sample(record: dict) -> Sample: instance_id = record['instance_id'] # image naming: local swebench/ set uses {repo}_1776_{repo}-{num}; the # newer official layout is sweb.eval.x86_64.{repo}__{repo}-{num} # OFFICIAL docker layout: repo-level BASE images # (swebench/sweb.eval.x86_64.{repo}) -- per-instance images are # NEVER published; the harness builds them on top of the base _repo = instance_id.rsplit('-', 1)[0] img = f'swebench/sweb.eval.x86_64.{_repo}:latest' if '__' in instance_id: repo, num = instance_id.rsplit('-', 1) r = repo.split('__')[0] img = f'swebench/sweb.eval.x86_64.{r}_1776_{r.split("__")[-1]}-{num}' return Sample( input=record['problem_statement'], target=record['patch'], # gold patch (for oracle/oracle-check only) sandbox=SandboxSpec(image=img), metadata={ 'instance_id': instance_id, 'repo': record['repo'], 'base_commit': record['base_commit'], 'test_patch': record['test_patch'], 'FAIL_TO_PASS': record['FAIL_TO_PASS'], 'PASS_TO_PASS': record['PASS_TO_PASS'], 'environment_setup_commit': record.get('environment_setup_commit'), 'difficulty': record.get('difficulty'), }, ) return to_sample