"""HumanEval (official source: openai/openai_humaneval).""" from ..sample import Sample from ..registry import register_dataset from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='humaneval', source='openai/openai_humaneval', # official: https://huggingface.co/datasets/openai/openai_humaneval split='test', gen_config={'temperature': 1.0, 'max_tokens': 32768}, task_type='coding', tags=['code'], description='OpenAI HumanEval function synthesis (official). Tests in metadata for sandbox.', ) ) def humaneval(): def to_sample(record: dict) -> Sample: return Sample( # es adapter: instruction header + prompt input='Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n' + record['prompt'], target=record['canonical_solution'], metadata={'prompt': record['prompt'], # original bare prompt (harness assembles from this) 'task_id': record['task_id'], 'test': record['test'], 'entry_point': record['entry_point'], }, ) return to_sample