370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
33 lines
1.2 KiB
Python
33 lines
1.2 KiB
Python
"""HumanEval (official source: openai/openai_humaneval)."""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='humaneval',
|
|
source='openai/openai_humaneval', # official: https://huggingface.co/datasets/openai/openai_humaneval
|
|
split='test',
|
|
gen_config={'temperature': 1.0, 'max_tokens': 32768},
|
|
task_type='coding',
|
|
tags=['code'],
|
|
description='OpenAI HumanEval function synthesis (official). Tests in metadata for sandbox.',
|
|
)
|
|
)
|
|
def humaneval():
|
|
def to_sample(record: dict) -> Sample:
|
|
return Sample(
|
|
# es adapter: instruction header + prompt
|
|
input='Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n' + record['prompt'],
|
|
target=record['canonical_solution'],
|
|
metadata={'prompt': record['prompt'], # original bare prompt (harness assembles from this)
|
|
'task_id': record['task_id'],
|
|
'test': record['test'],
|
|
'entry_point': record['entry_point'],
|
|
},
|
|
)
|
|
|
|
return to_sample
|