24 lines
758 B
Python
24 lines
758 B
Python
"""BIG-Bench Hard (standard mirror: lukaemon/bbh; original: github.com/suzgunmirac/BIG-Bench-Hard)."""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='bbh',
|
|
source='lukaemon/bbh', # https://huggingface.co/datasets/lukaemon/bbh
|
|
subset='boolean_expressions', # 27 subtasks; override with --subset <subtask>
|
|
split='test',
|
|
task_type='qa',
|
|
tags=['reasoning'],
|
|
description='BIG-Bench Hard, 27 subtasks (each subset caches under bbh/<hash>).',
|
|
)
|
|
)
|
|
def bbh():
|
|
def to_sample(record: dict) -> Sample:
|
|
return Sample(input=record['input'], target=str(record['target']).strip())
|
|
|
|
return to_sample
|