"""BIG-Bench Hard (standard mirror: lukaemon/bbh; original: github.com/suzgunmirac/BIG-Bench-Hard). Paper-faithful 3-shot CoT: the official hand-written exemplars (vendored in _bbh_cot_prompts.py, MIT) are injected per-subtask via few_shot hook. """ from ..sample import Sample from ..registry import register_dataset from ..spec import DatasetSpec def bbh_few_shot(split: str, subset: str, n: int): """Return the official 3-shot CoT prompt text for this subtask.""" if n <= 0: return None from ._bbh_cot_prompts import COT_PROMPTS text = COT_PROMPTS.get(subset) return text.strip() + '\n\n' if text else None @register_dataset( DatasetSpec( name='bbh', source='lukaemon/bbh', # https://huggingface.co/datasets/lukaemon/bbh subset='boolean_expressions', # 27 subtasks; override with --subset split='test', task_type='qa', tags=['reasoning'], description='BIG-Bench Hard, 27 subtasks (each subset caches under bbh/).', few_shot_split='official_cot', # -> bbh_few_shot hook (3-shot official CoT) ) ) def bbh(): def to_sample(record: dict) -> Sample: return Sample(input=record['input'], target=str(record['target']).strip()) return to_sample