370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
41 lines
1.4 KiB
Python
41 lines
1.4 KiB
Python
"""BIG-Bench Hard (standard mirror: lukaemon/bbh; original: github.com/suzgunmirac/BIG-Bench-Hard).
|
|
|
|
Paper-faithful 3-shot CoT: the official hand-written exemplars (vendored in
|
|
_bbh_cot_prompts.py, MIT) are injected per-subtask via few_shot hook.
|
|
"""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
def bbh_few_shot(split: str, subset: str, n: int):
|
|
"""Return the official 3-shot CoT prompt text for this subtask."""
|
|
if n <= 0:
|
|
return None
|
|
from ._bbh_cot_prompts import COT_PROMPTS
|
|
|
|
text = COT_PROMPTS.get(subset)
|
|
return text.strip() + '\n\n' if text else None
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='bbh',
|
|
source='lukaemon/bbh', # https://huggingface.co/datasets/lukaemon/bbh
|
|
subset='boolean_expressions', # 27 subtasks; override with --subset <subtask>
|
|
split='test',
|
|
task_type='qa',
|
|
tags=['reasoning'],
|
|
description='BIG-Bench Hard, 27 subtasks (each subset caches under bbh/<hash>).',
|
|
prompt_style='bbh_es', # es test-question wrapper: Q:/A: think-step-by-step
|
|
few_shot_split='official_cot', # -> bbh_few_shot hook (official CoT)
|
|
few_shot_num=3, # paper/es default: 3-shot
|
|
)
|
|
)
|
|
def bbh():
|
|
def to_sample(record: dict) -> Sample:
|
|
return Sample(input=record['input'], target=str(record['target']).strip())
|
|
|
|
return to_sample
|