diff --git a/evalharness/data/datasets/bbh.py b/evalharness/data/datasets/bbh.py index 5d3e09f..b86dfcd 100644 --- a/evalharness/data/datasets/bbh.py +++ b/evalharness/data/datasets/bbh.py @@ -28,7 +28,8 @@ def bbh_few_shot(split: str, subset: str, n: int): task_type='qa', tags=['reasoning'], description='BIG-Bench Hard, 27 subtasks (each subset caches under bbh/).', - few_shot_split='official_cot', # -> bbh_few_shot hook (3-shot official CoT) + few_shot_split='official_cot', # -> bbh_few_shot hook (official CoT) + few_shot_num=3, # paper/es default: 3-shot ) ) def bbh():