Parity pre-flight check (truncation/few-shot/judge/no-think endpoints vs es run.py); bbh few_shot_num=3 fixed (was 0, es/paper default 3-shot CoT)

2026-08-27 06:37:57 +00:00 · 2026-08-27 06:37:57 +00:00 · e11cae038b
commit e11cae038b
parent e32da10ce9
1 changed files with 2 additions and 1 deletions
--- a/evalharness/data/datasets/bbh.py
+++ b/evalharness/data/datasets/bbh.py
@ -28,7 +28,8 @@ def bbh_few_shot(split: str, subset: str, n: int):
        task_type='qa',
        tags=['reasoning'],
        description='BIG-Bench Hard, 27 subtasks (each subset caches under bbh/<hash>).',
-        few_shot_split='official_cot',  # -> bbh_few_shot hook (3-shot official CoT)
+        few_shot_split='official_cot',  # -> bbh_few_shot hook (official CoT)
+        few_shot_num=3,               # paper/es default: 3-shot
    )
 )
 def bbh():