From e11cae038bf20f8e6b849ce02931527f30450eba Mon Sep 17 00:00:00 2001 From: sora Date: Thu, 27 Aug 2026 06:37:57 +0000 Subject: [PATCH] Parity pre-flight check (truncation/few-shot/judge/no-think endpoints vs es run.py); bbh few_shot_num=3 fixed (was 0, es/paper default 3-shot CoT) --- evalharness/data/datasets/bbh.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/evalharness/data/datasets/bbh.py b/evalharness/data/datasets/bbh.py index 5d3e09f..b86dfcd 100644 --- a/evalharness/data/datasets/bbh.py +++ b/evalharness/data/datasets/bbh.py @@ -28,7 +28,8 @@ def bbh_few_shot(split: str, subset: str, n: int): task_type='qa', tags=['reasoning'], description='BIG-Bench Hard, 27 subtasks (each subset caches under bbh/).', - few_shot_split='official_cot', # -> bbh_few_shot hook (3-shot official CoT) + few_shot_split='official_cot', # -> bbh_few_shot hook (official CoT) + few_shot_num=3, # paper/es default: 3-shot ) ) def bbh():