diff --git a/evalharness/data/datasets/gpqa_diamond.py b/evalharness/data/datasets/gpqa_diamond.py index 7d6f8fe..7331887 100644 --- a/evalharness/data/datasets/gpqa_diamond.py +++ b/evalharness/data/datasets/gpqa_diamond.py @@ -28,17 +28,26 @@ from ..spec import DatasetSpec ) def gpqa_diamond(): def to_sample(record: dict) -> Sample: + # position-bias protection, ported from the es adapter: deterministic + # per-question shuffle (seed = sha256(question)) keeps reruns identical + import hashlib + import random as _rnd + choices = [ - record['Correct Answer'], - record['Incorrect Answer 1'], - record['Incorrect Answer 2'], - record['Incorrect Answer 3'], + str(record['Incorrect Answer 1'] or '').strip(), + str(record['Incorrect Answer 2'] or '').strip(), + str(record['Incorrect Answer 3'] or '').strip(), + str(record['Correct Answer'] or '').strip(), ] + seed = int.from_bytes( + hashlib.sha256(str(record['Question']).strip().encode('utf-8')).digest()[:8], 'big') + _rnd.Random(seed).shuffle(choices) + target = 'ABCD'[choices.index(str(record['Correct Answer'] or '').strip())] return Sample( input=record['Question'], choices=choices, - target='A', # correct answer is first; shuffle at eval time - metadata={'subdomain': record.get('Subdomain'), 'unshuffled': True}, + target=target, + metadata={'subdomain': record.get('Subdomain')}, ) return to_sample diff --git a/evalharness/data/datasets/longbench_v2.py b/evalharness/data/datasets/longbench_v2.py index 27c7c1b..2b5466c 100644 --- a/evalharness/data/datasets/longbench_v2.py +++ b/evalharness/data/datasets/longbench_v2.py @@ -27,6 +27,7 @@ def longbench_v2(): 'sub_domain': record.get('sub_domain'), 'difficulty': record.get('difficulty'), 'length': record.get('length'), + 'subset': record.get('length'), # official subsets: short/medium/long }, ) diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 4e51fea..e6867a6 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -271,9 +271,11 @@ def _apply_limits(samples: List[Sample], total: Optional[int], seen: Dict[str, int] = {} out = [] for s in samples: - key = str((s.metadata or {}).get('category') + key = str((s.metadata or {}).get('subset') + or (s.metadata or {}).get('category') or (s.metadata or {}).get('subject') or (s.metadata or {}).get('test_category') + or (s.metadata or {}).get('length') or getattr(getattr(dataset, 'spec', None), 'subset', 'default')) if seen.get(key, 0) < per_task: seen[key] = seen.get(key, 0) + 1