Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GPQA deterministic per-question choice shuffle (es-adapter parity: sha256(question) seed, position-bias protection) — was fixed-A; lb2 subset grouping aligned to es (length field, 60 samples)

Browse Source
This commit is contained in:
sora 2026-08-28 16:24:11 +00:00
parent f7c7536400
commit 60b2bfc0dc
3 changed files with 19 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
evalharness/data/datasets/gpqa_diamond.py
View File

@ -28,17 +28,26 @@ from ..spec import DatasetSpec
) )
def gpqa_diamond(): def gpqa_diamond():
def to_sample(record: dict) -> Sample: def to_sample(record: dict) -> Sample:
# position-bias protection, ported from the es adapter: deterministic
# per-question shuffle (seed = sha256(question)) keeps reruns identical
import hashlib
import random as _rnd
choices = [ choices = [
record['Correct Answer'], str(record['Incorrect Answer 1'] or '').strip(),
record['Incorrect Answer 1'], str(record['Incorrect Answer 2'] or '').strip(),
record['Incorrect Answer 2'], str(record['Incorrect Answer 3'] or '').strip(),
record['Incorrect Answer 3'], str(record['Correct Answer'] or '').strip(),
] ]
seed = int.from_bytes(
hashlib.sha256(str(record['Question']).strip().encode('utf-8')).digest()[:8], 'big')
_rnd.Random(seed).shuffle(choices)
target = 'ABCD'[choices.index(str(record['Correct Answer'] or '').strip())]
return Sample( return Sample(
input=record['Question'], input=record['Question'],
choices=choices, choices=choices,
target='A', # correct answer is first; shuffle at eval time target=target,
metadata={'subdomain': record.get('Subdomain'), 'unshuffled': True}, metadata={'subdomain': record.get('Subdomain')},
) )
return to_sample return to_sample

1
evalharness/data/datasets/longbench_v2.py
View File

@ -27,6 +27,7 @@ def longbench_v2():
'sub_domain': record.get('sub_domain'), 'sub_domain': record.get('sub_domain'),
'difficulty': record.get('difficulty'), 'difficulty': record.get('difficulty'),
'length': record.get('length'), 'length': record.get('length'),
'subset': record.get('length'), # official subsets: short/medium/long
}, },
) )

4
evalharness/model/runner.py
View File

@ -271,9 +271,11 @@ def _apply_limits(samples: List[Sample], total: Optional[int],
seen: Dict[str, int] = {} seen: Dict[str, int] = {}
out = [] out = []
for s in samples: for s in samples:
key = str((s.metadata or {}).get('category') key = str((s.metadata or {}).get('subset')
or (s.metadata or {}).get('category')
or (s.metadata or {}).get('subject') or (s.metadata or {}).get('subject')
or (s.metadata or {}).get('test_category') or (s.metadata or {}).get('test_category')
or (s.metadata or {}).get('length')
or getattr(getattr(dataset, 'spec', None), 'subset', 'default')) or getattr(getattr(dataset, 'spec', None), 'subset', 'default'))
if seen.get(key, 0) < per_task: if seen.get(key, 0) < per_task:
seen[key] = seen.get(key, 0) + 1 seen[key] = seen.get(key, 0) + 1