"""GPQA Diamond. Official source Idavidrein/gpqa is gated on HF (needs auth); we load the nmayorga7 CSV export which keeps the official column layout (Question / Correct Answer / Incorrect Answer 1-3). Note: choices are stored with the correct answer first (target 'A'). Option shuffling is an eval-time concern (the evaluator should shuffle choices and remap the target, like Dataset.shuffle_choices would). """ from ..sample import Sample from ..registry import register_dataset from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='gpqa_diamond', source='nmayorga7/gpqa_diamond', # HF CSV export of the gated official split='train', gen_config={'temperature': 1.0, 'max_tokens': 8192}, task_type='mcq', tags=['knowledge', 'science'], description='GPQA diamond split, graduate-level science MCQ (official content).', params={'hub': 'hf_raw'}, ) ) def gpqa_diamond(): def to_sample(record: dict) -> Sample: # position-bias protection, ported from the es adapter: deterministic # per-question shuffle (seed = sha256(question)) keeps reruns identical import hashlib import random as _rnd choices = [ str(record['Incorrect Answer 1'] or '').strip(), str(record['Incorrect Answer 2'] or '').strip(), str(record['Incorrect Answer 3'] or '').strip(), str(record['Correct Answer'] or '').strip(), ] seed = int.from_bytes( hashlib.sha256(str(record['Question']).strip().encode('utf-8')).digest()[:8], 'big') _rnd.Random(seed).shuffle(choices) target = 'ABCD'[choices.index(str(record['Correct Answer'] or '').strip())] return Sample( input=record['Question'], choices=choices, target=target, metadata={'subdomain': record.get('Subdomain')}, ) return to_sample