54 lines
1.9 KiB
Python
54 lines
1.9 KiB
Python
"""GPQA Diamond.
|
|
|
|
Official source Idavidrein/gpqa is gated on HF (needs auth); we load the
|
|
nmayorga7 CSV export which keeps the official column layout (Question /
|
|
Correct Answer / Incorrect Answer 1-3).
|
|
|
|
Note: choices are stored with the correct answer first (target 'A').
|
|
Option shuffling is an eval-time concern (the evaluator should shuffle
|
|
choices and remap the target, like Dataset.shuffle_choices would).
|
|
"""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='gpqa_diamond',
|
|
source='nmayorga7/gpqa_diamond', # HF CSV export of the gated official
|
|
split='train',
|
|
gen_config={'temperature': 1.0, 'max_tokens': 8192},
|
|
task_type='mcq',
|
|
tags=['knowledge', 'science'],
|
|
description='GPQA diamond split, graduate-level science MCQ (official content).',
|
|
params={'hub': 'hf_raw'},
|
|
)
|
|
)
|
|
def gpqa_diamond():
|
|
def to_sample(record: dict) -> Sample:
|
|
# position-bias protection, ported from the es adapter: deterministic
|
|
# per-question shuffle (seed = sha256(question)) keeps reruns identical
|
|
import hashlib
|
|
import random as _rnd
|
|
|
|
choices = [
|
|
str(record['Incorrect Answer 1'] or '').strip(),
|
|
str(record['Incorrect Answer 2'] or '').strip(),
|
|
str(record['Incorrect Answer 3'] or '').strip(),
|
|
str(record['Correct Answer'] or '').strip(),
|
|
]
|
|
seed = int.from_bytes(
|
|
hashlib.sha256(str(record['Question']).strip().encode('utf-8')).digest()[:8], 'big')
|
|
_rnd.Random(seed).shuffle(choices)
|
|
target = 'ABCD'[choices.index(str(record['Correct Answer'] or '').strip())]
|
|
return Sample(
|
|
input=record['Question'],
|
|
choices=choices,
|
|
target=target,
|
|
metadata={'subdomain': record.get('Subdomain')},
|
|
)
|
|
|
|
return to_sample
|