370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
75 lines
2.9 KiB
Python
75 lines
2.9 KiB
Python
"""GPQA Diamond.
|
|
|
|
Official source Idavidrein/gpqa is gated on HF (needs auth); we load the
|
|
nmayorga7 CSV export which keeps the official column layout (Question /
|
|
Correct Answer / Incorrect Answer 1-3).
|
|
|
|
Note: choices are stored with the correct answer first (target 'A').
|
|
Option shuffling is an eval-time concern (the evaluator should shuffle
|
|
choices and remap the target, like Dataset.shuffle_choices would).
|
|
"""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='gpqa_diamond',
|
|
source='nmayorga7/gpqa_diamond', # HF CSV export of the gated official
|
|
split='train',
|
|
gen_config={'temperature': 1.0, 'max_tokens': 8192},
|
|
prompt_style='cot_letter', # CoT then ANSWER:X (es GPQA template)
|
|
task_type='mcq',
|
|
tags=['knowledge', 'science'],
|
|
description='GPQA diamond split, graduate-level science MCQ (official content).',
|
|
params={'hub': 'hf_raw'},
|
|
# order_policy: 'sha256' (deterministic per-question shuffle -- position
|
|
# bias protection) | 'es-dump:<path>' (pin the exact option order es
|
|
# used in a specific run, for same-order alignment) | 'official'
|
|
# (keep the CSV's raw order: incorrect 1-3 then correct)
|
|
# set via spec params at get_dataset time or the default below.
|
|
)
|
|
)
|
|
def gpqa_diamond(order_policy: str = 'sha256', order_dump: str = ''):
|
|
import hashlib
|
|
import random as _rnd
|
|
import json as _json
|
|
import os as _os
|
|
|
|
dump = {}
|
|
if order_policy.startswith('es-dump'):
|
|
path = order_dump or order_policy.split(':', 1)[1] if ':' in order_policy else order_dump
|
|
path = path or _os.environ.get('EVALHARNESS_CACHE', '') + '/../gpqa_es_order.json'
|
|
if _os.path.exists(path):
|
|
dump = _json.load(open(path))
|
|
|
|
def to_sample(record: dict) -> Sample:
|
|
choices = [
|
|
str(record['Incorrect Answer 1'] or '').strip(),
|
|
str(record['Incorrect Answer 2'] or '').strip(),
|
|
str(record['Incorrect Answer 3'] or '').strip(),
|
|
str(record['Correct Answer'] or '').strip(),
|
|
]
|
|
q = str(record['Question']).strip()
|
|
if q in dump:
|
|
# pinned order from an es run dump
|
|
choices = list(dump[q]['order'])
|
|
target = dump[q]['target']
|
|
elif order_policy == 'sha256':
|
|
seed = int.from_bytes(
|
|
hashlib.sha256(q.encode('utf-8')).digest()[:8], 'big')
|
|
_rnd.Random(seed).shuffle(choices)
|
|
target = 'ABCD'[choices.index(str(record['Correct Answer'] or '').strip())]
|
|
else: # 'official': raw order, correct is D
|
|
target = 'D'
|
|
return Sample(
|
|
input=record['Question'],
|
|
choices=choices,
|
|
target=target,
|
|
metadata={'subdomain': record.get('Subdomain')},
|
|
)
|
|
|
|
return to_sample
|