370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
32 lines
1.1 KiB
Python
32 lines
1.1 KiB
Python
"""MMLU-Pro (official source: TIGER-Lab/MMLU-Pro)."""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='mmlu_pro',
|
|
source='TIGER-Lab/MMLU-Pro', # official: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro
|
|
split='test',
|
|
prompt_style='cot_letter_plain', # es mmlu-pro template (Question:/Options:/A x)
|
|
few_shot_split='validation',
|
|
few_shot_num=5,
|
|
task_type='mcq',
|
|
tags=['knowledge'],
|
|
description='MMLU-Pro: 10-option harder MMLU (official).',
|
|
)
|
|
)
|
|
def mmlu_pro():
|
|
def to_sample(record: dict) -> Sample:
|
|
return Sample(
|
|
input=record['question'],
|
|
choices=list(record['options']),
|
|
target=str(record['answer']).strip(), # already a letter
|
|
metadata={'category': record.get('category'), 'question_id': record.get('question_id'),
|
|
'cot_content': record.get('cot_content')}, # dev-split CoT exemplars (es few-shot style)
|
|
)
|
|
|
|
return to_sample
|