370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
54 lines
1.4 KiB
Python
54 lines
1.4 KiB
Python
"""MCQ benchmarks: mmlu, cmmlu, mmlu_pro, arc, hellaswag, winogrande,
|
|
gpqa_diamond, longbench_v2. All: extract a letter, compare to target letter."""
|
|
|
|
from ..recipe import EvalRecipe, register_eval
|
|
|
|
|
|
def _mcq(name: str, desc: str) -> EvalRecipe:
|
|
return EvalRecipe(
|
|
name=name,
|
|
extract=['mcq_letter'],
|
|
scorers={'acc': {'name': 'exact', 'mode': 'raw'}}, # letter == letter
|
|
description=desc,
|
|
)
|
|
|
|
|
|
@register_eval('mmlu')
|
|
def mmlu():
|
|
return _mcq('mmlu', 'MMLU; letter extraction vs answer letter.')
|
|
|
|
|
|
@register_eval('cmmlu')
|
|
def cmmlu():
|
|
return _mcq('cmmlu', 'CMMLU; letter extraction (supports 答案是X) vs answer letter.')
|
|
|
|
|
|
@register_eval('mmlu_pro')
|
|
def mmlu_pro():
|
|
return _mcq('mmlu_pro', 'MMLU-Pro 10-option; letter vs letter.')
|
|
|
|
|
|
@register_eval('arc')
|
|
def arc():
|
|
return _mcq('arc', 'AI2 ARC; letter vs answerKey.')
|
|
|
|
|
|
@register_eval('hellaswag')
|
|
def hellaswag():
|
|
return _mcq('hellaswag', 'HellaSwag; letter vs label (acc_norm needs logprobs: model layer).')
|
|
|
|
|
|
@register_eval('winogrande')
|
|
def winogrande():
|
|
return _mcq('winogrande', 'Winogrande; letter vs answer.')
|
|
|
|
|
|
@register_eval('gpqa_diamond')
|
|
def gpqa_diamond():
|
|
return _mcq('gpqa_diamond', 'GPQA diamond; letter vs target (choices shuffled at eval time).')
|
|
|
|
|
|
@register_eval('longbench_v2')
|
|
def longbench_v2():
|
|
return _mcq('longbench_v2', 'LongBench v2; letter vs answer.')
|