54 lines
1.4 KiB
Python
54 lines
1.4 KiB
Python
"""MCQ benchmarks: mmlu, cmmlu, mmlu_pro, arc, hellaswag, winogrande,
|
|
gpqa_diamond, longbench_v2. All: extract a letter, compare to target letter."""
|
|
|
|
from ..recipe import EvalRecipe, register_eval
|
|
|
|
|
|
def _mcq(name: str, desc: str) -> EvalRecipe:
|
|
return EvalRecipe(
|
|
name=name,
|
|
extract=['mcq_letter'],
|
|
scorers={'acc': {'name': 'exact', 'mode': 'raw'}}, # letter == letter
|
|
description=desc,
|
|
)
|
|
|
|
|
|
@register_eval('mmlu')
|
|
def mmlu():
|
|
return _mcq('mmlu', 'MMLU; letter extraction vs answer letter.')
|
|
|
|
|
|
@register_eval('cmmlu')
|
|
def cmmlu():
|
|
return _mcq('cmmlu', 'CMMLU; letter extraction (supports 答案是X) vs answer letter.')
|
|
|
|
|
|
@register_eval('mmlu_pro')
|
|
def mmlu_pro():
|
|
return _mcq('mmlu_pro', 'MMLU-Pro 10-option; letter vs letter.')
|
|
|
|
|
|
@register_eval('arc')
|
|
def arc():
|
|
return _mcq('arc', 'AI2 ARC; letter vs answerKey.')
|
|
|
|
|
|
@register_eval('hellaswag')
|
|
def hellaswag():
|
|
return _mcq('hellaswag', 'HellaSwag; letter vs label (acc_norm needs logprobs: model layer).')
|
|
|
|
|
|
@register_eval('winogrande')
|
|
def winogrande():
|
|
return _mcq('winogrande', 'Winogrande; letter vs answer.')
|
|
|
|
|
|
@register_eval('gpqa_diamond')
|
|
def gpqa_diamond():
|
|
return _mcq('gpqa_diamond', 'GPQA diamond; letter vs target (choices shuffled at eval time).')
|
|
|
|
|
|
@register_eval('longbench_v2')
|
|
def longbench_v2():
|
|
return _mcq('longbench_v2', 'LongBench v2; letter vs answer.')
|