51 lines
1.5 KiB
Python
51 lines
1.5 KiB
Python
"""Math benchmarks: gsm8k, aime24/25/26, hmmt26, imo_answerbench, competition_math."""
|
|
|
|
from ..recipe import EvalRecipe, register_eval
|
|
|
|
MATH_EXTRACT = ['math_boxed', 'answer_phrase', 'last_number']
|
|
MATH_SCORE = {'acc': 'math_equal'}
|
|
|
|
|
|
@register_eval('gsm8k')
|
|
def gsm8k():
|
|
return EvalRecipe(
|
|
name='gsm8k',
|
|
extract=['math_boxed', 'gsm8k_hash', 'answer_phrase', 'last_number'],
|
|
scorers={'acc': {'name': 'math_equal', 'sympy': False}}, # integer answers: no sympy needed
|
|
description='Grade-school math; #### and boxed markers, numeric compare.',
|
|
)
|
|
|
|
|
|
def _math_comp(name: str, desc: str) -> EvalRecipe:
|
|
return EvalRecipe(name=name, extract=MATH_EXTRACT, scorers=MATH_SCORE, description=desc)
|
|
|
|
|
|
@register_eval('aime24')
|
|
def aime24():
|
|
return _math_comp('aime24', 'AIME 2024; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('aime25')
|
|
def aime25():
|
|
return _math_comp('aime25', 'AIME 2025; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('aime26')
|
|
def aime26():
|
|
return _math_comp('aime26', 'AIME 2026; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('hmmt26')
|
|
def hmmt26():
|
|
return _math_comp('hmmt26', 'HMMT Feb 2026; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('imo_answerbench')
|
|
def imo_answerbench():
|
|
return _math_comp('imo_answerbench', 'IMO AnswerBench; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('competition_math')
|
|
def competition_math():
|
|
return _math_comp('competition_math', 'Hendrycks MATH; boxed + sympy (PRM800K grader).')
|