370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
51 lines
1.5 KiB
Python
51 lines
1.5 KiB
Python
"""Math benchmarks: gsm8k, aime24/25/26, hmmt26, imo_answerbench, competition_math."""
|
|
|
|
from ..recipe import EvalRecipe, register_eval
|
|
|
|
MATH_EXTRACT = ['math_boxed', 'answer_phrase', 'last_number']
|
|
MATH_SCORE = {'acc': 'math_equal'}
|
|
|
|
|
|
@register_eval('gsm8k')
|
|
def gsm8k():
|
|
return EvalRecipe(
|
|
name='gsm8k',
|
|
extract=['math_boxed', 'gsm8k_hash', 'answer_phrase', 'last_number'],
|
|
scorers={'acc': 'math_equal'}, # numeric normalization handles $18/540 meters/70,000
|
|
description='Grade-school math; #### and boxed markers, numeric compare.',
|
|
)
|
|
|
|
|
|
def _math_comp(name: str, desc: str) -> EvalRecipe:
|
|
return EvalRecipe(name=name, extract=MATH_EXTRACT, scorers=MATH_SCORE, description=desc)
|
|
|
|
|
|
@register_eval('aime24')
|
|
def aime24():
|
|
return _math_comp('aime24', 'AIME 2024; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('aime25')
|
|
def aime25():
|
|
return _math_comp('aime25', 'AIME 2025; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('aime26')
|
|
def aime26():
|
|
return _math_comp('aime26', 'AIME 2026; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('hmmt26')
|
|
def hmmt26():
|
|
return _math_comp('hmmt26', 'HMMT Feb 2026; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('imo_answerbench')
|
|
def imo_answerbench():
|
|
return _math_comp('imo_answerbench', 'IMO AnswerBench; boxed extraction + sympy equivalence.')
|
|
|
|
|
|
@register_eval('competition_math')
|
|
def competition_math():
|
|
return _math_comp('competition_math', 'Hendrycks MATH; boxed + sympy (PRM800K grader).')
|