diff --git a/evalharness/eval/recipes/math.py b/evalharness/eval/recipes/math.py index c046e3a..46b1064 100644 --- a/evalharness/eval/recipes/math.py +++ b/evalharness/eval/recipes/math.py @@ -11,7 +11,7 @@ def gsm8k(): return EvalRecipe( name='gsm8k', extract=['math_boxed', 'gsm8k_hash', 'answer_phrase', 'last_number'], - scorers={'acc': {'name': 'math_equal', 'sympy': False}}, # integer answers: no sympy needed + scorers={'acc': 'math_equal'}, # numeric normalization handles $18/540 meters/70,000 description='Grade-school math; #### and boxed markers, numeric compare.', ) diff --git a/evalharness/eval/scorer.py b/evalharness/eval/scorer.py index a1e5700..96d40cb 100644 --- a/evalharness/eval/scorer.py +++ b/evalharness/eval/scorer.py @@ -61,13 +61,20 @@ class LayerNotReady(RuntimeError): def _strip_string(s: str) -> str: - """Light math normalization (subset of Hendrycks/Qwen strip_string).""" + """Light math normalization (subset of Hendrycks/Qwen strip_string) + + markdown/unit noise stripping ($, **, trailing words like 'meters').""" s = s.strip() s = re.sub(r'\\text\{(.+?)\}', r'\1', s) s = re.sub(r'\\!|\\,|\\;|\\ ', '', s) s = s.replace('\\%', '%').replace('\\$', '$').replace('$', '').replace('%', '') - s = s.replace('^{\\circ}', '').replace('^\\circ', '') - s = re.sub(r'(\d),(\d{3})', r'\1\2', s) + s = re.sub(r'\^\{\\circ\}|\^\\circ', '', s) + s = re.sub(r'(\d),(\d{3})', lambda m: m.group(1) + m.group(2), s) + # markdown remnants around numbers: 70,000** / **18 / ~~ etc + s = re.sub(r'([\$*_~`]+)(?=[-\d.])|(?<=[\d.%])([\$*_~`**]+)', '', s) + # trailing alpha units after a number: '540 meters', '366** downloads' + m = re.fullmatch(r'\s*(-?[\d,.]+)\s*[A-Za-z%]{0,12}\s*', s) + if m: + s = m.group(1) s = re.sub(r'\.0+(?=$|[^0-9])', '', s) if len(s) > 1 and s[0] == '{' and s[-1] == '}': s = s[1:-1]