Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix numeric normalization in math_equal: strip $, commas, markdown (**), trailing units ('540 meters') — gsm8k 0.45->0.80 on ladder20 cross-check; winogrande cross-scoring 20/20 agreement (delta = generation variance)

Browse Source
This commit is contained in:
sora 2026-08-26 17:15:13 +00:00
parent 111336cbee
commit 456d304a69
2 changed files with 11 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
evalharness/eval/recipes/math.py
View File

@ -11,7 +11,7 @@ def gsm8k():
return EvalRecipe( return EvalRecipe(
name='gsm8k', name='gsm8k',
extract=['math_boxed', 'gsm8k_hash', 'answer_phrase', 'last_number'], extract=['math_boxed', 'gsm8k_hash', 'answer_phrase', 'last_number'],
scorers={'acc': {'name': 'math_equal', 'sympy': False}}, # integer answers: no sympy needed scorers={'acc': 'math_equal'}, # numeric normalization handles $18/540 meters/70,000
description='Grade-school math; #### and boxed markers, numeric compare.', description='Grade-school math; #### and boxed markers, numeric compare.',
) )

13
evalharness/eval/scorer.py
View File

@ -61,13 +61,20 @@ class LayerNotReady(RuntimeError):
def _strip_string(s: str) -> str: def _strip_string(s: str) -> str:
"""Light math normalization (subset of Hendrycks/Qwen strip_string).""" """Light math normalization (subset of Hendrycks/Qwen strip_string) +
markdown/unit noise stripping ($, **, trailing words like 'meters')."""
s = s.strip() s = s.strip()
s = re.sub(r'\\text\{(.+?)\}', r'\1', s) s = re.sub(r'\\text\{(.+?)\}', r'\1', s)
s = re.sub(r'\\!|\\,|\\;|\\ ', '', s) s = re.sub(r'\\!|\\,|\\;|\\ ', '', s)
s = s.replace('\\%', '%').replace('\\$', '$').replace('$', '').replace('%', '') s = s.replace('\\%', '%').replace('\\$', '$').replace('$', '').replace('%', '')
s = s.replace('^{\\circ}', '').replace('^\\circ', '') s = re.sub(r'\^\{\\circ\}|\^\\circ', '', s)
s = re.sub(r'(\d),(\d{3})', r'\1\2', s) s = re.sub(r'(\d),(\d{3})', lambda m: m.group(1) + m.group(2), s)
# markdown remnants around numbers: 70,000** / **18 / ~~ etc
s = re.sub(r'([\$*_~`]+)(?=[-\d.])|(?<=[\d.%])([\$*_~`**]+)', '', s)
# trailing alpha units after a number: '540 meters', '366** downloads'
m = re.fullmatch(r'\s*(-?[\d,.]+)\s*[A-Za-z%]{0,12}\s*', s)
if m:
s = m.group(1)
s = re.sub(r'\.0+(?=$|[^0-9])', '', s) s = re.sub(r'\.0+(?=$|[^0-9])', '', s)
if len(s) > 1 and s[0] == '{' and s[-1] == '}': if len(s) > 1 and s[0] == '{' and s[-1] == '}':
s = s[1:-1] s = s[1:-1]