From 456d304a69e56ef1ee9f5ace869e2cb76db97039 Mon Sep 17 00:00:00 2001 From: sora Date: Wed, 26 Aug 2026 17:15:13 +0000 Subject: [PATCH] =?UTF-8?q?Fix=20numeric=20normalization=20in=20math=5Fequ?= =?UTF-8?q?al:=20strip=20$,=20commas,=20markdown=20(**),=20trailing=20unit?= =?UTF-8?q?s=20('540=20meters')=20=E2=80=94=20gsm8k=200.45->0.80=20on=20la?= =?UTF-8?q?dder20=20cross-check;=20winogrande=20cross-scoring=2020/20=20ag?= =?UTF-8?q?reement=20(delta=20=3D=20generation=20variance)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- evalharness/eval/recipes/math.py | 2 +- evalharness/eval/scorer.py | 13 ++++++++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/evalharness/eval/recipes/math.py b/evalharness/eval/recipes/math.py index c046e3a..46b1064 100644 --- a/evalharness/eval/recipes/math.py +++ b/evalharness/eval/recipes/math.py @@ -11,7 +11,7 @@ def gsm8k(): return EvalRecipe( name='gsm8k', extract=['math_boxed', 'gsm8k_hash', 'answer_phrase', 'last_number'], - scorers={'acc': {'name': 'math_equal', 'sympy': False}}, # integer answers: no sympy needed + scorers={'acc': 'math_equal'}, # numeric normalization handles $18/540 meters/70,000 description='Grade-school math; #### and boxed markers, numeric compare.', ) diff --git a/evalharness/eval/scorer.py b/evalharness/eval/scorer.py index a1e5700..96d40cb 100644 --- a/evalharness/eval/scorer.py +++ b/evalharness/eval/scorer.py @@ -61,13 +61,20 @@ class LayerNotReady(RuntimeError): def _strip_string(s: str) -> str: - """Light math normalization (subset of Hendrycks/Qwen strip_string).""" + """Light math normalization (subset of Hendrycks/Qwen strip_string) + + markdown/unit noise stripping ($, **, trailing words like 'meters').""" s = s.strip() s = re.sub(r'\\text\{(.+?)\}', r'\1', s) s = re.sub(r'\\!|\\,|\\;|\\ ', '', s) s = s.replace('\\%', '%').replace('\\$', '$').replace('$', '').replace('%', '') - s = s.replace('^{\\circ}', '').replace('^\\circ', '') - s = re.sub(r'(\d),(\d{3})', r'\1\2', s) + s = re.sub(r'\^\{\\circ\}|\^\\circ', '', s) + s = re.sub(r'(\d),(\d{3})', lambda m: m.group(1) + m.group(2), s) + # markdown remnants around numbers: 70,000** / **18 / ~~ etc + s = re.sub(r'([\$*_~`]+)(?=[-\d.])|(?<=[\d.%])([\$*_~`**]+)', '', s) + # trailing alpha units after a number: '540 meters', '366** downloads' + m = re.fullmatch(r'\s*(-?[\d,.]+)\s*[A-Za-z%]{0,12}\s*', s) + if m: + s = m.group(1) s = re.sub(r'\.0+(?=$|[^0-9])', '', s) if len(s) > 1 and s[0] == '{' and s[-1] == '}': s = s[1:-1]