Fix numeric normalization in math_equal: strip $, commas, markdown (**), trailing units ('540 meters') — gsm8k 0.45->0.80 on ladder20 cross-check; winogrande cross-scoring 20/20 agreement (delta = generation variance)

evalharness/eval/recipes/math.py

@@ -11,7 +11,7 @@ def gsm8k():
     return EvalRecipe(
         name='gsm8k',
         extract=['math_boxed', 'gsm8k_hash', 'answer_phrase', 'last_number'],
-        scorers={'acc': {'name': 'math_equal', 'sympy': False}},  # integer answers: no sympy needed
+        scorers={'acc': 'math_equal'},  # numeric normalization handles $18/540 meters/70,000
         description='Grade-school math; #### and boxed markers, numeric compare.',
     )
 

evalharness/eval/scorer.py

@@ -61,13 +61,20 @@ class LayerNotReady(RuntimeError):
 
 
 def _strip_string(s: str) -> str:
-    """Light math normalization (subset of Hendrycks/Qwen strip_string)."""
+    """Light math normalization (subset of Hendrycks/Qwen strip_string) +
+    markdown/unit noise stripping ($, **, trailing words like 'meters')."""
     s = s.strip()
     s = re.sub(r'\\text\{(.+?)\}', r'\1', s)
     s = re.sub(r'\\!|\\,|\\;|\\ ', '', s)
     s = s.replace('\\%', '%').replace('\\$', '$').replace('$', '').replace('%', '')
-    s = s.replace('^{\\circ}', '').replace('^\\circ', '')
-    s = re.sub(r'(\d),(\d{3})', r'\1\2', s)
+    s = re.sub(r'\^\{\\circ\}|\^\\circ', '', s)
+    s = re.sub(r'(\d),(\d{3})', lambda m: m.group(1) + m.group(2), s)
+    # markdown remnants around numbers: 70,000** / **18 / ~~ etc
+    s = re.sub(r'([\$*_~`]+)(?=[-\d.])|(?<=[\d.%])([\$*_~`**]+)', '', s)
+    # trailing alpha units after a number: '540 meters', '366** downloads'
+    m = re.fullmatch(r'\s*(-?[\d,.]+)\s*[A-Za-z%]{0,12}\s*', s)
+    if m:
+        s = m.group(1)
     s = re.sub(r'\.0+(?=$|[^0-9])', '', s)
     if len(s) > 1 and s[0] == '{' and s[-1] == '}':
         s = s[1:-1]