Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixes: theme markup closing tags ([bold green]...[/bold green]); evaluate() runs off the event loop (sympy equivalence on hard math froze the progress clock for the whole bench)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 04:35:27 +00:00
parent 09b2add673
commit 8bac3483b6
2 changed files with 6 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
evalharness/model/runner.py
View File

@ -745,7 +745,11 @@ async def run_eval(
if status_callback: if status_callback:
status_callback('Scoring predictions against the benchmark recipe') status_callback('Scoring predictions against the benchmark recipe')
report = evaluate( # scoring off the event loop: math_equal/sympy equivalence can chew a
# single hard problem for minutes (es's checker famously hangs on one) --
# running it inline froze the progress bar's clock for the whole bench
report = await asyncio.to_thread(
evaluate,
samples, preds, recipe, samples, preds, recipe,
model=model_spec, model=model_spec,
judge=judge, judge=judge,

2
evalharness/themes/default.py
View File

@ -32,7 +32,7 @@ def narrate(msg: str) -> str:
m = re.search(r'[:·] ([a-zA-Z_@]+ [0-9.]+%)(?=\s|$)', msg) m = re.search(r'[:·] ([a-zA-Z_@]+ [0-9.]+%)(?=\s|$)', msg)
if m: if m:
head = f'{icon}{msg[:m.start()]}: [bold {FACT_COLOR}]{m.group(1)}[/{FACT_COLOR}]' head = f'{icon}{msg[:m.start()]}: [bold {FACT_COLOR}]{m.group(1)}[/bold {FACT_COLOR}]'
return head + fact(msg[m.end():]) return head + fact(msg[m.end():])
for sep in ('-> ', 'to '): for sep in ('-> ', 'to '):
head, _, tail = msg.rpartition(sep) head, _, tail = msg.rpartition(sep)