Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scoring line: colon restored, full sentence (score over N samples), score keeps solid green with yellow phrase tail; path coloring covers 'to /abs' endings

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 03:16:08 +00:00
parent d7f06395a0
commit 0ca8aedefe
2 changed files with 17 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
evalharness/cli.py
View File

@ -228,10 +228,16 @@ def _narration(msg: str) -> str:
# gets bold green -- it is what the eye should find first # gets bold green -- it is what the eye should find first
import re as _re1 import re as _re1
m = _re1.search(r'[:·] ([a-zA-Z_@]+ [0-9.]+%)$', msg) m = _re1.search(r'[:·] ([a-zA-Z_@]+ [0-9.]+%)(?=\s|$)', msg)
if m: if m:
return (f'{icon}{msg[:m.start()]}' head = f'{icon}{msg[:m.start()]}: [bold green]{m.group(1)}[/bold green]'
f'[bold green]{m.group(1)}[/bold green]') tail = msg[m.end():]
if tail:
import re as _re2
tail = _re2.sub(r'(?<![\w/%.])(\d+(?:/\d+)?(?:\s+[a-z-]+){0,4})(?=[\s,.]|$)',
r'[bold yellow]\1[/bold yellow]', tail)
return head + tail
# semantic PHRASES get color, not bare numbers: '164 samples from cache', # semantic PHRASES get color, not bare numbers: '164 samples from cache',
# '4/4 predictions' -- a lone number is easy to lose. Runs AFTER the # '4/4 predictions' -- a lone number is easy to lose. Runs AFTER the
@ -240,10 +246,11 @@ def _narration(msg: str) -> str:
msg = _re0.sub(r'(?<![\w/%.])(\d+(?:/\d+)?(?:\s+[a-z-]+){0,4})(?=[\s,.]|$)', msg = _re0.sub(r'(?<![\w/%.])(\d+(?:/\d+)?(?:\s+[a-z-]+){0,4})(?=[\s,.]|$)',
r'[bold yellow]\1[/bold yellow]', msg) r'[bold yellow]\1[/bold yellow]', msg)
# highlight any trailing path after '->' # highlight any trailing path after '->' or 'to' (only absolute paths)
if '-> ' in msg: for sep in ('-> ', 'to '):
head, _, tail = msg.partition('-> ') head, _, tail = msg.rpartition(sep)
return f'{icon}{head}-> [cyan]{tail}[/cyan]' if head and tail.startswith('/'):
return f'{icon}{head}{sep}[cyan]{tail}[/cyan]'
return f'{icon}{msg}' return f'{icon}{msg}'

5
evalharness/model/runner.py
View File

@ -746,8 +746,9 @@ async def run_eval(
if status_callback: if status_callback:
_m = next(((k, v) for k, v in report.metrics.items() _m = next(((k, v) for k, v in report.metrics.items()
if k != 'extraction_failure_rate'), None) if k != 'extraction_failure_rate'), None)
status_callback(f'Scoring complete: {_m[0]} {_m[1] * 100:.1f}%' status_callback(f'Scoring complete: {_m[0]} {_m[1] * 100:.1f}% '
if _m else 'scoring complete') f'over {report.num_samples} samples'
if _m else 'Scoring complete')
# performance profile: pool success rate + latency/ttft percentiles # performance profile: pool success rate + latency/ttft percentiles
try: try:
from .aggregator import get_aggregator from .aggregator import get_aggregator