Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Narration lines get stage icons (⬇ download / 📦 ready / ✳ few-shot / ◷ checkpoint / 🤖 generating / ⏭ skipped / ★ scoring / 📝 writing / 🔗 endpoint) and cyan-highlighted paths

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 11:29:07 +00:00
parent 3a045e0569
commit e7792a559c
1 changed files with 40 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
evalharness/cli.py
View File

@ -198,6 +198,39 @@ def _print_run_plan(console, args, model_spec):
justify='center')
def _narration(msg: str) -> str:
"""Icon + path-highlighting for narration lines (visual separation at
a glance; paths in cyan)."""
icon = ''
m = msg.lower()
if m.startswith('loading/'):
icon = '⬇ '
elif 'dataset ready' in m:
icon = '📦 '
elif 'few-shot' in m:
icon = '✳ '
elif 'checkpoint' in m or m.endswith('samples') or 'to generate' in m:
icon = '◷ '
elif 'generation skipped' in m:
icon = '⏭ '
elif 'generating' in m:
icon = '🤖 '
elif 'generation complete' in m:
icon = '✓ '
elif 'scoring' in m:
icon = '★ '
elif 'writing' in m:
icon = '📝 '
elif 'endpoint ok' in m:
icon = '🔗 '
# highlight any trailing path after '->'
if '-> ' in msg:
head, _, tail = msg.partition('-> ')
return f'{icon}{head}-> [cyan]{tail}[/cyan]'
return f'{icon}{msg}'
def _phase_color(message: str) -> str:
"""Narration line color. Currently PLAIN WHITE for everything (user
preference); flip the returns to 'yellow'/'green' to restore the
@ -213,7 +246,7 @@ def _phase_color(message: str) -> str:
def _print_phase(console, index, total, name, message):
# single-benchmark runs: the [1/1] tag is noise, drop it
prefix = f'[{index}/{total}] ' if total > 1 else ''
text = f'{prefix}{name}: {message}'
text = f'{prefix}{name}: {_narration(message)}'
color = _phase_color(message)
if console is not None:
if color:
@ -221,7 +254,9 @@ def _print_phase(console, index, total, name, message):
else:
console.print(text, highlight=False)
else:
print(text, flush=True)
import re as _re0
print(_re0.sub(r'\[/?[a-z ]+\]', '', text), flush=True)
def _print_benchmark_result(console, index, total, name, status, elapsed):
@ -496,9 +531,8 @@ def _cmd_eval_run(args) -> int:
def _emit(msg, _i=i, _n=name):
if _shared_reporter is not None:
_col = _phase_color(msg)
_line = f'[{_i + 1}/{total_runs}] {_n}: {msg}'
_shared_reporter.log(f'[{_col}]{_line}[/{_col}]' if _col else _line)
_shared_reporter.log(
f'[{_i + 1}/{total_runs}] {_n}: {_narration(msg)}')
else:
_print_phase(console, _i + 1, total_runs, _n, msg)
@ -535,9 +569,7 @@ def _cmd_eval_run(args) -> int:
_console=console):
_tag = f'[{_idx}/{total_runs}] ' if total_runs > 1 else ''
if _reporter is not None:
_col = _phase_color(msg)
_line = f'{_tag}{_name}: {msg}'
_reporter.log(f'[{_col}]{_line}[/{_col}]' if _col else _line)
_reporter.log(f'{_tag}{_name}: {_narration(msg)}')
if 'scoring' in msg:
_reporter.set_phase('scoring')
elif 'generating model responses' in msg: