Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Narration colors by stage semantics: start/in-progress events yellow, completion events green (dataset ready / generation complete / scoring complete / checkpoint restored)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 11:09:38 +00:00
parent 89f950d8a0
commit 9818515247
1 changed files with 19 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
evalharness/cli.py
View File

@ -198,13 +198,26 @@ def _print_run_plan(console, args, model_spec):
justify='center') justify='center')
def _phase_color(message: str) -> str:
"""Start/finish color code: events that START or are IN progress are
yellow; events that COMPLETE are green. One glance = what stage,
did it finish."""
m = message.lower()
if any(k in m for k in ('complete', 'ready', 'downloaded', 'parsed',
'restored', 'ok (')):
return 'green'
return 'yellow'
def _print_phase(console, index, total, name, message): def _print_phase(console, index, total, name, message):
# single-benchmark runs: the [1/1] tag is noise, drop it. # single-benchmark runs: the [1/1] tag is noise, drop it
# narration lines stay UNCOLORED everywhere (matches reporter.log and
# the runner's plain prints): color is reserved for ✓/✗/errors only
prefix = f'[{index}/{total}] ' if total > 1 else '' prefix = f'[{index}/{total}] ' if total > 1 else ''
text = f'{prefix}{name}: {message}' text = f'{prefix}{name}: {message}'
print(text, flush=True) color = _phase_color(message)
if console is not None:
console.print(f'[{color}]{text}[/{color}]', highlight=False)
else:
print(text, flush=True)
def _print_benchmark_result(console, index, total, name, status, elapsed): def _print_benchmark_result(console, index, total, name, status, elapsed):
@ -508,7 +521,8 @@ def _cmd_eval_run(args) -> int:
_reporter=progress_reporter, _reporter=progress_reporter,
_console=console): _console=console):
if _reporter is not None: if _reporter is not None:
_reporter.log(f'{_name}: {msg}') _col = _phase_color(msg)
_reporter.log(f'[{_col}]{_name}: {msg}[/{_col}]')
if 'scoring' in msg: if 'scoring' in msg:
_reporter.set_phase('scoring') _reporter.set_phase('scoring')
elif 'generating model responses' in msg: elif 'generating model responses' in msg: