Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Progress bars self-describing: overall bar carries the current benchmark (benches · 1/6 humaneval), sample bar carries the stage tag (humaneval · generating/scoring/writing)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 11:02:41 +00:00
parent dd7e1933ff
commit b7be2a8df3
2 changed files with 22 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
evalharness/cli.py
View File

@ -504,14 +504,20 @@ def _cmd_eval_run(args) -> int:
_shared_reporter.owned_externally = True _shared_reporter.owned_externally = True
progress_reporter = _shared_reporter progress_reporter = _shared_reporter
if total_runs > 1: if total_runs > 1:
progress_reporter.set_overall(total_runs, i, 'benches') progress_reporter.set_overall(
total_runs, i, f'benches · {i + 1}/{total_runs} {name}')
def status_callback(msg, _idx=i + 1, _name=name, def status_callback(msg, _idx=i + 1, _name=name,
_reporter=progress_reporter, _reporter=progress_reporter,
_console=console): _console=console):
if _reporter is not None: if _reporter is not None:
tag = f'[{_idx}/{total_runs}] ' if total_runs > 1 else '' _reporter.log(f'{_name}: {msg}')
_reporter.log(f'{tag}{_name}: {msg}') if 'scoring' in msg:
_reporter.set_phase('scoring')
elif 'generating model responses' in msg:
_reporter.set_phase('generating')
elif 'writing' in msg:
_reporter.set_phase('writing')
else: else:
_print_phase(_console, _idx, total_runs, _name, msg) _print_phase(_console, _idx, total_runs, _name, msg)
report = asyncio.run(run_eval( report = asyncio.run(run_eval(

15
evalharness/progress/rich_terminal.py
View File

@ -38,6 +38,7 @@ class RichTerminalProgress:
refresh_per_second=4, refresh_per_second=4,
) )
self.task_id = None self.task_id = None
self.bench_name = ''
self.started = 0.0 self.started = 0.0
self.current_started = 0.0 self.current_started = 0.0
self.inflight = 0 self.inflight = 0
@ -84,19 +85,29 @@ class RichTerminalProgress:
"""Start (or re-target) the per-sample task for the next benchmark.""" """Start (or re-target) the per-sample task for the next benchmark."""
self.started = time.monotonic() self.started = time.monotonic()
self.inflight = 0 self.inflight = 0
self.bench_name = description
desc = f'[green]{description} · generating[/green]'
if self.task_id is None: if self.task_id is None:
self.progress.start() self.progress.start()
self.task_id = self.progress.add_task( self.task_id = self.progress.add_task(
f'[green]{description}', total=total, completed=min(completed, total), desc, total=total, completed=min(completed, total),
success=completed, failed=0, rate='0.00', inflight=0, success=completed, failed=0, rate='0.00', inflight=0,
waiting='00:00', last_result='restored') waiting='00:00', last_result='restored')
self.heartbeat_task = asyncio.create_task(self._heartbeat()) self.heartbeat_task = asyncio.create_task(self._heartbeat())
else: else:
self.progress.update(self.task_id, description=f'[green]{description}', self.progress.update(self.task_id, description=desc,
total=total, completed=min(completed, total), total=total, completed=min(completed, total),
success=completed, failed=0, rate='0.00', success=completed, failed=0, rate='0.00',
inflight=0, last_result='restored') inflight=0, last_result='restored')
def set_phase(self, phase: str):
"""Retag the sample bar with what is happening (generating/scoring/
writing) -- the bar alone does not say which stage we are in."""
if self.task_id is not None:
self.progress.update(
self.task_id,
description=f'[green]{self.bench_name} · {phase}[/green]')
def begin_sample(self, label: str): def begin_sample(self, label: str):
if self.task_id is None: if self.task_id is None:
return return