Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Persistent benchmark counter [i/N] on the single progress bar; phase lines all route through the live channel for strict ordering

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 11:13:34 +00:00
parent 9818515247
commit 392cc8daca
2 changed files with 24 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
evalharness/cli.py
View File

@ -487,8 +487,17 @@ def _cmd_eval_run(args) -> int:
for i, name in enumerate(args.datasets): for i, name in enumerate(args.datasets):
t0 = _time.time() t0 = _time.time()
try: try:
_print_phase(console, i + 1, total_runs, name, if _shared_reporter is not None and total_runs > 1:
'loading/downloading dataset') _shared_reporter.set_bench_tag(f'[{i + 1}/{total_runs}]')
def _emit(msg, _i=i, _n=name):
if _shared_reporter is not None:
_col = _phase_color(msg)
_shared_reporter.log(f'[{_col}][{_i + 1}/{total_runs}] {_n}: {msg}[/{_col}]')
else:
_print_phase(console, _i + 1, total_runs, _n, msg)
_emit('loading/downloading dataset')
ds = get_dataset(name, **overrides) ds = get_dataset(name, **overrides)
if _shared_reporter is not None: if _shared_reporter is not None:
_shared_reporter.pause() # let hub tqdm print cleanly _shared_reporter.pause() # let hub tqdm print cleanly
@ -496,8 +505,7 @@ def _cmd_eval_run(args) -> int:
if _shared_reporter is not None: if _shared_reporter is not None:
_shared_reporter.resume() _shared_reporter.resume()
origin = ds.lineage.get('from', 'unknown') origin = ds.lineage.get('from', 'unknown')
_print_phase(console, i + 1, total_runs, name, _emit(f'dataset ready · samples={sample_count} · source={origin}')
f'dataset ready · samples={sample_count} · source={origin}')
if model_spec: # generate + score in one go if model_spec: # generate + score in one go
from evalharness.model import run_eval from evalharness.model import run_eval
@ -556,8 +564,7 @@ def _cmd_eval_run(args) -> int:
if args.out: if args.out:
report.save(args.out) report.save(args.out)
if out_dir: if out_dir:
_print_phase(console, i + 1, total_runs, name, _emit('writing results')
'writing results')
from pathlib import Path as _P from pathlib import Path as _P
bench_dir = _P(out_dir) / name bench_dir = _P(out_dir) / name

13
evalharness/progress/rich_terminal.py
View File

@ -39,6 +39,8 @@ class RichTerminalProgress:
) )
self.task_id = None self.task_id = None
self.bench_name = '' self.bench_name = ''
self.bench_tag = '' # e.g. '[1/6]': persistent benchmark counter
self._last_phase = ''
self.started = 0.0 self.started = 0.0
self.current_started = 0.0 self.current_started = 0.0
self.inflight = 0 self.inflight = 0
@ -86,7 +88,7 @@ class RichTerminalProgress:
self.started = time.monotonic() self.started = time.monotonic()
self.inflight = 0 self.inflight = 0
self.bench_name = description self.bench_name = description
desc = f'[green]{description} · generating[/green]' desc = f'[green]{self.bench_tag}{description} · generating[/green]'
if self.task_id is None: if self.task_id is None:
self.progress.start() self.progress.start()
self.task_id = self.progress.add_task( self.task_id = self.progress.add_task(
@ -100,13 +102,20 @@ class RichTerminalProgress:
success=completed, failed=0, rate='0.00', success=completed, failed=0, rate='0.00',
inflight=0, last_result='restored') inflight=0, last_result='restored')
def set_bench_tag(self, tag: str):
"""Persistent counter shown on the sample bar, e.g. '[1/6]'."""
self.bench_tag = tag + ' ' if tag else ''
if self._last_phase:
self.set_phase(self._last_phase)
def set_phase(self, phase: str): def set_phase(self, phase: str):
self._last_phase = phase
"""Retag the sample bar with what is happening (generating/scoring/ """Retag the sample bar with what is happening (generating/scoring/
writing) -- the bar alone does not say which stage we are in.""" writing) -- the bar alone does not say which stage we are in."""
if self.task_id is not None: if self.task_id is not None:
self.progress.update( self.progress.update(
self.task_id, self.task_id,
description=f'[green]{self.bench_name} · {phase}[/green]') description=f'[green]{self.bench_tag}{self.bench_name} · {phase}[/green]')
def begin_sample(self, label: str): def begin_sample(self, label: str):
if self.task_id is None: if self.task_id is None: