Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Scoring phase retargets the same progress bar

Browse Source 
The bar kept its stale 100% generation state during scoring with only
the description counter moving. Now set_scoring() refills the bar with
judged samples (0->100%, fresh clock/rate/eta for the phase), clears
the generation '+N new' marker, and the next bench retargets back to
generating cleanly.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-14 03:51:25 +00:00
parent 25e51c5b30
commit e1a96f18c0
2 changed files with 31 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
evalharness/cli.py
View File

@ -720,9 +720,13 @@ def _cmd_eval_run(args) -> int:
for minutes with zero feedback otherwise (bar sits at for minutes with zero feedback otherwise (bar sits at
'generating 100%' and looks hung).""" 'generating 100%' and looks hung)."""
if _reporter is not None: if _reporter is not None:
# live terminal: the bar carries the counter -- log # live terminal: RETARGET the bar to scoring -- it
# lines here are pure noise (21 lines per bench) # refills 0->100% with judged samples
_reporter.set_phase(f'scoring {done}/{total_s}') _ss = getattr(_reporter, 'set_scoring', None)
if _ss is not None:
_ss(done, total_s)
else:
_reporter.set_phase(f'scoring {done}/{total_s}')
return return
# pipes/redirects (no live bar): milestone lines instead, # pipes/redirects (no live bar): milestone lines instead,
# first completion immediately then every ~10% # first completion immediately then every ~10%

24
evalharness/progress/rich_terminal.py
View File

@ -123,6 +123,7 @@ class RichTerminalProgress:
self.inflight = 0 self.inflight = 0
self.bench_name = description self.bench_name = description
self.restored = max(completed, 0) # checkpoint head start this bench self.restored = max(completed, 0) # checkpoint head start this bench
self._scoring_for = None # next scoring phase retargets anew
desc = f'[green]{self.bench_tag}{description} · generating[/green]' desc = f'[green]{self.bench_tag}{description} · generating[/green]'
if self.task_id is None: if self.task_id is None:
self.progress.start() self.progress.start()
@ -143,6 +144,29 @@ class RichTerminalProgress:
self.heartbeat_task.cancel() self.heartbeat_task.cancel()
self.heartbeat_task = asyncio.create_task(self._heartbeat()) self.heartbeat_task = asyncio.create_task(self._heartbeat())
def set_scoring(self, done: int, total: int):
"""Retarget the SAME bar to the scoring phase: generation is finished
and its filled 100% state is stale -- now the bar refills with judged
samples (0% -> 100%), with a fresh clock/eta for this phase."""
if self.disabled or self.task_id is None:
return
if getattr(self, '_scoring_for', None) != total:
# first callback of this scoring phase: restart the clock, clear
# generation markers (the '+N new' tag is about generating)
self._scoring_for = total
self._last_phase = 'scoring'
self.started = time.monotonic()
self.inflight = 0
self.restored = 0
elapsed = max(time.monotonic() - self.started, 1e-6)
self.progress.update(
self.task_id,
description=f'[green]{self.bench_tag}{self.bench_name} · scoring[/green]',
total=total, completed=min(done, total), new='',
rate=f'{done / elapsed:.2f}', inflight=0, cur='0s',
elapsed=_fmt(elapsed),
eta=_fmt((total - done) * elapsed / done) if done and total > done else '-')
def set_bench_tag(self, tag: str): def set_bench_tag(self, tag: str):
if self.disabled: if self.disabled:
return return