Idle watchdog back to 300s default; begin_bench clears the stale bar

- watchdog: 10s was the user's experiment; back to 5min default
  (EVALHARNESS_PULL_IDLE_S overrides). Zero-byte detection itself is
  sound: select(fd, timeout) + any-chunk reset, unit-verified.
- a FAILED bench left its 'scoring 0/2' bar on screen through the
  NEXT bench's entire dataset download; begin_bench() relabels to
  '<name> · loading' and zeroes counters at each bench start.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-17 06:08:33 +00:00
parent cfc1f2904e
commit ca8e1d6038
3 changed files with 24 additions and 1 deletions

View File

@ -745,6 +745,9 @@ def _cmd_eval_run(args) -> int:
try: try:
if _shared_reporter is not None and total_runs > 1: if _shared_reporter is not None and total_runs > 1:
_shared_reporter.set_bench_tag(f'[{i + 1}/{total_runs}]') _shared_reporter.set_bench_tag(f'[{i + 1}/{total_runs}]')
if _shared_reporter is not None:
_shared_reporter.begin_bench(name) # clear the previous
# bench's stale scoring bar before this one's dataset load
def _emit(msg, _i=i, _n=name): def _emit(msg, _i=i, _n=name):
if _shared_reporter is not None: if _shared_reporter is not None:

View File

@ -162,6 +162,26 @@ class RichTerminalProgress:
self.heartbeat_task.cancel() self.heartbeat_task.cancel()
self.heartbeat_task = asyncio.create_task(self._heartbeat()) self.heartbeat_task = asyncio.create_task(self._heartbeat())
def begin_bench(self, name: str):
"""New benchmark starting: immediately re-label the bar to its
loading phase. Without this the PREVIOUS bench's stale scoring
state (e.g. a failed '0/2') stayed on screen through the whole
dataset download of the next one."""
if self.disabled or self.task_id is None:
return
self.bench_name = name
self._last_phase = 'loading'
self._scoring_for = None
self.started = time.monotonic()
self.inflight = 0
self.restored = 0
self.admitted = None
self.progress.update(
self.task_id,
description=f'[green]{self.bench_tag}{name} · loading[/green]',
total=1, completed=0, new='', rate='0.00', inflight=0,
cur='0s', elapsed='0s', eta='-', retries=0)
def set_scoring(self, done: int, total: int): def set_scoring(self, done: int, total: int):
"""Retarget the SAME bar to the scoring phase: generation is finished """Retarget the SAME bar to the scoring phase: generation is finished
and its filled 100% state is stale -- now the bar refills with judged and its filled 100% state is stale -- now the bar refills with judged

View File

@ -166,7 +166,7 @@ def _pull_one(image: str, verbose: bool = True) -> None:
# override with EVALHARNESS_PULL_IDLE_S for slow-negotiating mirrors. # override with EVALHARNESS_PULL_IDLE_S for slow-negotiating mirrors.
import os as _os2 import os as _os2
_IDLE_S = float(_os2.environ.get('EVALHARNESS_PULL_IDLE_S', '10')) _IDLE_S = float(_os2.environ.get('EVALHARNESS_PULL_IDLE_S', '300'))
def _iter_lines(): def _iter_lines():
nonlocal buf nonlocal buf