Idle watchdog back to 300s default; begin_bench clears the stale bar
- watchdog: 10s was the user's experiment; back to 5min default (EVALHARNESS_PULL_IDLE_S overrides). Zero-byte detection itself is sound: select(fd, timeout) + any-chunk reset, unit-verified. - a FAILED bench left its 'scoring 0/2' bar on screen through the NEXT bench's entire dataset download; begin_bench() relabels to '<name> · loading' and zeroes counters at each bench start. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
cfc1f2904e
commit
ca8e1d6038
@ -745,6 +745,9 @@ def _cmd_eval_run(args) -> int:
|
|||||||
try:
|
try:
|
||||||
if _shared_reporter is not None and total_runs > 1:
|
if _shared_reporter is not None and total_runs > 1:
|
||||||
_shared_reporter.set_bench_tag(f'[{i + 1}/{total_runs}]')
|
_shared_reporter.set_bench_tag(f'[{i + 1}/{total_runs}]')
|
||||||
|
if _shared_reporter is not None:
|
||||||
|
_shared_reporter.begin_bench(name) # clear the previous
|
||||||
|
# bench's stale scoring bar before this one's dataset load
|
||||||
|
|
||||||
def _emit(msg, _i=i, _n=name):
|
def _emit(msg, _i=i, _n=name):
|
||||||
if _shared_reporter is not None:
|
if _shared_reporter is not None:
|
||||||
|
|||||||
@ -162,6 +162,26 @@ class RichTerminalProgress:
|
|||||||
self.heartbeat_task.cancel()
|
self.heartbeat_task.cancel()
|
||||||
self.heartbeat_task = asyncio.create_task(self._heartbeat())
|
self.heartbeat_task = asyncio.create_task(self._heartbeat())
|
||||||
|
|
||||||
|
def begin_bench(self, name: str):
|
||||||
|
"""New benchmark starting: immediately re-label the bar to its
|
||||||
|
loading phase. Without this the PREVIOUS bench's stale scoring
|
||||||
|
state (e.g. a failed '0/2') stayed on screen through the whole
|
||||||
|
dataset download of the next one."""
|
||||||
|
if self.disabled or self.task_id is None:
|
||||||
|
return
|
||||||
|
self.bench_name = name
|
||||||
|
self._last_phase = 'loading'
|
||||||
|
self._scoring_for = None
|
||||||
|
self.started = time.monotonic()
|
||||||
|
self.inflight = 0
|
||||||
|
self.restored = 0
|
||||||
|
self.admitted = None
|
||||||
|
self.progress.update(
|
||||||
|
self.task_id,
|
||||||
|
description=f'[green]{self.bench_tag}{name} · loading[/green]',
|
||||||
|
total=1, completed=0, new='', rate='0.00', inflight=0,
|
||||||
|
cur='0s', elapsed='0s', eta='-', retries=0)
|
||||||
|
|
||||||
def set_scoring(self, done: int, total: int):
|
def set_scoring(self, done: int, total: int):
|
||||||
"""Retarget the SAME bar to the scoring phase: generation is finished
|
"""Retarget the SAME bar to the scoring phase: generation is finished
|
||||||
and its filled 100% state is stale -- now the bar refills with judged
|
and its filled 100% state is stale -- now the bar refills with judged
|
||||||
|
|||||||
@ -166,7 +166,7 @@ def _pull_one(image: str, verbose: bool = True) -> None:
|
|||||||
# override with EVALHARNESS_PULL_IDLE_S for slow-negotiating mirrors.
|
# override with EVALHARNESS_PULL_IDLE_S for slow-negotiating mirrors.
|
||||||
import os as _os2
|
import os as _os2
|
||||||
|
|
||||||
_IDLE_S = float(_os2.environ.get('EVALHARNESS_PULL_IDLE_S', '10'))
|
_IDLE_S = float(_os2.environ.get('EVALHARNESS_PULL_IDLE_S', '300'))
|
||||||
|
|
||||||
def _iter_lines():
|
def _iter_lines():
|
||||||
nonlocal buf
|
nonlocal buf
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user