Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Progress bar keeps its own elapsed/ETA clock (rich's columns freeze on cross-benchmark task reuse); in-flight shows current sample's elapsed; adaptive read timeout scales with the generation budget (32k-token CoTs were timing out at the fixed 300s and retrying forever -- the 'hang')

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 11:36:19 +00:00
parent e7792a559c
commit c6046dd02e
2 changed files with 26 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
evalharness/model/adapter.py
View File

@ -391,9 +391,14 @@ class OpenAICompatible(ModelAdapter):
try:
import httpx
# read timeout scales with the generation budget: a 32k-token
# CoT legitimately takes 10+ minutes; a fixed 300s timeout would
# kill and retry it forever (looks like a hang)
_rt = max(self.extra.get('timeout', 300),
int(payload.get('max_tokens') or 0) * 0.15)
async with httpx.AsyncClient(timeout=httpx.Timeout(
connect=self.extra.get('connect_timeout', 15),
read=self.extra.get('timeout', 300), write=30, pool=15)) as client:
read=_rt, write=30, pool=15)) as client:
r = await client.post(url, json=payload, headers=headers)
r.raise_for_status()
return r.json()

34
evalharness/progress/rich_terminal.py
View File

@ -10,8 +10,6 @@ from rich.progress import (
SpinnerColumn,
TaskProgressColumn,
TextColumn,
TimeElapsedColumn,
TimeRemainingColumn,
)
@ -28,12 +26,10 @@ class RichTerminalProgress:
BarColumn(complete_style="green", finished_style="bold green"),
TaskProgressColumn(),
TextColumn("• Completed {task.completed}/{task.total}"),
TextColumn("• in-flight {task.fields[inflight]}"),
TextColumn("• in-flight {task.fields[inflight]} ({task.fields[cur]})"),
TextColumn("• {task.fields[rate]}/s"),
TextColumn("•"),
TimeElapsedColumn(),
TextColumn("•"),
TimeRemainingColumn(),
TextColumn("• {task.fields[elapsed]}"),
TextColumn("• eta {task.fields[eta]}"),
console=self.console,
refresh_per_second=4,
)
@ -100,7 +96,8 @@ class RichTerminalProgress:
self.progress.update(self.task_id, description=desc,
total=total, completed=min(completed, total),
success=completed, failed=0, rate='0.00',
inflight=0, last_result='restored')
inflight=0, cur='0s', elapsed='0s', eta='-',
last_result='restored')
def set_bench_tag(self, tag: str):
"""Persistent counter shown on the sample bar, e.g. '[1/6]'."""
@ -122,7 +119,7 @@ class RichTerminalProgress:
return
self.inflight += 1
self.current_started = time.monotonic()
self.progress.update(self.task_id, inflight=self.inflight,
self.progress.update(self.task_id, inflight=self.inflight, cur='0s',
waiting="00:00", last_result=f"waiting {label}")
def rollback(self):
@ -130,7 +127,7 @@ class RichTerminalProgress:
just decrement the in-flight count, no success/fail bookkeeping."""
self.inflight = max(0, self.inflight - 1)
if self.task_id is not None:
self.progress.update(self.task_id, inflight=self.inflight)
self.progress.update(self.task_id, inflight=self.inflight, cur='0s')
def advance(self, success: bool = True):
if self.task_id is None:
@ -147,16 +144,25 @@ class RichTerminalProgress:
success=ok,
failed=failed,
rate=f"{completed / elapsed:.2f}",
inflight=self.inflight,
inflight=self.inflight, cur='0s',
elapsed=_fmt(elapsed),
eta=_fmt((task.total - completed) * elapsed / completed)
if completed and task.total and task.total > completed else '-',
waiting="00:00",
last_result="success" if success else "failed",
)
async def _heartbeat(self):
while self.task_id is not None:
if self.task_id is not None and self.inflight:
waiting = int(time.monotonic() - self.current_started)
self.progress.update(self.task_id, waiting=f"{waiting // 60:02d}:{waiting % 60:02d}")
if self.task_id is not None:
e = time.monotonic() - self.started
upd = {'elapsed': f'{int(e) // 60}m{int(e) % 60:02d}s' if e >= 60
else f'{int(e)}s'}
if self.inflight:
secs = int(time.monotonic() - self.current_started)
upd['cur'] = (f'{secs // 60}m{secs % 60:02d}s'
if secs >= 60 else f'{secs}s')
self.progress.update(self.task_id, **upd)
await asyncio.sleep(1)
def log(self, message: str):