diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index d5d7509..4dea6a4 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -391,9 +391,14 @@ class OpenAICompatible(ModelAdapter): try: import httpx + # read timeout scales with the generation budget: a 32k-token + # CoT legitimately takes 10+ minutes; a fixed 300s timeout would + # kill and retry it forever (looks like a hang) + _rt = max(self.extra.get('timeout', 300), + int(payload.get('max_tokens') or 0) * 0.15) async with httpx.AsyncClient(timeout=httpx.Timeout( connect=self.extra.get('connect_timeout', 15), - read=self.extra.get('timeout', 300), write=30, pool=15)) as client: + read=_rt, write=30, pool=15)) as client: r = await client.post(url, json=payload, headers=headers) r.raise_for_status() return r.json() diff --git a/evalharness/progress/rich_terminal.py b/evalharness/progress/rich_terminal.py index a7778f0..3da217b 100644 --- a/evalharness/progress/rich_terminal.py +++ b/evalharness/progress/rich_terminal.py @@ -10,8 +10,6 @@ from rich.progress import ( SpinnerColumn, TaskProgressColumn, TextColumn, - TimeElapsedColumn, - TimeRemainingColumn, ) @@ -28,12 +26,10 @@ class RichTerminalProgress: BarColumn(complete_style="green", finished_style="bold green"), TaskProgressColumn(), TextColumn("• Completed {task.completed}/{task.total}"), - TextColumn("• in-flight {task.fields[inflight]}"), + TextColumn("• in-flight {task.fields[inflight]} ({task.fields[cur]})"), TextColumn("• {task.fields[rate]}/s"), - TextColumn("•"), - TimeElapsedColumn(), - TextColumn("•"), - TimeRemainingColumn(), + TextColumn("• {task.fields[elapsed]}"), + TextColumn("• eta {task.fields[eta]}"), console=self.console, refresh_per_second=4, ) @@ -100,7 +96,8 @@ class RichTerminalProgress: self.progress.update(self.task_id, description=desc, total=total, completed=min(completed, total), success=completed, failed=0, rate='0.00', - inflight=0, last_result='restored') + inflight=0, cur='0s', elapsed='0s', eta='-', + last_result='restored') def set_bench_tag(self, tag: str): """Persistent counter shown on the sample bar, e.g. '[1/6]'.""" @@ -122,7 +119,7 @@ class RichTerminalProgress: return self.inflight += 1 self.current_started = time.monotonic() - self.progress.update(self.task_id, inflight=self.inflight, + self.progress.update(self.task_id, inflight=self.inflight, cur='0s', waiting="00:00", last_result=f"waiting {label}") def rollback(self): @@ -130,7 +127,7 @@ class RichTerminalProgress: just decrement the in-flight count, no success/fail bookkeeping.""" self.inflight = max(0, self.inflight - 1) if self.task_id is not None: - self.progress.update(self.task_id, inflight=self.inflight) + self.progress.update(self.task_id, inflight=self.inflight, cur='0s') def advance(self, success: bool = True): if self.task_id is None: @@ -147,16 +144,25 @@ class RichTerminalProgress: success=ok, failed=failed, rate=f"{completed / elapsed:.2f}", - inflight=self.inflight, + inflight=self.inflight, cur='0s', + elapsed=_fmt(elapsed), + eta=_fmt((task.total - completed) * elapsed / completed) + if completed and task.total and task.total > completed else '-', waiting="00:00", last_result="success" if success else "failed", ) async def _heartbeat(self): while self.task_id is not None: - if self.task_id is not None and self.inflight: - waiting = int(time.monotonic() - self.current_started) - self.progress.update(self.task_id, waiting=f"{waiting // 60:02d}:{waiting % 60:02d}") + if self.task_id is not None: + e = time.monotonic() - self.started + upd = {'elapsed': f'{int(e) // 60}m{int(e) % 60:02d}s' if e >= 60 + else f'{int(e)}s'} + if self.inflight: + secs = int(time.monotonic() - self.current_started) + upd['cur'] = (f'{secs // 60}m{secs % 60:02d}s' + if secs >= 60 else f'{secs}s') + self.progress.update(self.task_id, **upd) await asyncio.sleep(1) def log(self, message: str):