diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index 06a1417..0e345b1 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -203,8 +203,19 @@ class OpenAICompatible(ModelAdapter): out = await self._post_stream_perf( f'{self.api_base}/chat/completions', payload, headers, t0) elif int(payload.get('max_tokens') or 0) > 8192: - # long generation: stream and aggregate (gateway-safe) + # long generation: stream and aggregate (gateway-safe). + # Some gateways drop chat_template_kwargs on the STREAM + # path only (non-stream honors it) -- append the /no_think + # soft switch into the prompt itself as a belt-and-braces + # fallback (it IS the prompt, cannot be stripped) payload['stream'] = True + if payload.get('chat_template_kwargs', {}).get('enable_thinking') is False: + msgs = payload.get('messages') or [] + for m in reversed(msgs): + if m.get('role') == 'user': + if '/no_think' not in (m.get('content') or ''): + m['content'] = (m.get('content') or '') + ' /no_think' + break data = await self._post_stream_aggregate( f'{self.api_base}/chat/completions', payload, headers) out = self._parse(data) diff --git a/evalharness/progress/rich_terminal.py b/evalharness/progress/rich_terminal.py index 6250c58..77f6fca 100644 --- a/evalharness/progress/rich_terminal.py +++ b/evalharness/progress/rich_terminal.py @@ -115,13 +115,18 @@ class RichTerminalProgress: success=completed, failed=0, rate='0.00', inflight=0, cur='0s', elapsed='0s', eta='-', waiting='00:00', last_result='restored') - self.heartbeat_task = asyncio.create_task(self._heartbeat()) else: self.progress.update(self.task_id, description=desc, total=total, completed=min(completed, total), success=completed, failed=0, rate='0.00', inflight=0, cur='0s', elapsed='0s', eta='-', last_result='restored') + # ALWAYS recreate the heartbeat: the previous one may have died during + # pause/resume cycles between benchmarks (stale reference -> silent + # death -> frozen clock while the spinner still animates) + if self.heartbeat_task is not None: + self.heartbeat_task.cancel() + self.heartbeat_task = asyncio.create_task(self._heartbeat()) def set_bench_tag(self, tag: str): if self.disabled: @@ -187,8 +192,11 @@ class RichTerminalProgress: ) async def _heartbeat(self): - while self.task_id is not None: - if self.task_id is not None: + """One tick per second: refresh elapsed + current-sample timer. + Recreated on every reset_samples; must never raise or the clock + freezes silently.""" + try: + while self.task_id is not None: e = time.monotonic() - self.started upd = {'elapsed': f'{int(e) // 60}m{int(e) % 60:02d}s' if e >= 60 else f'{int(e)}s'} @@ -196,8 +204,13 @@ class RichTerminalProgress: secs = int(time.monotonic() - self.current_started) upd['cur'] = (f'{secs // 60}m{secs % 60:02d}s' if secs >= 60 else f'{secs}s') - self.progress.update(self.task_id, **upd) - await asyncio.sleep(1) + try: + self.progress.update(self.task_id, **upd) + except Exception: + pass # task may have been removed mid-tick + await asyncio.sleep(1) + except asyncio.CancelledError: + pass def log(self, message: str): """Print a status line ABOVE the live bar (safe during live display).