Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix frozen progress clock: heartbeat recreated on every reset_samples (previous one died during pause/resume cycles); auto-stream path appends /no_think when gateway drops chat_template_kwargs on streaming

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 06:39:55 +00:00
parent 8bac3483b6
commit d00f330430
2 changed files with 30 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
evalharness/model/adapter.py
View File

@ -203,8 +203,19 @@ class OpenAICompatible(ModelAdapter):
out = await self._post_stream_perf(
f'{self.api_base}/chat/completions', payload, headers, t0)
elif int(payload.get('max_tokens') or 0) > 8192:
# long generation: stream and aggregate (gateway-safe)
# long generation: stream and aggregate (gateway-safe).
# Some gateways drop chat_template_kwargs on the STREAM
# path only (non-stream honors it) -- append the /no_think
# soft switch into the prompt itself as a belt-and-braces
# fallback (it IS the prompt, cannot be stripped)
payload['stream'] = True
if payload.get('chat_template_kwargs', {}).get('enable_thinking') is False:
msgs = payload.get('messages') or []
for m in reversed(msgs):
if m.get('role') == 'user':
if '/no_think' not in (m.get('content') or ''):
m['content'] = (m.get('content') or '') + ' /no_think'
break
data = await self._post_stream_aggregate(
f'{self.api_base}/chat/completions', payload, headers)
out = self._parse(data)

23
evalharness/progress/rich_terminal.py
View File

@ -115,13 +115,18 @@ class RichTerminalProgress:
success=completed, failed=0, rate='0.00', inflight=0,
cur='0s', elapsed='0s', eta='-',
waiting='00:00', last_result='restored')
self.heartbeat_task = asyncio.create_task(self._heartbeat())
else:
self.progress.update(self.task_id, description=desc,
total=total, completed=min(completed, total),
success=completed, failed=0, rate='0.00',
inflight=0, cur='0s', elapsed='0s', eta='-',
last_result='restored')
# ALWAYS recreate the heartbeat: the previous one may have died during
# pause/resume cycles between benchmarks (stale reference -> silent
# death -> frozen clock while the spinner still animates)
if self.heartbeat_task is not None:
self.heartbeat_task.cancel()
self.heartbeat_task = asyncio.create_task(self._heartbeat())
def set_bench_tag(self, tag: str):
if self.disabled:
@ -187,8 +192,11 @@ class RichTerminalProgress:
)
async def _heartbeat(self):
while self.task_id is not None:
if self.task_id is not None:
"""One tick per second: refresh elapsed + current-sample timer.
Recreated on every reset_samples; must never raise or the clock
freezes silently."""
try:
while self.task_id is not None:
e = time.monotonic() - self.started
upd = {'elapsed': f'{int(e) // 60}m{int(e) % 60:02d}s' if e >= 60
else f'{int(e)}s'}
@ -196,8 +204,13 @@ class RichTerminalProgress:
secs = int(time.monotonic() - self.current_started)
upd['cur'] = (f'{secs // 60}m{secs % 60:02d}s'
if secs >= 60 else f'{secs}s')
self.progress.update(self.task_id, **upd)
await asyncio.sleep(1)
try:
self.progress.update(self.task_id, **upd)
except Exception:
pass # task may have been removed mid-tick
await asyncio.sleep(1)
except asyncio.CancelledError:
pass
def log(self, message: str):
"""Print a status line ABOVE the live bar (safe during live display).