Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

longbench_v2 chain of fixes: reasoning_content output fallback (Qwen3 thinking channel), nothink soft switch /no_think (template-400 immune), truncation input+output budget, adaptive context shrink on server 400 (cross-tokenizer margin, body-aware detection), pool fails over on 4xx

Browse Source
This commit is contained in:
sora 2026-08-27 06:27:59 +00:00
parent c4473a1e1c
commit e32da10ce9
3 changed files with 66 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
evalharness/model/adapter.py
View File

@ -185,6 +185,27 @@ class OpenAICompatible(ModelAdapter):
return out
except Exception as e: # 5xx/429/timeouts: worth retrying
last_exc = e
# context-overflow 400: OUR tokenizer counted fewer tokens than
# the server's -> shrink the prompt by 15% and retry (cross-
# tokenizer safety margin, converges in 1-2 attempts)
msg = str(e)
resp_body = ''
try:
resp_body = e.response.text or ''
except AttributeError:
pass
if ('maximum context length' in msg or 'maximum context length' in resp_body) \
and ('reduce the length' in resp_body or 'reduce the length' in msg):
msgs = payload.get('messages') or []
for m in reversed(msgs):
if m.get('role') == 'user':
c = m.get('content') or ''
if len(c) > 4000:
keep = int(len(c) * 0.85) // 2
m['content'] = (f'{c[:keep]}\n\n...[context trimmed]...\n\n'
f'{c[-keep:]}')
break
continue
retryable = 'Server error' in str(e) or '504' in str(e) or '502' in str(e) \
or '429' in str(e) or 'timeout' in str(e).lower() \
or 'TimeoutException' in type(e).__name__
@ -231,7 +252,8 @@ class OpenAICompatible(ModelAdapter):
except ValueError:
continue
now = _time.time()
piece = ((ev.get('choices') or [{}])[0].get('delta') or {}).get('content')
delta = (ev.get('choices') or [{}])[0].get('delta') or {}
piece = delta.get('content') or delta.get('reasoning_content')
if piece:
if ttft is None:
ttft = now - t0
@ -240,8 +262,11 @@ class OpenAICompatible(ModelAdapter):
last_tok_t = now
chunks.append(ev)
text = ''.join((((ev.get('choices') or [{}])[0].get('delta') or {}).get('content') or '')
for ev in chunks)
def _piece(ev):
d = (ev.get('choices') or [{}])[0].get('delta') or {}
return d.get('content') or d.get('reasoning_content') or ''
text = ''.join(_piece(ev) for ev in chunks)
finish = ''
for ev in chunks:
fr = (ev.get('choices') or [{}])[0].get('finish_reason')
@ -285,8 +310,16 @@ class OpenAICompatible(ModelAdapter):
if kw.get(k) is not None:
payload[k] = kw[k]
payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096)) # CoT room
if self.extra.get('no_think'): # Qwen3-style: disable thinking at the adapter level
payload.setdefault('chat_template_kwargs', {'enable_thinking': False})
if self.extra.get('no_think'):
# Qwen3 soft switch: /no_think appended to the last user message.
# (chat_template_kwargs + long/odd payloads hit template 400s on
# some vLLM builds; the soft switch is payload-independent)
msgs = payload.get('messages') or []
for m in reversed(msgs):
if m.get('role') == 'user':
if '/no_think' not in (m.get('content') or ''):
m['content'] = (m.get('content') or '') + ' /no_think'
break
return payload
def _parse(self, data: Dict[str, Any]) -> ModelOutput:
@ -314,7 +347,13 @@ class OpenAICompatible(ModelAdapter):
output_tokens=u.get('completion_tokens', 0),
total_tokens=u.get('total_tokens', 0),
finish_reason=choice.get('finish_reason', ''))
return ModelOutput(text=msg.get('content') or '', tool_calls=calls, usage=usage,
text = msg.get('content') or ''
if not text.strip():
# Qwen3/DeepSeek thinking models may put EVERYTHING in reasoning_content
rc = msg.get('reasoning_content') or msg.get('reasoning')
if isinstance(rc, str) and rc.strip():
text = rc
return ModelOutput(text=text, tool_calls=calls, usage=usage,
raw=data, model=data.get('model', self.model))
async def _post(self, url: str, payload: Dict, headers: Dict) -> Dict[str, Any]:

5
evalharness/model/pool.py
View File

@ -58,8 +58,11 @@ class PooledAdapter(ModelAdapter):
if out.usage.retries:
self.stats['retried'] += 1
return out
except Exception as e: # dead instance -> next
except Exception as e: # dead/overloaded instance -> next
last_exc = e
# 4xx (e.g. 400 overloaded) still worth trying ANOTHER instance:
# one backend's state can differ from the rest
continue
self.stats['failed'] += 1
raise last_exc

20
evalharness/model/runner.py
View File

@ -53,6 +53,10 @@ async def generate_predictions(
dev/train-split samples (few_shot_samples) are prepended.
"""
gen_kwargs = gen_kwargs or {}
def _default_max_tokens() -> int:
return 4096
sem = asyncio.Semaphore(concurrency)
total_usage = Usage()
done_count = 0
@ -116,13 +120,23 @@ async def generate_predictions(
parts.append(question)
text = '\n\n'.join(parts)
if max_input_tokens:
# reserve room for the OUTPUT budget + safety margin, else the
# server rejects input+max_tokens > context_limit by 1 token
budget = max(1024, max_input_tokens
- int(gen_kwargs.get('max_tokens') or 4096) - 2048)
try:
from .truncation import truncate_middle_tokens, default_tokenizer_path
text = truncate_middle_tokens(text, max_input_tokens,
text = truncate_middle_tokens(text, budget,
tokenizer_path or default_tokenizer_path())
except Exception:
pass # no tokenizer: fall through to chars truncation
except Exception as e:
# no tokenizer/transformers: degrade to a CHARS budget that
# approximates the token cap (never send the raw 2M-token input)
approx_chars = budget * 3
if len(text) > approx_chars:
keep = approx_chars // 2
text = f'{text[:keep]}\n\n...[truncated {len(text) - 2 * keep} chars]...\n\n{text[-keep:]}'
print(f'truncation degraded to chars ({type(e).__name__})', flush=True)
if max_input_chars and len(text) > max_input_chars:
keep = max_input_chars // 2
head = text[:keep]