longbench_v2 chain of fixes: reasoning_content output fallback (Qwen3 thinking channel), nothink soft switch /no_think (template-400 immune), truncation input+output budget, adaptive context shrink on server 400 (cross-tokenizer margin, body-aware detection), pool fails over on 4xx

evalharness/model/adapter.py

@@ -185,6 +185,27 @@ class OpenAICompatible(ModelAdapter):
                 return out
             except Exception as e:  # 5xx/429/timeouts: worth retrying
                 last_exc = e
+                # context-overflow 400: OUR tokenizer counted fewer tokens than
+                # the server's -> shrink the prompt by 15% and retry (cross-
+                # tokenizer safety margin, converges in 1-2 attempts)
+                msg = str(e)
+                resp_body = ''
+                try:
+                    resp_body = e.response.text or ''
+                except AttributeError:
+                    pass
+                if ('maximum context length' in msg or 'maximum context length' in resp_body) \
+                        and ('reduce the length' in resp_body or 'reduce the length' in msg):
+                    msgs = payload.get('messages') or []
+                    for m in reversed(msgs):
+                        if m.get('role') == 'user':
+                            c = m.get('content') or ''
+                            if len(c) > 4000:
+                                keep = int(len(c) * 0.85) // 2
+                                m['content'] = (f'{c[:keep]}\n\n...[context trimmed]...\n\n'
+                                                f'{c[-keep:]}')
+                                break
+                    continue
                 retryable = 'Server error' in str(e) or '504' in str(e) or '502' in str(e) \
                     or '429' in str(e) or 'timeout' in str(e).lower() \
                     or 'TimeoutException' in type(e).__name__
@@ -231,7 +252,8 @@ class OpenAICompatible(ModelAdapter):
                     except ValueError:
                         continue
                     now = _time.time()
-                    piece = ((ev.get('choices') or [{}])[0].get('delta') or {}).get('content')
+                    delta = (ev.get('choices') or [{}])[0].get('delta') or {}
+                    piece = delta.get('content') or delta.get('reasoning_content')
                     if piece:
                         if ttft is None:
                             ttft = now - t0
@@ -240,8 +262,11 @@ class OpenAICompatible(ModelAdapter):
                         last_tok_t = now
                     chunks.append(ev)
 
-        text = ''.join((((ev.get('choices') or [{}])[0].get('delta') or {}).get('content') or '')
+        def _piece(ev):
-                       for ev in chunks)
+            d = (ev.get('choices') or [{}])[0].get('delta') or {}
+            return d.get('content') or d.get('reasoning_content') or ''
+
+        text = ''.join(_piece(ev) for ev in chunks)
         finish = ''
         for ev in chunks:
             fr = (ev.get('choices') or [{}])[0].get('finish_reason')
@@ -285,8 +310,16 @@ class OpenAICompatible(ModelAdapter):
             if kw.get(k) is not None:
                 payload[k] = kw[k]
         payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096))  # CoT room
-        if self.extra.get('no_think'):  # Qwen3-style: disable thinking at the adapter level
+        if self.extra.get('no_think'):
-            payload.setdefault('chat_template_kwargs', {'enable_thinking': False})
+            # Qwen3 soft switch: /no_think appended to the last user message.
+            # (chat_template_kwargs + long/odd payloads hit template 400s on
+            # some vLLM builds; the soft switch is payload-independent)
+            msgs = payload.get('messages') or []
+            for m in reversed(msgs):
+                if m.get('role') == 'user':
+                    if '/no_think' not in (m.get('content') or ''):
+                        m['content'] = (m.get('content') or '') + ' /no_think'
+                    break
         return payload
 
     def _parse(self, data: Dict[str, Any]) -> ModelOutput:
@@ -314,7 +347,13 @@ class OpenAICompatible(ModelAdapter):
                       output_tokens=u.get('completion_tokens', 0),
                       total_tokens=u.get('total_tokens', 0),
                       finish_reason=choice.get('finish_reason', ''))
-        return ModelOutput(text=msg.get('content') or '', tool_calls=calls, usage=usage,
+        text = msg.get('content') or ''
+        if not text.strip():
+            # Qwen3/DeepSeek thinking models may put EVERYTHING in reasoning_content
+            rc = msg.get('reasoning_content') or msg.get('reasoning')
+            if isinstance(rc, str) and rc.strip():
+                text = rc
+        return ModelOutput(text=text, tool_calls=calls, usage=usage,
                            raw=data, model=data.get('model', self.model))
 
     async def _post(self, url: str, payload: Dict, headers: Dict) -> Dict[str, Any]:

evalharness/model/pool.py

@@ -58,8 +58,11 @@ class PooledAdapter(ModelAdapter):
                 if out.usage.retries:
                     self.stats['retried'] += 1
                 return out
-            except Exception as e:  # dead instance -> next
+            except Exception as e:  # dead/overloaded instance -> next
                 last_exc = e
+                # 4xx (e.g. 400 overloaded) still worth trying ANOTHER instance:
+                # one backend's state can differ from the rest
+                continue
         self.stats['failed'] += 1
         raise last_exc
 

evalharness/model/runner.py

@@ -53,6 +53,10 @@ async def generate_predictions(
     dev/train-split samples (few_shot_samples) are prepended.
     """
     gen_kwargs = gen_kwargs or {}
+
+    def _default_max_tokens() -> int:
+        return 4096
+
     sem = asyncio.Semaphore(concurrency)
     total_usage = Usage()
     done_count = 0
@@ -116,13 +120,23 @@ async def generate_predictions(
         parts.append(question)
         text = '\n\n'.join(parts)
         if max_input_tokens:
+            # reserve room for the OUTPUT budget + safety margin, else the
+            # server rejects input+max_tokens > context_limit by 1 token
+            budget = max(1024, max_input_tokens
+                         - int(gen_kwargs.get('max_tokens') or 4096) - 2048)
             try:
                 from .truncation import truncate_middle_tokens, default_tokenizer_path
 
-                text = truncate_middle_tokens(text, max_input_tokens,
+                text = truncate_middle_tokens(text, budget,
                                               tokenizer_path or default_tokenizer_path())
-            except Exception:
+            except Exception as e:
-                pass  # no tokenizer: fall through to chars truncation
+                # no tokenizer/transformers: degrade to a CHARS budget that
+                # approximates the token cap (never send the raw 2M-token input)
+                approx_chars = budget * 3
+                if len(text) > approx_chars:
+                    keep = approx_chars // 2
+                    text = f'{text[:keep]}\n\n...[truncated {len(text) - 2 * keep} chars]...\n\n{text[-keep:]}'
+                print(f'truncation degraded to chars ({type(e).__name__})', flush=True)
         if max_input_chars and len(text) > max_input_chars:
             keep = max_input_chars // 2
             head = text[:keep]