diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index b83066a..1b4fbbb 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -185,6 +185,27 @@ class OpenAICompatible(ModelAdapter): return out except Exception as e: # 5xx/429/timeouts: worth retrying last_exc = e + # context-overflow 400: OUR tokenizer counted fewer tokens than + # the server's -> shrink the prompt by 15% and retry (cross- + # tokenizer safety margin, converges in 1-2 attempts) + msg = str(e) + resp_body = '' + try: + resp_body = e.response.text or '' + except AttributeError: + pass + if ('maximum context length' in msg or 'maximum context length' in resp_body) \ + and ('reduce the length' in resp_body or 'reduce the length' in msg): + msgs = payload.get('messages') or [] + for m in reversed(msgs): + if m.get('role') == 'user': + c = m.get('content') or '' + if len(c) > 4000: + keep = int(len(c) * 0.85) // 2 + m['content'] = (f'{c[:keep]}\n\n...[context trimmed]...\n\n' + f'{c[-keep:]}') + break + continue retryable = 'Server error' in str(e) or '504' in str(e) or '502' in str(e) \ or '429' in str(e) or 'timeout' in str(e).lower() \ or 'TimeoutException' in type(e).__name__ @@ -231,7 +252,8 @@ class OpenAICompatible(ModelAdapter): except ValueError: continue now = _time.time() - piece = ((ev.get('choices') or [{}])[0].get('delta') or {}).get('content') + delta = (ev.get('choices') or [{}])[0].get('delta') or {} + piece = delta.get('content') or delta.get('reasoning_content') if piece: if ttft is None: ttft = now - t0 @@ -240,8 +262,11 @@ class OpenAICompatible(ModelAdapter): last_tok_t = now chunks.append(ev) - text = ''.join((((ev.get('choices') or [{}])[0].get('delta') or {}).get('content') or '') - for ev in chunks) + def _piece(ev): + d = (ev.get('choices') or [{}])[0].get('delta') or {} + return d.get('content') or d.get('reasoning_content') or '' + + text = ''.join(_piece(ev) for ev in chunks) finish = '' for ev in chunks: fr = (ev.get('choices') or [{}])[0].get('finish_reason') @@ -285,8 +310,16 @@ class OpenAICompatible(ModelAdapter): if kw.get(k) is not None: payload[k] = kw[k] payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096)) # CoT room - if self.extra.get('no_think'): # Qwen3-style: disable thinking at the adapter level - payload.setdefault('chat_template_kwargs', {'enable_thinking': False}) + if self.extra.get('no_think'): + # Qwen3 soft switch: /no_think appended to the last user message. + # (chat_template_kwargs + long/odd payloads hit template 400s on + # some vLLM builds; the soft switch is payload-independent) + msgs = payload.get('messages') or [] + for m in reversed(msgs): + if m.get('role') == 'user': + if '/no_think' not in (m.get('content') or ''): + m['content'] = (m.get('content') or '') + ' /no_think' + break return payload def _parse(self, data: Dict[str, Any]) -> ModelOutput: @@ -314,7 +347,13 @@ class OpenAICompatible(ModelAdapter): output_tokens=u.get('completion_tokens', 0), total_tokens=u.get('total_tokens', 0), finish_reason=choice.get('finish_reason', '')) - return ModelOutput(text=msg.get('content') or '', tool_calls=calls, usage=usage, + text = msg.get('content') or '' + if not text.strip(): + # Qwen3/DeepSeek thinking models may put EVERYTHING in reasoning_content + rc = msg.get('reasoning_content') or msg.get('reasoning') + if isinstance(rc, str) and rc.strip(): + text = rc + return ModelOutput(text=text, tool_calls=calls, usage=usage, raw=data, model=data.get('model', self.model)) async def _post(self, url: str, payload: Dict, headers: Dict) -> Dict[str, Any]: diff --git a/evalharness/model/pool.py b/evalharness/model/pool.py index 1d295ee..6d00f8f 100644 --- a/evalharness/model/pool.py +++ b/evalharness/model/pool.py @@ -58,8 +58,11 @@ class PooledAdapter(ModelAdapter): if out.usage.retries: self.stats['retried'] += 1 return out - except Exception as e: # dead instance -> next + except Exception as e: # dead/overloaded instance -> next last_exc = e + # 4xx (e.g. 400 overloaded) still worth trying ANOTHER instance: + # one backend's state can differ from the rest + continue self.stats['failed'] += 1 raise last_exc diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 682cc0d..5bb6203 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -53,6 +53,10 @@ async def generate_predictions( dev/train-split samples (few_shot_samples) are prepended. """ gen_kwargs = gen_kwargs or {} + + def _default_max_tokens() -> int: + return 4096 + sem = asyncio.Semaphore(concurrency) total_usage = Usage() done_count = 0 @@ -116,13 +120,23 @@ async def generate_predictions( parts.append(question) text = '\n\n'.join(parts) if max_input_tokens: + # reserve room for the OUTPUT budget + safety margin, else the + # server rejects input+max_tokens > context_limit by 1 token + budget = max(1024, max_input_tokens + - int(gen_kwargs.get('max_tokens') or 4096) - 2048) try: from .truncation import truncate_middle_tokens, default_tokenizer_path - text = truncate_middle_tokens(text, max_input_tokens, + text = truncate_middle_tokens(text, budget, tokenizer_path or default_tokenizer_path()) - except Exception: - pass # no tokenizer: fall through to chars truncation + except Exception as e: + # no tokenizer/transformers: degrade to a CHARS budget that + # approximates the token cap (never send the raw 2M-token input) + approx_chars = budget * 3 + if len(text) > approx_chars: + keep = approx_chars // 2 + text = f'{text[:keep]}\n\n...[truncated {len(text) - 2 * keep} chars]...\n\n{text[-keep:]}' + print(f'truncation degraded to chars ({type(e).__name__})', flush=True) if max_input_chars and len(text) > max_input_chars: keep = max_input_chars // 2 head = text[:keep]