diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index a4153a4..603921a 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -115,6 +115,17 @@ def _key_for(api_base: str) -> str: +def _payload_chars(payload: Dict[str, Any]) -> int: + """Rough prompt size in characters (~3-4 chars/token). Used to decide + the auto-stream path: gateways that buffer whole requests make a huge + INPUT as hang-prone as a huge output budget.""" + n = 0 + for m in payload.get('messages') or []: + c = m.get('content') + n += len(c) if isinstance(c, str) else 256 + return n + + def _parse_text_tool_calls(text: str) -> list: """Extract tool calls from a text reply. Handles both shapes: - JSON array: [{"name":..,"arguments":{..}}] @@ -209,9 +220,15 @@ class OpenAICompatible(ModelAdapter): if stream: out = await self._post_stream_perf( f'{self.api_base}/chat/completions', payload, headers, t0) - elif int(payload.get('max_tokens') or 0) > 100000 \ + elif (int(payload.get('max_tokens') or 0) > 100000 + or _payload_chars(payload) > 300_000) \ and not os.environ.get('EVALHARNESS_NO_AUTOSTREAM'): - # long generation: stream and aggregate (gateway-safe). + # long generation OR LONG INPUT: stream and aggregate + # (gateway-safe). Some gateways buffer the whole request + # before answering on the non-stream path -- a 128k-token + # longbench_v2 prompt sat there past every read timeout; + # streaming starts emitting immediately, so a stuck + # endpoint surfaces in ~60s instead of after 20 minutes. # Some gateways drop chat_template_kwargs on the STREAM # path only (non-stream honors it) -- append the /no_think # soft switch into the prompt itself as a belt-and-braces