diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index 44cbc75..a4153a4 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -164,8 +164,12 @@ def _parse_text_tool_calls(text: str) -> list: args[kw_.arg] = _ast.unparse(kw_.value) except SyntaxError: return None + # literal_eval happily returns Ellipsis (code like `f(key=...)`) and + # other non-JSON constants; default=str keeps the serializer alive + # instead of killing the whole benchmark at parse time return {'id': '', 'type': 'function', - 'function': {'name': name, 'arguments': json.dumps(args)}} + 'function': {'name': name, + 'arguments': json.dumps(args, default=str)}} for m_ in _re.finditer(r'([A-Za-z_][A-Za-z0-9_]*)\((.*?)\)', text): c = _py_call(m_) @@ -222,10 +226,10 @@ class OpenAICompatible(ModelAdapter): break data = await self._post_stream_aggregate( f'{self.api_base}/chat/completions', payload, headers) - out = self._parse(data) + out = self._parse(data, allow_text_calls=bool(tools)) else: data = await self._post(f'{self.api_base}/chat/completions', payload, headers) - out = self._parse(data) + out = self._parse(data, allow_text_calls=bool(tools)) out.usage.latency_s = round(_time.time() - t0, 3) out.usage.retries = attempt return out @@ -333,7 +337,7 @@ class OpenAICompatible(ModelAdapter): 'usage': (usage_ev or {}).get('usage') or {}, 'model': self.model, } - out = self._parse(data) + out = self._parse(data, allow_text_calls=True) out.usage.ttft_s = round(ttft, 3) if ttft is not None else None out.usage.itl_mean_s = round(sum(itl_vals) / len(itl_vals), 4) if itl_vals else None out.usage.http_status = status @@ -379,12 +383,17 @@ class OpenAICompatible(ModelAdapter): payload['chat_template_kwargs'] = {'enable_thinking': False} return payload - def _parse(self, data: Dict[str, Any]) -> ModelOutput: + def _parse(self, data: Dict[str, Any], + allow_text_calls: bool = True) -> ModelOutput: choice = (data.get('choices') or [{}])[0] msg = choice.get('message') or {} calls = [] raw_calls = list(msg.get('tool_calls') or []) - if not raw_calls: + if not raw_calls and allow_text_calls: + # text-protocol fallback ONLY for requests that carried tools: + # running it on plain prose/code (humaneval!) regex-matched + # `f(key=...)` style code as "calls", literal_eval'd the `...` + # into an Ellipsis and crashed json.dumps mid-generation raw_calls = _parse_text_tool_calls(msg.get('content') or '') for c in raw_calls: fn = c.get('function') or {}