Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix the ellipsis crash: text-tool-call parsing ran on plain code replies

Browse Source 
Full traceback finally caught it: adapter._parse ALWAYS ran the
text-protocol tool-call fallback, even for requests with NO tools. On
humaneval, model code like  regex-matched as a
'call', ast.literal_eval turned the literal  into an Ellipsis
(no exception -- it's a legal literal), and json.dumps(args) died
mid-generation, killing the benchmark.

Two layers:
- the fallback now only runs when the request actually carried tools
  (also stops polluting plain predictions with phantom calls, and the
  SyntaxWarning spam from ast.parse-ing model code disappears)
- json.dumps(args, default=str) as belt-and-braces for the
  text-tools path where an Ellipsis arg now stringifies

Reproduced the exact crash input as a unit case: no-tools code reply
yields 0 tool_calls;  in text mode serializes
{'key': 'Ellipsis'} without raising; normal fc calls unchanged.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-14 09:43:24 +00:00
parent f6ee6c7a8b
commit a07431b324
1 changed files with 15 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
evalharness/model/adapter.py
View File

@ -164,8 +164,12 @@ def _parse_text_tool_calls(text: str) -> list:
args[kw_.arg] = _ast.unparse(kw_.value)
except SyntaxError:
return None
# literal_eval happily returns Ellipsis (code like `f(key=...)`) and
# other non-JSON constants; default=str keeps the serializer alive
# instead of killing the whole benchmark at parse time
return {'id': '', 'type': 'function',
'function': {'name': name, 'arguments': json.dumps(args)}}
'function': {'name': name,
'arguments': json.dumps(args, default=str)}}
for m_ in _re.finditer(r'([A-Za-z_][A-Za-z0-9_]*)\((.*?)\)', text):
c = _py_call(m_)
@ -222,10 +226,10 @@ class OpenAICompatible(ModelAdapter):
break
data = await self._post_stream_aggregate(
f'{self.api_base}/chat/completions', payload, headers)
out = self._parse(data)
out = self._parse(data, allow_text_calls=bool(tools))
else:
data = await self._post(f'{self.api_base}/chat/completions', payload, headers)
out = self._parse(data)
out = self._parse(data, allow_text_calls=bool(tools))
out.usage.latency_s = round(_time.time() - t0, 3)
out.usage.retries = attempt
return out
@ -333,7 +337,7 @@ class OpenAICompatible(ModelAdapter):
'usage': (usage_ev or {}).get('usage') or {},
'model': self.model,
}
out = self._parse(data)
out = self._parse(data, allow_text_calls=True)
out.usage.ttft_s = round(ttft, 3) if ttft is not None else None
out.usage.itl_mean_s = round(sum(itl_vals) / len(itl_vals), 4) if itl_vals else None
out.usage.http_status = status
@ -379,12 +383,17 @@ class OpenAICompatible(ModelAdapter):
payload['chat_template_kwargs'] = {'enable_thinking': False}
return payload
def _parse(self, data: Dict[str, Any]) -> ModelOutput:
def _parse(self, data: Dict[str, Any],
allow_text_calls: bool = True) -> ModelOutput:
choice = (data.get('choices') or [{}])[0]
msg = choice.get('message') or {}
calls = []
raw_calls = list(msg.get('tool_calls') or [])
if not raw_calls:
if not raw_calls and allow_text_calls:
# text-protocol fallback ONLY for requests that carried tools:
# running it on plain prose/code (humaneval!) regex-matched
# `f(key=...)` style code as "calls", literal_eval'd the `...`
# into an Ellipsis and crashed json.dumps mid-generation
raw_calls = _parse_text_tool_calls(msg.get('content') or '')
for c in raw_calls:
fn = c.get('function') or {}