evalstone/evalscope/tests/models/test_anthropic_tool_id_sanitize.py

"""Tests for Anthropic tool call id sanitization (issue #1523).

Dataset-provided histories (e.g. general_fc) carry tool call ids like
``functions.search:0`` which violate Anthropic's ``^[a-zA-Z0-9_-]+$``
constraint and repeat across turns. The conversion layer must rewrite them
while preserving tool_use/tool_result pairing.
"""
import re

import pytest

from evalscope.api.messages import ChatMessageAssistant, ChatMessageTool, ChatMessageUser
from evalscope.api.tool import ToolCall, ToolFunction

VALID_ID = re.compile(r'^[a-zA-Z0-9_-]+$')


def _anthropic_utils():
    pytest.importorskip('anthropic')
    from evalscope.models.utils import anthropic

    return anthropic


def _tool_call(call_id: str, name: str = 'search') -> ToolCall:
    return ToolCall(id=call_id, function=ToolFunction(name=name, arguments={'query': 'q'}))


def _collect_blocks(messages, block_type: str):
    blocks = []
    for message in messages:
        content = message['content']
        if isinstance(content, str):
            continue
        blocks.extend(block for block in content if block.get('type') == block_type)
    return blocks


def test_illegal_and_duplicate_tool_ids_are_sanitized_and_paired():
    anthropic = _anthropic_utils()

    # Two turns reusing the same illegal id, mirroring general_fc data
    _, messages = anthropic.anthropic_chat_messages([
        ChatMessageUser(content='first question'),
        ChatMessageAssistant(content='', tool_calls=[_tool_call('functions.search:0')]),
        ChatMessageTool(content='result 1', tool_call_id='functions.search:0'),
        ChatMessageAssistant(content='', tool_calls=[_tool_call('functions.search:0')]),
        ChatMessageTool(content='result 2', tool_call_id='functions.search:0'),
        ChatMessageUser(content='second question'),
    ])

    tool_use_blocks = _collect_blocks(messages, 'tool_use')
    tool_result_blocks = _collect_blocks(messages, 'tool_result')
    assert len(tool_use_blocks) == 2
    assert len(tool_result_blocks) == 2

    tool_use_ids = [block['id'] for block in tool_use_blocks]
    tool_result_ids = [block['tool_use_id'] for block in tool_result_blocks]

    # All ids must satisfy Anthropic's constraint
    for tool_id in tool_use_ids + tool_result_ids:
        assert VALID_ID.match(tool_id), f'illegal id: {tool_id}'
    # Ids must be unique across turns
    assert len(set(tool_use_ids)) == 2
    # Each tool_result must pair with its own turn's tool_use
    assert tool_result_ids == tool_use_ids


def test_multiple_tool_calls_in_one_turn_keep_distinct_pairing():
    anthropic = _anthropic_utils()

    _, messages = anthropic.anthropic_chat_messages([
        ChatMessageUser(content='question'),
        ChatMessageAssistant(
            content='',
            tool_calls=[_tool_call('functions.search:0'), _tool_call('functions.search:1')],
        ),
        ChatMessageTool(content='result 0', tool_call_id='functions.search:0'),
        ChatMessageTool(content='result 1', tool_call_id='functions.search:1'),
        ChatMessageUser(content='follow up'),
    ])

    tool_use_ids = [block['id'] for block in _collect_blocks(messages, 'tool_use')]
    tool_result_ids = [block['tool_use_id'] for block in _collect_blocks(messages, 'tool_result')]

    assert len(set(tool_use_ids)) == 2
    assert tool_result_ids == tool_use_ids


def test_duplicate_ids_within_one_turn_pair_positionally():
    anthropic = _anthropic_utils()

    # Malformed history: one assistant turn reuses the SAME id for two calls
    _, messages = anthropic.anthropic_chat_messages([
        ChatMessageUser(content='question'),
        ChatMessageAssistant(
            content='',
            tool_calls=[_tool_call('functions.search:0'), _tool_call('functions.search:0')],
        ),
        ChatMessageTool(content='result A', tool_call_id='functions.search:0'),
        ChatMessageTool(content='result B', tool_call_id='functions.search:0'),
        ChatMessageUser(content='follow up'),
    ])

    tool_use_ids = [block['id'] for block in _collect_blocks(messages, 'tool_use')]
    tool_result_ids = [block['tool_use_id'] for block in _collect_blocks(messages, 'tool_result')]

    # Each tool_use gets a distinct id and is referenced exactly once, in order
    assert len(set(tool_use_ids)) == 2
    assert tool_result_ids == tool_use_ids


def test_legal_tool_ids_are_kept_unchanged():
    anthropic = _anthropic_utils()

    _, messages = anthropic.anthropic_chat_messages([
        ChatMessageUser(content='question'),
        ChatMessageAssistant(content='', tool_calls=[_tool_call('toolu_abc123')]),
        ChatMessageTool(content='result', tool_call_id='toolu_abc123'),
        ChatMessageUser(content='follow up'),
    ])

    tool_use_ids = [block['id'] for block in _collect_blocks(messages, 'tool_use')]
    tool_result_ids = [block['tool_use_id'] for block in _collect_blocks(messages, 'tool_result')]

    assert tool_use_ids == ['toolu_abc123']
    assert tool_result_ids == ['toolu_abc123']


def test_sanitization_does_not_mutate_original_messages():
    anthropic = _anthropic_utils()

    assistant = ChatMessageAssistant(content='', tool_calls=[_tool_call('functions.search:0')])
    tool = ChatMessageTool(content='result', tool_call_id='functions.search:0')

    anthropic.anthropic_chat_messages([ChatMessageUser(content='q'), assistant, tool])

    assert assistant.tool_calls[0].id == 'functions.search:0'
    assert tool.tool_call_id == 'functions.search:0'