13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
355 lines
11 KiB
Python
355 lines
11 KiB
Python
import asyncio
|
|
from types import SimpleNamespace
|
|
|
|
import pytest
|
|
from pydantic import ValidationError
|
|
|
|
from evalscope.api.messages import (
|
|
ChatMessageAssistant,
|
|
ChatMessageSystem,
|
|
ChatMessageTool,
|
|
ChatMessageUser,
|
|
ContentText,
|
|
)
|
|
from evalscope.api.model import GenerateConfig
|
|
from evalscope.api.tool import ToolCall, ToolFunction, ToolInfo
|
|
|
|
|
|
def _anthropic_utils():
|
|
pytest.importorskip('anthropic')
|
|
from evalscope.models.utils import anthropic
|
|
|
|
return anthropic
|
|
|
|
|
|
def _cache_control_blocks(messages):
|
|
blocks = []
|
|
for message in messages:
|
|
content = message['content']
|
|
if isinstance(content, str):
|
|
continue
|
|
blocks.extend(block for block in content if isinstance(block, dict) and 'cache_control' in block)
|
|
return blocks
|
|
|
|
|
|
def test_anthropic_cache_control_accepts_typed_values():
|
|
config = GenerateConfig(
|
|
anthropic_cache_control={
|
|
'type': 'ephemeral',
|
|
'ttl': '1h',
|
|
},
|
|
anthropic_cache_strategy='recent_messages',
|
|
)
|
|
|
|
assert config.anthropic_cache_control is not None
|
|
assert config.anthropic_cache_control.model_dump(exclude_none=True) == {
|
|
'type': 'ephemeral',
|
|
'ttl': '1h',
|
|
}
|
|
assert config.anthropic_cache_strategy == 'recent_messages'
|
|
|
|
|
|
def test_anthropic_cache_control_rejects_unknown_keys():
|
|
with pytest.raises(ValidationError):
|
|
GenerateConfig(anthropic_cache_control={'type': 'ephemeral', 'unknown': True})
|
|
|
|
|
|
def test_legacy_anthropic_content_cache_control_is_rejected():
|
|
with pytest.raises(ValidationError, match='anthropic_content_cache_control'):
|
|
GenerateConfig(anthropic_content_cache_control={'type': 'ephemeral'})
|
|
|
|
|
|
def test_completion_params_do_not_enable_cache_by_default():
|
|
anthropic = _anthropic_utils()
|
|
|
|
params = anthropic.anthropic_completion_params('claude', GenerateConfig())
|
|
|
|
assert 'cache_control' not in params
|
|
|
|
|
|
def test_recent_messages_sets_top_level_cache_control():
|
|
anthropic = _anthropic_utils()
|
|
|
|
params = anthropic.anthropic_completion_params(
|
|
'claude',
|
|
GenerateConfig(
|
|
anthropic_cache_control={
|
|
'type': 'ephemeral',
|
|
'ttl': '5m'
|
|
},
|
|
anthropic_cache_strategy='recent_messages',
|
|
),
|
|
)
|
|
|
|
assert params['cache_control'] == {'type': 'ephemeral', 'ttl': '5m'}
|
|
|
|
|
|
def test_anthropic_api_splits_automatic_and_explicit_cache_control():
|
|
pytest.importorskip('anthropic')
|
|
from evalscope.models.anthropic_compatible import AnthropicCompatibleAPI
|
|
|
|
api = AnthropicCompatibleAPI.__new__(AnthropicCompatibleAPI)
|
|
recent_config = GenerateConfig(
|
|
anthropic_cache_control={'type': 'ephemeral'},
|
|
anthropic_cache_strategy='recent_messages',
|
|
)
|
|
evaluation_config = GenerateConfig(
|
|
anthropic_cache_control={'type': 'ephemeral'},
|
|
anthropic_cache_strategy='evaluation',
|
|
)
|
|
|
|
assert api.cache_control_params(recent_config) == {'type': 'ephemeral'}
|
|
assert api.explicit_cache_control_params(recent_config) is None
|
|
assert api.cache_control_params(evaluation_config) == {'type': 'ephemeral'}
|
|
assert api.explicit_cache_control_params(evaluation_config) == {'type': 'ephemeral'}
|
|
|
|
|
|
def test_evaluation_strategy_marks_system_tools_and_fewshot_but_not_final_question():
|
|
anthropic = _anthropic_utils()
|
|
cache_control = {'type': 'ephemeral'}
|
|
|
|
system, messages = anthropic.anthropic_chat_messages(
|
|
[
|
|
ChatMessageSystem(content='Stable evaluator instructions.'),
|
|
ChatMessageUser(content='Example question.'),
|
|
ChatMessageAssistant(content='Example answer.'),
|
|
ChatMessageUser(content='Current sample question.'),
|
|
],
|
|
cache_control=cache_control,
|
|
cache_strategy='evaluation',
|
|
)
|
|
tools = anthropic.anthropic_chat_tools(
|
|
[
|
|
ToolInfo(name='search', description='Search docs.'),
|
|
ToolInfo(name='read', description='Read docs.'),
|
|
],
|
|
cache_control=cache_control,
|
|
cache_strategy='evaluation',
|
|
)
|
|
|
|
assert isinstance(system, list)
|
|
assert system[-1]['cache_control'] == cache_control
|
|
assert tools[-1]['cache_control'] == cache_control
|
|
assert 'cache_control' not in tools[0]
|
|
assert messages[-2]['content'][-1]['cache_control'] == cache_control
|
|
assert 'cache_control' not in messages[-1]['content'][-1]
|
|
assert len(_cache_control_blocks(messages)) == 1
|
|
|
|
|
|
def test_recent_messages_uses_top_level_cache_control_without_manual_block_markers():
|
|
anthropic = _anthropic_utils()
|
|
cache_control = {'type': 'ephemeral'}
|
|
|
|
system, messages = anthropic.anthropic_chat_messages(
|
|
[
|
|
ChatMessageSystem(content='Stable agent instructions.'),
|
|
ChatMessageUser(content='First turn.'),
|
|
ChatMessageAssistant(content='First answer.'),
|
|
ChatMessageUser(content='Second turn.'),
|
|
],
|
|
cache_control=cache_control,
|
|
cache_strategy='recent_messages',
|
|
)
|
|
tools = anthropic.anthropic_chat_tools(
|
|
[ToolInfo(name='search', description='Search docs.')],
|
|
cache_control=cache_control,
|
|
cache_strategy='recent_messages',
|
|
)
|
|
|
|
assert system == 'Stable agent instructions.'
|
|
assert 'cache_control' not in tools[-1]
|
|
assert len(_cache_control_blocks(messages)) == 0
|
|
|
|
|
|
def test_user_supplied_cache_control_is_preserved_and_not_overwritten():
|
|
anthropic = _anthropic_utils()
|
|
explicit = {'type': 'ephemeral', 'ttl': '1h'}
|
|
automatic = {'type': 'ephemeral', 'ttl': '5m'}
|
|
|
|
system, messages = anthropic.anthropic_chat_messages(
|
|
[
|
|
ChatMessageSystem(
|
|
content=[ContentText(text='Stable.', internal={'anthropic': {
|
|
'cache_control': explicit
|
|
}})]
|
|
),
|
|
ChatMessageUser(
|
|
content=[ContentText(text='Question.', internal={'anthropic': {
|
|
'cache_control': explicit
|
|
}})]
|
|
),
|
|
],
|
|
cache_control=automatic,
|
|
cache_strategy='recent_messages',
|
|
)
|
|
|
|
assert system[-1]['cache_control'] == explicit
|
|
assert messages[-1]['content'][-1]['cache_control'] == explicit
|
|
|
|
|
|
def test_evaluation_cache_control_skips_tool_result_blocks():
|
|
anthropic = _anthropic_utils()
|
|
|
|
messages = [
|
|
ChatMessageUser(content='What is the weather?'),
|
|
ChatMessageAssistant(
|
|
content='',
|
|
tool_calls=[ToolCall(id='call_123', function=ToolFunction(name='get_weather', arguments={'city': 'SF'}))],
|
|
),
|
|
ChatMessageTool(tool_call_id='call_123', content='Sunny, 72F'),
|
|
ChatMessageUser(content='What should I wear?'),
|
|
]
|
|
|
|
system, message_params = anthropic.anthropic_chat_messages(
|
|
messages, cache_control={'type': 'ephemeral'}, cache_strategy='evaluation'
|
|
)
|
|
|
|
assert system is None
|
|
assert len(_cache_control_blocks(message_params)) == 1
|
|
assert message_params[0]['content'][-1]['cache_control'] == {'type': 'ephemeral'}
|
|
|
|
tool_result_block = [
|
|
block for message in message_params for block in message.get('content', [])
|
|
if isinstance(block, dict) and block.get('type') == 'tool_result'
|
|
][0]
|
|
assert 'cache_control' not in tool_result_block
|
|
|
|
|
|
def test_tool_result_explicit_cache_control_is_preserved():
|
|
anthropic = _anthropic_utils()
|
|
cache_control = {'type': 'ephemeral', 'ttl': '1h'}
|
|
|
|
message = ChatMessageTool(
|
|
tool_call_id='call_123',
|
|
content='Sunny',
|
|
internal={'anthropic': {
|
|
'cache_control': cache_control
|
|
}},
|
|
)
|
|
|
|
message_param = anthropic.anthropic_message_param(message)
|
|
|
|
assert message_param['content'][0]['cache_control'] == cache_control
|
|
|
|
|
|
class _FakeUsage:
|
|
|
|
def __init__(
|
|
self,
|
|
input_tokens=0,
|
|
output_tokens=0,
|
|
cache_creation_input_tokens=None,
|
|
cache_read_input_tokens=None,
|
|
):
|
|
self.input_tokens = input_tokens
|
|
self.output_tokens = output_tokens
|
|
self.cache_creation_input_tokens = cache_creation_input_tokens
|
|
self.cache_read_input_tokens = cache_read_input_tokens
|
|
|
|
def model_dump(self):
|
|
return {
|
|
'input_tokens': self.input_tokens,
|
|
'output_tokens': self.output_tokens,
|
|
'cache_creation_input_tokens': self.cache_creation_input_tokens,
|
|
'cache_read_input_tokens': self.cache_read_input_tokens,
|
|
}
|
|
|
|
|
|
class _FakeMessage:
|
|
|
|
def __init__(self, **kwargs):
|
|
self.__dict__.update(kwargs)
|
|
|
|
|
|
class _FakeMessageStartEvent:
|
|
|
|
def __init__(self, message):
|
|
self.message = message
|
|
|
|
|
|
class _FakeMessageDeltaEvent:
|
|
|
|
def __init__(self, usage):
|
|
self.delta = SimpleNamespace(stop_reason='end_turn')
|
|
self.usage = usage
|
|
|
|
|
|
class _FakeContentBlockStartEvent:
|
|
pass
|
|
|
|
|
|
class _FakeContentBlockDeltaEvent:
|
|
pass
|
|
|
|
|
|
def _patch_stream_event_types(monkeypatch, anthropic_utils):
|
|
import anthropic.types as anthropic_types
|
|
|
|
monkeypatch.setattr(anthropic_utils, 'Message', _FakeMessage)
|
|
monkeypatch.setattr(anthropic_types, 'MessageStartEvent', _FakeMessageStartEvent)
|
|
monkeypatch.setattr(anthropic_types, 'MessageDeltaEvent', _FakeMessageDeltaEvent)
|
|
monkeypatch.setattr(anthropic_types, 'ContentBlockStartEvent', _FakeContentBlockStartEvent)
|
|
monkeypatch.setattr(anthropic_types, 'ContentBlockDeltaEvent', _FakeContentBlockDeltaEvent)
|
|
monkeypatch.setattr(anthropic_types, 'Usage', _FakeUsage)
|
|
|
|
|
|
def test_collect_stream_response_preserves_cache_usage(monkeypatch):
|
|
anthropic = _anthropic_utils()
|
|
_patch_stream_event_types(monkeypatch, anthropic)
|
|
|
|
message, _ = anthropic.collect_stream_response([
|
|
_FakeMessageStartEvent(
|
|
_FakeMessage(
|
|
id='msg_1',
|
|
model='claude',
|
|
role='assistant',
|
|
usage=_FakeUsage(
|
|
input_tokens=10,
|
|
cache_creation_input_tokens=3,
|
|
cache_read_input_tokens=4,
|
|
),
|
|
)
|
|
),
|
|
_FakeMessageDeltaEvent(_FakeUsage(output_tokens=2)),
|
|
])
|
|
|
|
assert message.usage.input_tokens == 10
|
|
assert message.usage.output_tokens == 2
|
|
assert message.usage.cache_creation_input_tokens == 3
|
|
assert message.usage.cache_read_input_tokens == 4
|
|
|
|
|
|
async def _fake_async_events(events):
|
|
for event in events:
|
|
yield event
|
|
|
|
|
|
def test_async_collect_stream_response_preserves_cache_usage(monkeypatch):
|
|
anthropic = _anthropic_utils()
|
|
_patch_stream_event_types(monkeypatch, anthropic)
|
|
|
|
message, _ = asyncio.run(
|
|
anthropic.async_collect_stream_response(
|
|
_fake_async_events([
|
|
_FakeMessageStartEvent(
|
|
_FakeMessage(
|
|
id='msg_1',
|
|
model='claude',
|
|
role='assistant',
|
|
usage=_FakeUsage(
|
|
input_tokens=10,
|
|
cache_creation_input_tokens=3,
|
|
cache_read_input_tokens=4,
|
|
),
|
|
)
|
|
),
|
|
_FakeMessageDeltaEvent(_FakeUsage(output_tokens=2)),
|
|
])
|
|
)
|
|
)
|
|
|
|
assert message.usage.input_tokens == 10
|
|
assert message.usage.output_tokens == 2
|
|
assert message.usage.cache_creation_input_tokens == 3
|
|
assert message.usage.cache_read_input_tokens == 4
|