evalstone/evalscope/tests/models/test_openai_audio_input.py

"""Regression tests for OpenAI-compatible audio request preprocessing."""

import asyncio
import base64
import time
from pathlib import Path
from types import SimpleNamespace
from typing import Any, Dict, List, Tuple
from unittest.mock import Mock, patch

import pytest
import requests
from openai.types.chat import ChatCompletion

from evalscope.api.messages import ChatMessageUser, ContentAudio
from evalscope.api.model import ChatCompletionChoice, GenerateConfig
from evalscope.api.tool import ToolChoice, ToolInfo
from evalscope.models import openai_compatible
from evalscope.models.openai_compatible import OpenAICompatibleAPI
from evalscope.models.utils.openai import openai_chat_completion_part

DASHSCOPE_BASE_URL = 'https://dashscope.aliyuncs.com/compatible-mode/v1'
REMOTE_AUDIO_URL = 'https://example.test/audio'


def _mock_response(content: bytes) -> Mock:
    response = Mock()
    response.content = content
    return response


def _completion() -> ChatCompletion:
    return ChatCompletion.model_validate({
        'id': 'completion-id',
        'created': 1,
        'model': 'test-model',
        'object': 'chat.completion',
        'choices': [{
            'index': 0,
            'finish_reason': 'stop',
            'message': {
                'role': 'assistant',
                'content': 'done',
            },
        }],
    })


def _prepare_async_api(monkeypatch: pytest.MonkeyPatch, captured_request: Dict[str, Any]) -> OpenAICompatibleAPI:
    api = object.__new__(OpenAICompatibleAPI)
    api.base_url = 'https://example.test/v1'
    api.model_name = 'test-model'

    def resolve_tools(
        tools: List[ToolInfo], tool_choice: ToolChoice, config: GenerateConfig
    ) -> Tuple[List[ToolInfo], ToolChoice, GenerateConfig]:
        return tools, tool_choice, config

    def completion_params(config: GenerateConfig, tools: bool) -> Dict[str, Any]:
        return {'model': 'test-model'}

    def validate_request_params(request: Dict[str, Any]) -> None:
        return None

    def on_response(response: Dict[str, Any]) -> None:
        return None

    def chat_choices_from_completion(
        completion: ChatCompletion, tools: List[ToolInfo]
    ) -> List[ChatCompletionChoice]:
        return []

    def model_output_from_openai(*args: Any, **kwargs: Any) -> Any:
        return SimpleNamespace(usage=None, message=SimpleNamespace(), time=None)

    async def create(**request: Any) -> ChatCompletion:
        captured_request.update(request)
        return _completion()

    def async_client(_: OpenAICompatibleAPI) -> Any:
        return SimpleNamespace(chat=SimpleNamespace(completions=SimpleNamespace(create=create)))

    api.resolve_tools = resolve_tools
    api.completion_params = completion_params
    api.validate_request_params = validate_request_params
    api.on_response = on_response
    api.chat_choices_from_completion = chat_choices_from_completion
    monkeypatch.setattr(openai_compatible, 'model_output_from_openai', model_output_from_openai)
    monkeypatch.setattr(OpenAICompatibleAPI, 'async_client', property(async_client))
    return api


def test_http_audio_url_is_downloaded_and_sent_as_raw_base64() -> None:
    with patch('requests.get', return_value=_mock_response(b'WAVE')) as get:
        part = openai_chat_completion_part(ContentAudio(audio=REMOTE_AUDIO_URL, format='wav'))

    get.assert_called_once_with(REMOTE_AUDIO_URL, timeout=30)
    assert part['input_audio']['data'] == 'V0FWRQ=='
    assert base64.b64decode(part['input_audio']['data'], validate=True) == b'WAVE'


def test_http_audio_url_uses_audio_data_uri_for_dashscope() -> None:
    with patch('requests.get', return_value=_mock_response(b'WAVE')):
        part = openai_chat_completion_part(
            ContentAudio(audio=REMOTE_AUDIO_URL, format='wav'), base_url=DASHSCOPE_BASE_URL
        )

    assert part['input_audio']['data'] == 'data:audio/wav;base64,V0FWRQ=='


def test_extensionless_local_audio_uses_declared_mime_type(tmp_path: Path) -> None:
    audio_path = tmp_path / 'recording'
    audio_path.write_bytes(b'WAVE')

    part = openai_chat_completion_part(
        ContentAudio(audio=str(audio_path), format='mp3'), base_url=DASHSCOPE_BASE_URL
    )

    assert part['input_audio']['data'] == 'data:audio/mpeg;base64,V0FWRQ=='


def test_audio_data_uri_preserves_provider_compatibility() -> None:
    data_uri = 'data:audio/wav;base64,V0FWRQ=='

    dashscope_part = openai_chat_completion_part(
        ContentAudio(audio=data_uri, format='wav'), base_url=DASHSCOPE_BASE_URL
    )
    openai_part = openai_chat_completion_part(ContentAudio(audio=data_uri, format='wav'))

    assert dashscope_part['input_audio']['data'] == data_uri
    assert openai_part['input_audio']['data'] == 'V0FWRQ=='


def test_audio_download_failure_propagates() -> None:
    with patch('requests.get', side_effect=requests.Timeout('timed out')):
        with pytest.raises(requests.Timeout, match='timed out'):
            openai_chat_completion_part(ContentAudio(audio=REMOTE_AUDIO_URL, format='wav'))


def test_generate_async_keeps_event_loop_responsive_during_audio_download(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    captured_request: Dict[str, Any] = {}
    api = _prepare_async_api(monkeypatch, captured_request)

    def delayed_get(*args: Any, **kwargs: Any) -> Mock:
        time.sleep(0.2)
        return _mock_response(b'WAVE')

    async def run() -> None:
        started = time.monotonic()
        generation = asyncio.create_task(
            api.generate_async(
                [ChatMessageUser(content=[ContentAudio(audio=REMOTE_AUDIO_URL, format='wav')])],
                [],
                None,
                GenerateConfig(retries=1),
            )
        )
        await asyncio.sleep(0.02)
        assert time.monotonic() - started < 0.1
        await generation

    with patch('requests.get', side_effect=delayed_get):
        asyncio.run(run())

    audio_data = captured_request['messages'][0]['content'][0]['input_audio']['data']
    assert audio_data == 'V0FWRQ=='