13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
91 lines
3.2 KiB
Python
91 lines
3.2 KiB
Python
import asyncio
|
|
|
|
import pytest
|
|
from openai.types.chat import ChatCompletionChunk
|
|
|
|
from evalscope.models.utils.openai import async_collect_stream_response, collect_stream_response
|
|
|
|
|
|
def _chunk(*, reasoning_content=None, reasoning=None, content=None, finish_reason=None):
|
|
return ChatCompletionChunk.model_validate({
|
|
'id': 'completion-id',
|
|
'created': 1,
|
|
'model': 'test-model',
|
|
'object': 'chat.completion.chunk',
|
|
'choices': [{
|
|
'index': 0,
|
|
'finish_reason': finish_reason,
|
|
'delta': {'content': content, 'reasoning_content': reasoning_content, 'reasoning': reasoning},
|
|
}],
|
|
})
|
|
|
|
|
|
def test_collects_reasoning_and_measures_ttft_from_reasoning(monkeypatch):
|
|
clock = [10.0]
|
|
monkeypatch.setattr('evalscope.models.utils.openai.time.monotonic', lambda: clock[0])
|
|
|
|
def stream():
|
|
clock[0] = 10.25
|
|
yield _chunk(reasoning='thinking')
|
|
clock[0] = 11.0
|
|
yield _chunk(content='answer', finish_reason='stop')
|
|
|
|
completion, ttft = collect_stream_response(stream(), request_start=10.0)
|
|
|
|
assert completion.choices[0].message.content == 'answer'
|
|
assert (getattr(completion.choices[0].message, 'reasoning_content', None), ttft) == (
|
|
'thinking', pytest.approx(0.25)
|
|
)
|
|
|
|
|
|
def test_async_collects_reasoning_and_measures_ttft_from_reasoning(monkeypatch):
|
|
clock = [20.0]
|
|
monkeypatch.setattr('evalscope.models.utils.openai.time.monotonic', lambda: clock[0])
|
|
|
|
async def stream():
|
|
clock[0] = 20.25
|
|
yield _chunk(reasoning='thinking')
|
|
clock[0] = 21.0
|
|
yield _chunk(content='answer', finish_reason='stop')
|
|
|
|
completion, ttft = asyncio.run(async_collect_stream_response(stream(), request_start=20.0))
|
|
|
|
assert completion.choices[0].message.content == 'answer'
|
|
assert (getattr(completion.choices[0].message, 'reasoning_content', None), ttft) == (
|
|
'thinking', pytest.approx(0.25)
|
|
)
|
|
|
|
|
|
def test_falls_back_to_reasoning_when_reasoning_content_is_empty(monkeypatch):
|
|
clock = [10.0]
|
|
monkeypatch.setattr('evalscope.models.utils.openai.time.monotonic', lambda: clock[0])
|
|
|
|
def stream():
|
|
clock[0] = 10.25
|
|
yield _chunk(reasoning_content='', reasoning='thinking')
|
|
clock[0] = 11.0
|
|
yield _chunk(content='answer', finish_reason='stop')
|
|
|
|
completion, ttft = collect_stream_response(stream(), request_start=10.0)
|
|
|
|
assert completion.choices[0].message.content == 'answer'
|
|
assert getattr(completion.choices[0].message, 'reasoning_content', None) == 'thinking'
|
|
assert ttft == pytest.approx(0.25)
|
|
|
|
|
|
def test_async_falls_back_to_reasoning_when_reasoning_content_is_empty(monkeypatch):
|
|
clock = [20.0]
|
|
monkeypatch.setattr('evalscope.models.utils.openai.time.monotonic', lambda: clock[0])
|
|
|
|
async def stream():
|
|
clock[0] = 20.25
|
|
yield _chunk(reasoning_content='', reasoning='thinking')
|
|
clock[0] = 21.0
|
|
yield _chunk(content='answer', finish_reason='stop')
|
|
|
|
completion, ttft = asyncio.run(async_collect_stream_response(stream(), request_start=20.0))
|
|
|
|
assert completion.choices[0].message.content == 'answer'
|
|
assert getattr(completion.choices[0].message, 'reasoning_content', None) == 'thinking'
|
|
assert ttft == pytest.approx(0.25)
|