13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
73 lines
2.8 KiB
Python
73 lines
2.8 KiB
Python
"""Unit tests for perf ``Arguments`` validation guards.
|
|
|
|
Covers three previously missing validations:
|
|
- ``--open-loop --multi-turn``: used to be accepted silently; open-loop then
|
|
forced ``parallel=[-1]`` so the multi-turn strategy spawned zero workers and
|
|
the run produced no requests at all.
|
|
- ``--parallel <= 0`` (closed loop): used to reach the strategy and crash
|
|
opaquely ('Set of Tasks/Futures is empty' / 'Semaphore initial value must be
|
|
>= 0').
|
|
- ``--log-every-n-query 0``: used to cause a ZeroDivisionError in the metrics
|
|
consumer's modulo; now coerced to 1 like the other count-type knobs.
|
|
"""
|
|
import pytest
|
|
|
|
from evalscope.perf.arguments import Arguments
|
|
|
|
|
|
def _args(**kwargs) -> Arguments:
|
|
return Arguments(model='test-model', url='http://localhost:8080/v1/chat/completions', **kwargs)
|
|
|
|
|
|
class TestOpenLoopMultiTurnRejected:
|
|
|
|
def test_open_loop_with_multi_turn_raises(self):
|
|
with pytest.raises(ValueError, match='not supported in open-loop'):
|
|
_args(open_loop=True, multi_turn=True, rate=1.0, number=2)
|
|
|
|
def test_multi_turn_closed_loop_still_accepted(self):
|
|
args = _args(multi_turn=True, parallel=2, number=2)
|
|
assert args.multi_turn is True
|
|
|
|
def test_open_loop_single_turn_still_accepted(self):
|
|
args = _args(open_loop=True, rate=1.0, number=2)
|
|
assert args.parallel == [-1] # unbounded concurrency marker stays intact
|
|
|
|
|
|
class TestParallelPositivity:
|
|
|
|
@pytest.mark.parametrize('parallel', [0, -1, -5])
|
|
def test_non_positive_parallel_rejected(self, parallel):
|
|
with pytest.raises(ValueError, match='--parallel values must be > 0'):
|
|
_args(parallel=parallel)
|
|
|
|
@pytest.mark.parametrize('parallel', [1, 4, [8]])
|
|
def test_positive_parallel_accepted(self, parallel):
|
|
assert _args(parallel=parallel).parallel == ([parallel] if isinstance(parallel, int) else parallel)
|
|
|
|
def test_positive_parallel_sweep_accepted(self):
|
|
assert _args(parallel=[1, 2], number=[10, 10]).parallel == [1, 2]
|
|
|
|
def test_zero_inside_sweep_rejected(self):
|
|
with pytest.raises(ValueError, match='--parallel values must be > 0'):
|
|
_args(parallel=[1, 0, 2], number=[1, 1, 1])
|
|
|
|
def test_open_loop_parallel_marker_not_affected(self):
|
|
# Open-loop mode force-sets parallel=[-1] internally; the positivity
|
|
# rule applies only to closed-loop sweeps.
|
|
args = _args(open_loop=True, rate=1.0, number=2)
|
|
assert args.parallel == [-1]
|
|
|
|
|
|
class TestLogEveryNQuery:
|
|
|
|
@pytest.mark.parametrize('value', [0, -3])
|
|
def test_non_positive_is_coerced_to_one(self, value):
|
|
assert _args(log_every_n_query=value).log_every_n_query == 1
|
|
|
|
def test_positive_value_preserved(self):
|
|
assert _args(log_every_n_query=50).log_every_n_query == 50
|
|
|
|
def test_default_preserved(self):
|
|
assert _args().log_every_n_query == 100
|