13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
71 lines
2.3 KiB
Python
71 lines
2.3 KiB
Python
import math
|
|
from typing import List
|
|
|
|
import pytest
|
|
|
|
from evalscope.api.dataset import MemoryDataset, Sample
|
|
from evalscope.api.dataset.dataset import DatasetDict
|
|
|
|
|
|
def _build_dataset(subset_sizes: dict) -> MemoryDataset:
|
|
samples: List[Sample] = []
|
|
for subset_key, size in subset_sizes.items():
|
|
for idx in range(size):
|
|
samples.append(Sample(input=f'{subset_key}-{idx}', subset_key=subset_key))
|
|
return MemoryDataset(samples, name='dummy')
|
|
|
|
|
|
def test_from_dataset_float_limit_applies_per_subset() -> None:
|
|
# Regression test for issue #1525: a float limit must be resolved
|
|
# against each subset's own size, not the first subset's size.
|
|
dataset = _build_dataset({'small': 4, 'large': 10})
|
|
|
|
dataset_dict = DatasetDict.from_dataset(dataset, subset_list=['small', 'large'], limit=0.5)
|
|
|
|
assert len(dataset_dict['small']) == 2 # 4 * 0.5
|
|
assert len(dataset_dict['large']) == 5 # 10 * 0.5, was 2 before the fix
|
|
|
|
|
|
def test_from_dataset_int_limit_applies_per_subset() -> None:
|
|
dataset = _build_dataset({'small': 4, 'large': 10})
|
|
|
|
dataset_dict = DatasetDict.from_dataset(dataset, subset_list=['small', 'large'], limit=3)
|
|
|
|
assert len(dataset_dict['small']) == 3
|
|
assert len(dataset_dict['large']) == 3
|
|
|
|
|
|
def test_from_dataset_no_limit_keeps_all_samples() -> None:
|
|
dataset = _build_dataset({'small': 4, 'large': 10})
|
|
|
|
dataset_dict = DatasetDict.from_dataset(dataset, subset_list=['small', 'large'], limit=None)
|
|
|
|
assert len(dataset_dict['small']) == 4
|
|
assert len(dataset_dict['large']) == 10
|
|
|
|
|
|
@pytest.mark.parametrize('limit', [2.5, -1, -0.1, math.nan, math.inf])
|
|
def test_from_dataset_rejects_invalid_limits(limit: float | int) -> None:
|
|
dataset = _build_dataset({'small': 4})
|
|
|
|
with pytest.raises(ValueError, match='Limit must'):
|
|
DatasetDict.from_dataset(dataset, subset_list=['small'], limit=limit)
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
'limit, expected',
|
|
[
|
|
(None, 4),
|
|
(0, 0),
|
|
(0.5, 2),
|
|
(1.0, 4),
|
|
(2, 2),
|
|
],
|
|
)
|
|
def test_from_dataset_preserves_valid_limit_semantics(limit: float | int | None, expected: int) -> None:
|
|
dataset = _build_dataset({'small': 4})
|
|
|
|
dataset_dict = DatasetDict.from_dataset(dataset, subset_list=['small'], limit=limit)
|
|
|
|
assert len(dataset_dict['small']) == expected
|