13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
214 lines
7.6 KiB
Python
214 lines
7.6 KiB
Python
import copy
|
|
from typing import Any, ClassVar, List, Optional
|
|
|
|
from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
|
|
from evalscope.api.benchmark.statistics import SampleExample
|
|
from evalscope.api.dataset import DataLoader, DatasetDict, MemoryDataset, Sample
|
|
from evalscope.api.evaluator import TaskState
|
|
from evalscope.api.messages import ChatMessageSystem, ChatMessageUser
|
|
from evalscope.api.metric import Score
|
|
from evalscope.api.model import ModelOutput
|
|
from evalscope.config import TaskConfig
|
|
from evalscope.constants import JudgeStrategy
|
|
|
|
|
|
class CapturingDataLoader(DataLoader):
|
|
|
|
latest_limit: ClassVar[Optional[int]] = None
|
|
latest_repeats: ClassVar[Optional[int]] = None
|
|
latest_version: ClassVar[Optional[str]] = None
|
|
latest_seed: ClassVar[Optional[int]] = None
|
|
|
|
def load(self) -> MemoryDataset:
|
|
self.__class__.latest_limit = self.limit
|
|
self.__class__.latest_repeats = self.repeats
|
|
self.__class__.latest_version = self.version
|
|
self.__class__.latest_seed = self.seed
|
|
|
|
samples: List[Sample] = [
|
|
Sample(input='question-1', target='answer-1', subset_key='subset-a'),
|
|
Sample(input='question-2', target='answer-2', subset_key='subset-a'),
|
|
]
|
|
if self.limit is not None:
|
|
samples = samples[:self.limit]
|
|
if self.repeats > 1:
|
|
samples = [copy.deepcopy(sample) for sample in samples for _ in range(self.repeats)]
|
|
|
|
dataset = MemoryDataset(samples=samples, name='dummy')
|
|
dataset.reindex(group_size=self.repeats)
|
|
return dataset
|
|
|
|
|
|
class DummyReformatAdapter(DefaultDataAdapter):
|
|
|
|
def __init__(self, **kwargs: Any) -> None:
|
|
super().__init__(**kwargs)
|
|
self.reformat_subset = True
|
|
|
|
def record_to_sample(self, record: Any) -> Sample:
|
|
return Sample(input=str(record), target='', subset_key='subset-a')
|
|
|
|
|
|
class DummyLLMJudgeAdapter(DefaultDataAdapter):
|
|
|
|
llm_judge_default = True
|
|
|
|
def record_to_sample(self, record: Any) -> Sample:
|
|
return Sample(input=str(record), target='')
|
|
|
|
|
|
def make_adapter(
|
|
repeats: int = 3,
|
|
limit: Optional[int] = None,
|
|
system_prompt: Optional[str] = None,
|
|
dataset_revision: Optional[str] = None,
|
|
) -> DummyReformatAdapter:
|
|
task_config = TaskConfig(datasets=['dummy'], repeats=repeats, limit=limit)
|
|
benchmark_meta = BenchmarkMeta(
|
|
name='dummy',
|
|
dataset_id='dummy',
|
|
subset_list=['subset-a'],
|
|
default_subset='default',
|
|
eval_split='test',
|
|
prompt_template='{question}',
|
|
system_prompt=system_prompt,
|
|
dataset_revision=dataset_revision,
|
|
)
|
|
return DummyReformatAdapter(benchmark_meta=benchmark_meta, task_config=task_config)
|
|
|
|
|
|
def test_reformat_subset_repeats_are_applied_once_after_grouping() -> None:
|
|
CapturingDataLoader.latest_limit = None
|
|
CapturingDataLoader.latest_repeats = None
|
|
|
|
adapter = make_adapter(repeats=3)
|
|
|
|
dataset_dict: DatasetDict = adapter.load_subsets(
|
|
lambda subset: adapter.load_subset(subset=subset, data_loader=CapturingDataLoader)
|
|
)
|
|
|
|
assert CapturingDataLoader.latest_limit is None
|
|
assert CapturingDataLoader.latest_repeats == 1
|
|
assert len(dataset_dict['subset-a']) == 6
|
|
assert [sample.group_id for sample in dataset_dict['subset-a']] == [0, 0, 0, 1, 1, 1]
|
|
|
|
|
|
def test_load_subset_passes_the_resolved_dataset_revision() -> None:
|
|
adapter = make_adapter(dataset_revision='2026-08-21')
|
|
|
|
adapter.load_subset(subset='subset-a', data_loader=CapturingDataLoader)
|
|
|
|
assert CapturingDataLoader.latest_version == '2026-08-21'
|
|
assert CapturingDataLoader.latest_seed == 42
|
|
|
|
|
|
def test_shuffle_choices_configuration_survives_adapter_initialization() -> None:
|
|
meta = BenchmarkMeta(
|
|
name='choice_shuffle',
|
|
dataset_id='dummy',
|
|
eval_split='test',
|
|
shuffle_choices=True,
|
|
)
|
|
|
|
adapter = DummyReformatAdapter(benchmark_meta=meta, task_config=TaskConfig(datasets=['choice_shuffle']))
|
|
|
|
assert adapter.shuffle_choices is True
|
|
|
|
user_configured_meta = BenchmarkMeta(name='user_choice_shuffle', dataset_id='dummy', eval_split='test')
|
|
user_configured_meta._update({'shuffle_choices': True})
|
|
user_configured_adapter = DummyReformatAdapter(
|
|
benchmark_meta=user_configured_meta,
|
|
task_config=TaskConfig(datasets=['user_choice_shuffle']),
|
|
)
|
|
|
|
assert user_configured_adapter.shuffle_choices is True
|
|
|
|
|
|
def test_auto_judge_strategy_uses_adapter_class_default() -> None:
|
|
benchmark_meta = BenchmarkMeta(name='dummy_judge', dataset_id='dummy', eval_split='test')
|
|
adapter = DummyLLMJudgeAdapter(
|
|
benchmark_meta=benchmark_meta,
|
|
task_config=TaskConfig(datasets=['dummy_judge'], judge={'strategy': JudgeStrategy.AUTO}),
|
|
)
|
|
|
|
assert adapter.use_llm_judge is True
|
|
|
|
adapter._task_config.judge.strategy = JudgeStrategy.RULE
|
|
assert adapter.use_llm_judge is False
|
|
|
|
adapter._task_config.judge.strategy = JudgeStrategy.LLM
|
|
assert adapter.use_llm_judge is True
|
|
|
|
|
|
def test_llm_recall_skips_judge_when_rule_score_is_perfect(monkeypatch) -> None:
|
|
benchmark_meta = BenchmarkMeta(name='dummy_judge', dataset_id='dummy', eval_split='test')
|
|
adapter = DummyLLMJudgeAdapter(
|
|
benchmark_meta=benchmark_meta,
|
|
task_config=TaskConfig(
|
|
datasets=['dummy_judge'],
|
|
judge={'strategy': 'llm_recall', 'models': {'model_id': 'judge'}},
|
|
),
|
|
)
|
|
state = TaskState(
|
|
model='m',
|
|
sample=Sample(input='question', target='answer'),
|
|
output=ModelOutput(model='m', completion='answer'),
|
|
completed=True,
|
|
)
|
|
monkeypatch.setattr(adapter, 'match_score', lambda **_: Score(value={'acc': 1.0}, main_score_name='acc'))
|
|
monkeypatch.setattr(adapter, 'score_with_judge_contracts', lambda **_: (_ for _ in ()).throw(AssertionError()))
|
|
|
|
score = adapter.calculate_metrics(state).score
|
|
|
|
assert score.value == {'acc': 1.0}
|
|
|
|
|
|
def test_sample_score_preserves_repeat_position() -> None:
|
|
adapter = make_adapter(repeats=3)
|
|
state = TaskState(
|
|
model='m',
|
|
sample=Sample(id=4, group_id=1, input='question', target='answer'),
|
|
output=ModelOutput(model='m', completion='answer'),
|
|
completed=True,
|
|
)
|
|
|
|
sample_score = adapter.calculate_metrics(state)
|
|
|
|
assert sample_score.generation_index == 1
|
|
|
|
|
|
def test_sample_example_detects_parameterized_truncation_marker() -> None:
|
|
sample = Sample(input='prefix ... [TRUNCATED 123 chars] ... suffix', target='answer')
|
|
|
|
sample_example = SampleExample.from_sample(sample=sample)
|
|
|
|
assert sample_example.truncated is True
|
|
|
|
|
|
def test_empty_system_prompt_is_added_to_string_input() -> None:
|
|
adapter = make_adapter(system_prompt='')
|
|
|
|
messages = adapter.process_sample_str_input(Sample(input='question', target='answer'), subset='subset-a')
|
|
|
|
assert [message.role for message in messages] == ['system', 'user']
|
|
assert messages[0].content == ''
|
|
|
|
|
|
def test_empty_system_prompt_is_added_once_to_message_input() -> None:
|
|
adapter = make_adapter(system_prompt='')
|
|
existing_system = ChatMessageSystem(content='existing')
|
|
|
|
messages = adapter.process_sample_messages_input(
|
|
Sample(input=[ChatMessageUser(content='question')], target='answer'), subset='subset-a'
|
|
)
|
|
existing_messages = adapter.process_sample_messages_input(
|
|
Sample(input=[existing_system, ChatMessageUser(content='question')], target='answer'), subset='subset-a'
|
|
)
|
|
|
|
assert [message.role for message in messages] == ['system', 'user']
|
|
assert messages[0].content == ''
|
|
assert [message.role for message in existing_messages] == ['system', 'user']
|
|
assert len(existing_messages) == 2
|
|
assert existing_messages[0] is existing_system
|
|
assert existing_messages[0].content == 'existing'
|