13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
132 lines
4.5 KiB
Python
132 lines
4.5 KiB
Python
from types import SimpleNamespace
|
|
|
|
from evalscope.api.messages import ChatMessageAssistant
|
|
from evalscope.api.messages.perf_metrics import PerformanceMetrics
|
|
from evalscope.api.metric import SampleScore, Score
|
|
from evalscope.api.model import ModelOutput
|
|
from evalscope.evaluator.evaluator import DefaultEvaluator, _PoolContext, _WorkItem
|
|
from evalscope.evaluator.execution_tracker import ExecutionTracker
|
|
from evalscope.evaluator.perf_collector import PerfCollector
|
|
|
|
|
|
def _sample_score(sample_id: int) -> SampleScore:
|
|
return SampleScore(sample_id=sample_id, score=Score(value={'acc': 1.0}, main_score_name='acc'))
|
|
|
|
|
|
def test_execution_summary_tracks_partial_failure_and_cached_scores() -> None:
|
|
tracker = ExecutionTracker()
|
|
tracker.record_error('test')
|
|
|
|
summary = tracker.summarize({'test': [object(), object(), object()]}, {'test': [_sample_score(0), _sample_score(1)]})
|
|
|
|
assert summary.model_dump() == {
|
|
'requested': 3,
|
|
'succeeded': 2,
|
|
'errored': 1,
|
|
'incomplete': True,
|
|
'subsets': {
|
|
'test': {
|
|
'requested': 3,
|
|
'succeeded': 2,
|
|
'errored': 1,
|
|
}
|
|
},
|
|
}
|
|
|
|
|
|
def test_execution_summary_marks_all_success_complete() -> None:
|
|
tracker = ExecutionTracker()
|
|
|
|
summary = tracker.summarize({'test': [object()]}, {'test': [_sample_score(0)]})
|
|
|
|
assert not summary.incomplete
|
|
assert summary.errored == 0
|
|
|
|
|
|
def test_pool_records_all_ignored_failures() -> None:
|
|
evaluator = object.__new__(DefaultEvaluator)
|
|
evaluator.benchmark_name = 'test'
|
|
evaluator.task_config = SimpleNamespace(ignore_errors=True, eval_batch_size=1)
|
|
evaluator._execution_tracker = ExecutionTracker()
|
|
|
|
def fail_work_item(*args, **kwargs):
|
|
raise RuntimeError('inference failed')
|
|
|
|
evaluator._process_work_item = fail_work_item
|
|
context = _PoolContext(
|
|
work_items=[_WorkItem(subset='test', sample=object())],
|
|
cached_scores_by_subset={},
|
|
review_pending_by_subset={},
|
|
model_prediction_dir='',
|
|
total_cached=0,
|
|
)
|
|
|
|
assert evaluator._run_pool(context) == {}
|
|
summary = evaluator._execution_tracker.summarize({'test': [object()]}, {})
|
|
assert summary.model_dump(exclude={'subsets'}) == {
|
|
'requested': 1,
|
|
'succeeded': 0,
|
|
'errored': 1,
|
|
'incomplete': True,
|
|
}
|
|
|
|
|
|
def test_batch_review_error_tracks_cached_and_successful_samples() -> None:
|
|
evaluator = object.__new__(DefaultEvaluator)
|
|
evaluator.task_config = SimpleNamespace(ignore_errors=True)
|
|
evaluator._execution_tracker = ExecutionTracker()
|
|
evaluator._sample_scores_by_subset = {}
|
|
evaluator.benchmark = SimpleNamespace(
|
|
use_batch_scoring=True,
|
|
aggregate_scores=lambda sample_scores: [],
|
|
)
|
|
task_state = SimpleNamespace(sample_id=2)
|
|
|
|
def review_subset(subset, task_states, review_fn, on_error):
|
|
on_error(task_states[0], RuntimeError('batch review failed'))
|
|
return [_sample_score(1)]
|
|
|
|
evaluator.batch_reviewer = SimpleNamespace(review_subset=review_subset)
|
|
context = _PoolContext(
|
|
work_items=[],
|
|
cached_scores_by_subset={'test': [_sample_score(0)]},
|
|
review_pending_by_subset={'test': [task_state]},
|
|
model_prediction_dir='',
|
|
total_cached=1,
|
|
)
|
|
|
|
evaluator._aggregate_scores({'test': [object(), object(), object()]}, context, {'test': []})
|
|
summary = evaluator._execution_tracker.summarize(
|
|
{'test': [object(), object(), object()]}, evaluator._sample_scores_by_subset
|
|
)
|
|
assert summary.model_dump(exclude={'subsets'}) == {
|
|
'requested': 3,
|
|
'succeeded': 2,
|
|
'errored': 1,
|
|
'incomplete': True,
|
|
}
|
|
|
|
|
|
def test_perf_coverage_counts_missing_request_metrics() -> None:
|
|
evaluator = object.__new__(DefaultEvaluator)
|
|
evaluator.perf_collector = PerfCollector()
|
|
evaluator._perf_request_count = 0
|
|
evaluator._perf_metric_count = 0
|
|
task_state = type(
|
|
'TaskStateProbe',
|
|
(), {
|
|
'messages': [
|
|
ChatMessageAssistant(content='first', perf_metrics=PerformanceMetrics(latency=1.0)),
|
|
ChatMessageAssistant(content='second'),
|
|
],
|
|
'output': ModelOutput.from_content(model='test', content='second'),
|
|
'sample_id': 1,
|
|
},
|
|
)()
|
|
|
|
evaluator._record_perf(task_state)
|
|
|
|
assert evaluator._perf_request_count == 2
|
|
assert evaluator._perf_metric_count == 1
|
|
assert evaluator.perf_collector.get_perf_dict()['summary']['n_samples'] == 1
|