13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
96 lines
2.9 KiB
Python
96 lines
2.9 KiB
Python
"""Lazy ``llm_judges`` initialization must be thread-safe."""
|
|
|
|
import threading
|
|
import time
|
|
from typing import List, cast
|
|
|
|
import pytest
|
|
|
|
from evalscope.api.benchmark import BenchmarkMeta
|
|
from evalscope.api.mixin import LLMJudgeMixin
|
|
from evalscope.metrics import LLMJudge
|
|
|
|
|
|
class _CountingJudgeMixin(LLMJudgeMixin):
|
|
"""Minimal judge mixin with deliberately slow initialization."""
|
|
|
|
use_llm_judge = True
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(benchmark_meta=BenchmarkMeta(name='stub', dataset_id='stub'), task_config=None)
|
|
self.init_calls = 0
|
|
self._calls_lock = threading.Lock()
|
|
self._judge = cast(LLMJudge, object())
|
|
|
|
def init_llm_judges(self) -> List[LLMJudge]:
|
|
with self._calls_lock:
|
|
self.init_calls += 1
|
|
time.sleep(0.05)
|
|
return [self._judge]
|
|
|
|
|
|
class _FailOnceJudgeMixin(_CountingJudgeMixin):
|
|
"""Fails initialization once so callers can verify retry behavior."""
|
|
|
|
def init_llm_judges(self) -> List[LLMJudge]:
|
|
with self._calls_lock:
|
|
self.init_calls += 1
|
|
should_fail = self.init_calls == 1
|
|
if should_fail:
|
|
raise RuntimeError('judge initialization failed')
|
|
return [self._judge]
|
|
|
|
|
|
class _DisabledJudgeMixin(_CountingJudgeMixin):
|
|
"""Disables judge construction without requiring task-config plumbing."""
|
|
|
|
use_llm_judge = False
|
|
|
|
|
|
def test_llm_judges_initializes_exactly_once_under_concurrency() -> None:
|
|
"""All concurrent first accesses must receive one initialized judge list."""
|
|
mixin = _CountingJudgeMixin()
|
|
num_threads = 8
|
|
barrier = threading.Barrier(num_threads)
|
|
results: List[List[LLMJudge]] = []
|
|
errors: List[BaseException] = []
|
|
|
|
def access() -> None:
|
|
try:
|
|
barrier.wait()
|
|
results.append(mixin.llm_judges)
|
|
except BaseException as error: # noqa: BLE001 - surfaced after every worker exits
|
|
errors.append(error)
|
|
|
|
threads = [threading.Thread(target=access) for _ in range(num_threads)]
|
|
for thread in threads:
|
|
thread.start()
|
|
for thread in threads:
|
|
thread.join()
|
|
|
|
assert errors == []
|
|
assert mixin.init_calls == 1
|
|
assert len(results) == num_threads
|
|
assert all(result is results[0] for result in results)
|
|
assert all(result[0] is results[0][0] for result in results)
|
|
|
|
|
|
def test_llm_judges_retries_after_initialization_failure() -> None:
|
|
"""A failed initialization must not publish a partial result."""
|
|
mixin = _FailOnceJudgeMixin()
|
|
|
|
with pytest.raises(RuntimeError, match='judge initialization failed'):
|
|
_ = mixin.llm_judges
|
|
|
|
assert mixin._llm_judges is None
|
|
assert mixin.llm_judges == [mixin._judge]
|
|
assert mixin.init_calls == 2
|
|
|
|
|
|
def test_llm_judges_returns_empty_when_judge_is_disabled() -> None:
|
|
"""Rule-only scoring must not initialize any judge."""
|
|
mixin = _DisabledJudgeMixin()
|
|
|
|
assert mixin.llm_judges == []
|
|
assert mixin.init_calls == 0
|