13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
63 lines
2.3 KiB
Python
63 lines
2.3 KiB
Python
"""Regression tests for GPQA choice shuffling determinism (issue #1579).
|
|
|
|
``record_to_sample`` must be a pure function: an unseeded shuffle made the
|
|
option order differ on every dataset build, so ``--rerun-review`` scored
|
|
cached predictions against a freshly shuffled answer key.
|
|
"""
|
|
|
|
import random
|
|
from typing import Any, Dict, List
|
|
|
|
from evalscope.api.registry import get_benchmark
|
|
|
|
|
|
def _record(question: str) -> Dict[str, Any]:
|
|
return {
|
|
'Question': question,
|
|
'Correct Answer': '10^-4 eV',
|
|
'Incorrect Answer 1': '10^-8 eV',
|
|
'Incorrect Answer 2': '10^-9 eV',
|
|
'Incorrect Answer 3': '10^-11 eV',
|
|
}
|
|
|
|
|
|
QUESTIONS: List[str] = [
|
|
'Two quantum states with energies E1 and E2 have a lifetime of 10^-9 sec and 10^-8 sec.',
|
|
'Which of the following is the energy difference that can be clearly resolved?',
|
|
'A different graduate-level physics question about spectral line widths.',
|
|
]
|
|
|
|
|
|
def test_record_to_sample_is_deterministic_across_calls() -> None:
|
|
adapter = get_benchmark('gpqa_diamond')
|
|
record = _record(QUESTIONS[0])
|
|
|
|
samples = []
|
|
for seed in range(8):
|
|
# Perturb the global RNG the way a real run does (dataset shuffle, other
|
|
# adapters, model sampling) to prove the choice order does not depend on it.
|
|
random.seed(seed)
|
|
samples.append(adapter.record_to_sample(record))
|
|
|
|
targets = {str(sample.target) for sample in samples}
|
|
orders = {tuple(sample.choices) for sample in samples}
|
|
assert len(targets) == 1, f'answer letter drifted across rebuilds: {targets}'
|
|
assert len(orders) == 1, f'choice order drifted across rebuilds: {orders}'
|
|
|
|
|
|
def test_target_still_points_at_the_correct_answer() -> None:
|
|
adapter = get_benchmark('gpqa_diamond')
|
|
|
|
for question in QUESTIONS:
|
|
sample = adapter.record_to_sample(_record(question))
|
|
letter_index = ord(str(sample.target)) - ord('A')
|
|
assert sample.choices[letter_index] == '10^-4 eV'
|
|
|
|
|
|
def test_shuffle_still_varies_between_questions() -> None:
|
|
"""Position-bias protection must survive the switch to a seeded RNG."""
|
|
adapter = get_benchmark('gpqa_diamond')
|
|
|
|
orders = {tuple(adapter.record_to_sample(_record(question)).choices) for question in QUESTIONS}
|
|
assert len(orders) > 1, 'every question produced the same option order'
|