13274243a0
Keep K3 suite selection and report-schema scoring in bash, merge K3/vision dataset_args into dpv4 yamls, and pin EvalScope at 735d920ee911 with local patches. Co-authored-by: Cursor <cursoragent@cursor.com>
19 lines
729 B
Python
19 lines
729 B
Python
from evalscope.api.dataset import Sample
|
|
from evalscope.api.registry import get_benchmark
|
|
from evalscope.benchmarks.bbh.bbh_adapter import SUBSET_LIST
|
|
from evalscope.benchmarks.bbh.cot_prompts import COT_PROMPTS
|
|
|
|
|
|
def test_all_bbh_subsets_have_cot_prompts() -> None:
|
|
assert set(COT_PROMPTS) == set(SUBSET_LIST)
|
|
assert all(prompt.strip() for prompt in COT_PROMPTS.values())
|
|
|
|
|
|
def test_bbh_adapter_uses_imported_cot_prompt() -> None:
|
|
adapter = get_benchmark('bbh')
|
|
formatted = adapter.format_fewshot_template('fallback', Sample(input='Question?', subset_key='navigate'))
|
|
|
|
assert formatted.startswith(COT_PROMPTS['navigate'].strip())
|
|
assert 'fallback' not in formatted
|
|
assert 'Q: Question?' in formatted
|