"""MCQ benchmarks: mmlu, cmmlu, mmlu_pro, arc, hellaswag, winogrande,
gpqa_diamond, longbench_v2. All: extract a letter, compare to target letter."""

from ..recipe import EvalRecipe, register_eval


def _mcq(name: str, desc: str) -> EvalRecipe:
    return EvalRecipe(
        name=name,
        extract=['mcq_letter'],
        scorers={'acc': {'name': 'exact', 'mode': 'raw'}},  # letter == letter
        description=desc,
    )


@register_eval('mmlu')
def mmlu():
    return _mcq('mmlu', 'MMLU; letter extraction vs answer letter.')


@register_eval('cmmlu')
def cmmlu():
    return _mcq('cmmlu', 'CMMLU; letter extraction (supports 答案是X) vs answer letter.')


@register_eval('mmlu_pro')
def mmlu_pro():
    return _mcq('mmlu_pro', 'MMLU-Pro 10-option; letter vs letter.')


@register_eval('arc')
def arc():
    return _mcq('arc', 'AI2 ARC; letter vs answerKey.')


@register_eval('hellaswag')
def hellaswag():
    return _mcq('hellaswag', 'HellaSwag; letter vs label (acc_norm needs logprobs: model layer).')


@register_eval('winogrande')
def winogrande():
    return _mcq('winogrande', 'Winogrande; letter vs answer.')


@register_eval('gpqa_diamond')
def gpqa_diamond():
    return _mcq('gpqa_diamond', 'GPQA diamond; letter vs target (choices shuffled at eval time).')


@register_eval('longbench_v2')
def longbench_v2():
    return _mcq('longbench_v2', 'LongBench v2; letter vs answer.')