"""LLM-judged benchmarks: hle, simple_qa. Need runner(judge=...) wired to a ModelAdapter."""

from ..recipe import EvalRecipe, JudgeConfig, register_eval

_HLE_PROMPT = (
    'Judge whether the following [response] to [question] is correct or not based '
    'on the precise and unambiguous [correct_answer] below.\n\n'
    '[question]: {question}\n\n[response]: {prediction}\n\n'
    '[correct_answer]: {target}\n\n'
    'Focus only on whether the answers match. In one or two sentences explain, then '
    "write your final line as 'GRADE: C' for correct or 'GRADE: I' for incorrect."
)

_SIMPLE_QA_PROMPT = (
    'Your job is to look at a question, a gold target, and a predicted answer, and then '
    'assign a grade of either ["CORRECT", "INCORRECT", "NOT_ATTEMPTED"].\n'
    'Judge by: a predicted answer is CORRECT iff it fully contains the important '
    'information in the gold target and contains no contradicting information; hedged but '
    'complete answers are CORRECT; answers missing the key information without '
    'contradiction are NOT_ATTEMPTED. For numeric gold targets the prediction must be '
    'correct to the last significant figure of the gold answer.\n\n'
    'Question: {question}\nGold target: {target}\nPredicted answer: {prediction}\n\n'
    'Grade as one of:\nA: CORRECT\nB: INCORRECT\nC: NOT_ATTEMPTED\n\n'
    'Just return the letter "A", "B", or "C" with no text around it.'
)


@register_eval('hle')
def hle():
    return EvalRecipe(
        name='hle',
        extract='identity',
        scorers={'acc': {'name': 'llm_judge', 'prompt_template': _HLE_PROMPT,
                         'labels': {'C': {'acc': 1.0}, 'I': {'acc': 0.0}}, 'primary': 'acc'}},
        judge=JudgeConfig(model='judge'),
        description="HLE; official GRADE: C/I LLM judge.",
    )


@register_eval('simple_qa')
def simple_qa():
    return EvalRecipe(
        name='simple_qa',
        extract='identity',
        scorers={
            'is_correct': {
                'name': 'llm_judge',
                'prompt_template': _SIMPLE_QA_PROMPT,
                # official grading: match A|B|C, default to C (NOT_ATTEMPTED)
                'labels': {'A': {'is_correct': 1.0, 'is_incorrect': 0.0, 'is_not_attempted': 0.0},
                           'B': {'is_correct': 0.0, 'is_incorrect': 1.0, 'is_not_attempted': 0.0},
                           'C': {'is_correct': 0.0, 'is_incorrect': 0.0, 'is_not_attempted': 1.0}},
                'default_label': 'C',
                'primary': 'is_correct',
            },
        },
        aggregators={'is_correct': 'simpleqa_official'},
        judge=JudgeConfig(model='judge'),
        description='SimpleQA; official A/B/C judge, NOT_ATTEMPTED fallback + derived metrics.',
    )