"""LLM-judged benchmarks: hle, simple_qa. Need runner(judge=...) wired to a ModelAdapter."""

from pathlib import Path

from ..recipe import EvalRecipe, JudgeConfig, register_eval

# OFFICIAL OpenAI simple-evals grader prompt, verbatim (MIT), loaded from the
# vendored official source; {prediction} is the only rename (official: predicted_answer)
_SIMPLE_QA_PROMPT = Path(__file__).with_name('_simpleqa_grader.txt').read_text(encoding='utf-8')

_HLE_PROMPT = (
    # es hle_adapter JUDGE_PROMPT, verbatim (only placeholder names renamed)
    'Judge whether the following [response] to [question] is correct or not based '
    'on the precise and unambiguous [correct_answer] below.\n\n'
    '[question]: {question}\n\n[response]: {prediction}\n\n'
    '[correct_answer]: {target}\n\n'
    'Your judgment must focus only on if there are meaningful differences between '
    '[correct_answer] and the [response]. Do not comment on any background to the '
    'problem, do not attempt to solve the problem, do not argue for any answer '
    'different than [correct_answer], focus only on whether the answers match. '
    'Explain why the [response] is correct or incorrect based on [correct_answer] '
    'in one or two sentences. Finally, write your answer in the format '
    "'GRADE: C' for correct answer or 'GRADE: I' for incorrect answer.\n"
)

@register_eval('hle')
def hle():
    return EvalRecipe(
        name='hle',
        extract='identity',
        scorers={'acc': {'name': 'llm_judge', 'prompt_template': _HLE_PROMPT,
                         'label_pattern': r'GRADE:\s*([CI])',
                         'labels': {'C': {'acc': 1.0}, 'I': {'acc': 0.0}}, 'primary': 'acc'}},
        judge=JudgeConfig(model='judge'),
        description="HLE; official GRADE: C/I LLM judge.",
    )


@register_eval('simple_qa')
def simple_qa():
    return EvalRecipe(
        name='simple_qa',
        extract='identity',
        scorers={
            'is_correct': {
                'name': 'llm_judge',
                'prompt_template': _SIMPLE_QA_PROMPT,
                # official grading: match A|B|C, default to C (NOT_ATTEMPTED)
                'labels': {'A': {'is_correct': 1.0, 'is_incorrect': 0.0, 'is_not_attempted': 0.0},
                           'B': {'is_correct': 0.0, 'is_incorrect': 1.0, 'is_not_attempted': 0.0},
                           'C': {'is_correct': 0.0, 'is_incorrect': 0.0, 'is_not_attempted': 1.0}},
                'default_label': 'C',
                'primary': 'is_correct',
            },
        },
        aggregators={'is_correct': 'simpleqa_official'},
        judge=JudgeConfig(model='judge'),
        description='SimpleQA; official A/B/C judge, NOT_ATTEMPTED fallback + derived metrics.',
    )