53 lines
2.2 KiB
Python
53 lines
2.2 KiB
Python
"""LLM-judged benchmarks: hle, simple_qa. Need runner(judge=...) wired to a ModelAdapter."""
|
|
|
|
from pathlib import Path
|
|
|
|
from ..recipe import EvalRecipe, JudgeConfig, register_eval
|
|
|
|
# OFFICIAL OpenAI simple-evals grader prompt, verbatim (MIT), loaded from the
|
|
# vendored official source; {prediction} is the only rename (official: predicted_answer)
|
|
_SIMPLE_QA_PROMPT = Path(__file__).with_name('_simpleqa_grader.txt').read_text(encoding='utf-8')
|
|
|
|
_HLE_PROMPT = (
|
|
'Judge whether the following [response] to [question] is correct or not based '
|
|
'on the precise and unambiguous [correct_answer] below.\n\n'
|
|
'[question]: {question}\n\n[response]: {prediction}\n\n'
|
|
'[correct_answer]: {target}\n\n'
|
|
'Focus only on whether the answers match. In one or two sentences explain, then '
|
|
"write your final line as 'GRADE: C' for correct or 'GRADE: I' for incorrect."
|
|
)
|
|
|
|
@register_eval('hle')
|
|
def hle():
|
|
return EvalRecipe(
|
|
name='hle',
|
|
extract='identity',
|
|
scorers={'acc': {'name': 'llm_judge', 'prompt_template': _HLE_PROMPT,
|
|
'labels': {'C': {'acc': 1.0}, 'I': {'acc': 0.0}}, 'primary': 'acc'}},
|
|
judge=JudgeConfig(model='judge'),
|
|
description="HLE; official GRADE: C/I LLM judge.",
|
|
)
|
|
|
|
|
|
@register_eval('simple_qa')
|
|
def simple_qa():
|
|
return EvalRecipe(
|
|
name='simple_qa',
|
|
extract='identity',
|
|
scorers={
|
|
'is_correct': {
|
|
'name': 'llm_judge',
|
|
'prompt_template': _SIMPLE_QA_PROMPT,
|
|
# official grading: match A|B|C, default to C (NOT_ATTEMPTED)
|
|
'labels': {'A': {'is_correct': 1.0, 'is_incorrect': 0.0, 'is_not_attempted': 0.0},
|
|
'B': {'is_correct': 0.0, 'is_incorrect': 1.0, 'is_not_attempted': 0.0},
|
|
'C': {'is_correct': 0.0, 'is_incorrect': 0.0, 'is_not_attempted': 1.0}},
|
|
'default_label': 'C',
|
|
'primary': 'is_correct',
|
|
},
|
|
},
|
|
aggregators={'is_correct': 'simpleqa_official'},
|
|
judge=JudgeConfig(model='judge'),
|
|
description='SimpleQA; official A/B/C judge, NOT_ATTEMPTED fallback + derived metrics.',
|
|
)
|