61 lines
2.7 KiB
Python
61 lines
2.7 KiB
Python
"""LLM-judged benchmarks: hle, simple_qa. Need runner(judge=...) wired to a ModelAdapter."""
|
|
|
|
from ..recipe import EvalRecipe, JudgeConfig, register_eval
|
|
|
|
_HLE_PROMPT = (
|
|
'Judge whether the following [response] to [question] is correct or not based '
|
|
'on the precise and unambiguous [correct_answer] below.\n\n'
|
|
'[question]: {question}\n\n[response]: {prediction}\n\n'
|
|
'[correct_answer]: {target}\n\n'
|
|
'Focus only on whether the answers match. In one or two sentences explain, then '
|
|
"write your final line as 'GRADE: C' for correct or 'GRADE: I' for incorrect."
|
|
)
|
|
|
|
_SIMPLE_QA_PROMPT = (
|
|
'Your job is to look at a question, a gold target, and a predicted answer, and then '
|
|
'assign a grade of either ["CORRECT", "INCORRECT", "NOT_ATTEMPTED"].\n'
|
|
'Judge by: a predicted answer is CORRECT iff it fully contains the important '
|
|
'information in the gold target and contains no contradicting information; hedged but '
|
|
'complete answers are CORRECT; answers missing the key information without '
|
|
'contradiction are NOT_ATTEMPTED. For numeric gold targets the prediction must be '
|
|
'correct to the last significant figure of the gold answer.\n\n'
|
|
'Question: {question}\nGold target: {target}\nPredicted answer: {prediction}\n\n'
|
|
'Grade as one of:\nA: CORRECT\nB: INCORRECT\nC: NOT_ATTEMPTED\n\n'
|
|
'Just return the letter "A", "B", or "C" with no text around it.'
|
|
)
|
|
|
|
|
|
@register_eval('hle')
|
|
def hle():
|
|
return EvalRecipe(
|
|
name='hle',
|
|
extract='identity',
|
|
scorers={'acc': {'name': 'llm_judge', 'prompt_template': _HLE_PROMPT,
|
|
'labels': {'C': {'acc': 1.0}, 'I': {'acc': 0.0}}, 'primary': 'acc'}},
|
|
judge=JudgeConfig(model='judge'),
|
|
description="HLE; official GRADE: C/I LLM judge.",
|
|
)
|
|
|
|
|
|
@register_eval('simple_qa')
|
|
def simple_qa():
|
|
return EvalRecipe(
|
|
name='simple_qa',
|
|
extract='identity',
|
|
scorers={
|
|
'is_correct': {
|
|
'name': 'llm_judge',
|
|
'prompt_template': _SIMPLE_QA_PROMPT,
|
|
# official grading: match A|B|C, default to C (NOT_ATTEMPTED)
|
|
'labels': {'A': {'is_correct': 1.0, 'is_incorrect': 0.0, 'is_not_attempted': 0.0},
|
|
'B': {'is_correct': 0.0, 'is_incorrect': 1.0, 'is_not_attempted': 0.0},
|
|
'C': {'is_correct': 0.0, 'is_incorrect': 0.0, 'is_not_attempted': 1.0}},
|
|
'default_label': 'C',
|
|
'primary': 'is_correct',
|
|
},
|
|
},
|
|
aggregators={'is_correct': 'simpleqa_official'},
|
|
judge=JudgeConfig(model='judge'),
|
|
description='SimpleQA; official A/B/C judge, NOT_ATTEMPTED fallback + derived metrics.',
|
|
)
|