370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
60 lines
2.7 KiB
Python
60 lines
2.7 KiB
Python
"""LLM-judged benchmarks: hle, simple_qa. Need runner(judge=...) wired to a ModelAdapter."""
|
|
|
|
from pathlib import Path
|
|
|
|
from ..recipe import EvalRecipe, JudgeConfig, register_eval
|
|
|
|
# OFFICIAL OpenAI simple-evals grader prompt, verbatim (MIT), loaded from the
|
|
# vendored official source; {prediction} is the only rename (official: predicted_answer)
|
|
_SIMPLE_QA_PROMPT = Path(__file__).with_name('_simpleqa_grader.txt').read_text(encoding='utf-8')
|
|
|
|
_HLE_PROMPT = (
|
|
# es hle_adapter JUDGE_PROMPT, verbatim (only placeholder names renamed)
|
|
'Judge whether the following [response] to [question] is correct or not based '
|
|
'on the precise and unambiguous [correct_answer] below.\n\n'
|
|
'[question]: {question}\n\n[response]: {prediction}\n\n'
|
|
'[correct_answer]: {target}\n\n'
|
|
'Your judgment must focus only on if there are meaningful differences between '
|
|
'[correct_answer] and the [response]. Do not comment on any background to the '
|
|
'problem, do not attempt to solve the problem, do not argue for any answer '
|
|
'different than [correct_answer], focus only on whether the answers match. '
|
|
'Explain why the [response] is correct or incorrect based on [correct_answer] '
|
|
'in one or two sentences. Finally, write your answer in the format '
|
|
"'GRADE: C' for correct answer or 'GRADE: I' for incorrect answer.\n"
|
|
)
|
|
|
|
@register_eval('hle')
|
|
def hle():
|
|
return EvalRecipe(
|
|
name='hle',
|
|
extract='identity',
|
|
scorers={'acc': {'name': 'llm_judge', 'prompt_template': _HLE_PROMPT,
|
|
'label_pattern': r'GRADE:\s*([CI])',
|
|
'labels': {'C': {'acc': 1.0}, 'I': {'acc': 0.0}}, 'primary': 'acc'}},
|
|
judge=JudgeConfig(model='judge'),
|
|
description="HLE; official GRADE: C/I LLM judge.",
|
|
)
|
|
|
|
|
|
@register_eval('simple_qa')
|
|
def simple_qa():
|
|
return EvalRecipe(
|
|
name='simple_qa',
|
|
extract='identity',
|
|
scorers={
|
|
'is_correct': {
|
|
'name': 'llm_judge',
|
|
'prompt_template': _SIMPLE_QA_PROMPT,
|
|
# official grading: match A|B|C, default to C (NOT_ATTEMPTED)
|
|
'labels': {'A': {'is_correct': 1.0, 'is_incorrect': 0.0, 'is_not_attempted': 0.0},
|
|
'B': {'is_correct': 0.0, 'is_incorrect': 1.0, 'is_not_attempted': 0.0},
|
|
'C': {'is_correct': 0.0, 'is_incorrect': 0.0, 'is_not_attempted': 1.0}},
|
|
'default_label': 'C',
|
|
'primary': 'is_correct',
|
|
},
|
|
},
|
|
aggregators={'is_correct': 'simpleqa_official'},
|
|
judge=JudgeConfig(model='judge'),
|
|
description='SimpleQA; official A/B/C judge, NOT_ATTEMPTED fallback + derived metrics.',
|
|
)
|