370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
63 lines
1.9 KiB
Python
63 lines
1.9 KiB
Python
"""QA benchmarks with text-compare scoring: bbh, drop, trivia_qa, openai_mrcr."""
|
|
|
|
import re
|
|
|
|
from ..recipe import EvalRecipe, register_eval
|
|
|
|
|
|
def _bbh_extract(raw, sample):
|
|
"""Dispatch by TARGET FORMAT (robust): '(B)' -> MC letter, else free-form phrase.
|
|
|
|
The official BBH answers are either '(A)'-style or short free-form text
|
|
(True/False, numbers, sorted lists); dispatching on the target's shape
|
|
needs no per-subtask table and cannot drift from the data.
|
|
"""
|
|
from ..extractor import answer_phrase, mcq_letter
|
|
|
|
target = str(sample.target or '').strip()
|
|
if re.fullmatch(r'\([A-Z]\)', target):
|
|
val, ok, why = mcq_letter(raw, sample)
|
|
return (f'({val})' if ok else val), ok, why
|
|
return answer_phrase(raw, sample)
|
|
|
|
|
|
@register_eval('bbh')
|
|
def bbh():
|
|
return EvalRecipe(
|
|
name='bbh',
|
|
extract=_bbh_extract,
|
|
scorers={'acc': {'name': 'exact', 'mode': 'math'}},
|
|
description='BBH; MC subtasks -> letter, free-form -> answer phrase.',
|
|
)
|
|
|
|
|
|
@register_eval('drop')
|
|
def drop():
|
|
return EvalRecipe(
|
|
name='drop',
|
|
extract=['answer_spans', 'first_line'],
|
|
scorers={'em': 'em_f1', 'f1': 'em_f1'},
|
|
description='DROP; every Answer: line = one span; official Hungarian-align EM/F1.',
|
|
)
|
|
|
|
|
|
@register_eval('trivia_qa')
|
|
def trivia_qa():
|
|
return EvalRecipe(
|
|
name='trivia_qa',
|
|
extract=['answer_phrase', 'first_line'],
|
|
scorers={'em': 'alias_match'},
|
|
description='TriviaQA; any alias counts.',
|
|
)
|
|
|
|
|
|
@register_eval('openai_mrcr')
|
|
def openai_mrcr():
|
|
return EvalRecipe(
|
|
name='openai_mrcr',
|
|
extract=['quoted_list', 'identity'],
|
|
scorers={'mrcr_score': 'exact'}, # placeholder scorer; official prefix grading lands with long-context skill
|
|
aggregators={'mrcr_score': 'binned_avg'},
|
|
description='MRCR; quoted-marker extraction, binned by context length.',
|
|
)
|