370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
38 lines
1.2 KiB
Python
38 lines
1.2 KiB
Python
"""GSM8K dataset plugin (official source: openai/gsm8k).
|
|
|
|
Offline demo: examples/data/gsm8k_main_test.jsonl ships a tiny subset, e.g.
|
|
``evalharness data fetch gsm8k --source examples/data/gsm8k_main_test.jsonl``
|
|
"""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='gsm8k',
|
|
source='openai/gsm8k', # official: https://huggingface.co/datasets/openai/gsm8k
|
|
subset='main',
|
|
split='test',
|
|
few_shot_split='train',
|
|
few_shot_num=4,
|
|
gen_config={'temperature': 0.0, 'max_tokens': 32768},
|
|
prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}.",
|
|
task_type='math',
|
|
tags=['math', 'cot'],
|
|
description='Grade school math word problems (OpenAI, official).',
|
|
)
|
|
)
|
|
def gsm8k():
|
|
def to_sample(record: dict) -> Sample:
|
|
parts = record['answer'].split('####')
|
|
target = parts.pop().strip()
|
|
return Sample(
|
|
input=record['question'],
|
|
target=target,
|
|
metadata={'reasoning': '####'.join(parts).strip()},
|
|
)
|
|
|
|
return to_sample
|