370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
30 lines
1022 B
Python
30 lines
1022 B
Python
"""HMMT Feb 2026. No official standalone release; MathArena community curation."""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='hmmt26',
|
|
source='MathArena/hmmt_feb_2026', # curated by MathArena (HuggingFace)
|
|
split='train', # the dataset ships a single split
|
|
gen_config={'temperature': 1.0, 'max_tokens': 8192},
|
|
prompt_style='imo_es', # es template: Problem: prefix + boxed suffix,
|
|
task_type='math',
|
|
tags=['math', 'competition'],
|
|
description='HMMT February 2026 (community-curated, no official upstream).',
|
|
params={'hub': 'hf_raw'},
|
|
)
|
|
)
|
|
def hmmt26():
|
|
def to_sample(record: dict) -> Sample:
|
|
return Sample(
|
|
input=record['problem'],
|
|
target=str(record['answer']).strip(),
|
|
metadata={'problem_idx': record.get('problem_idx'), 'problem_type': record.get('problem_type')},
|
|
)
|
|
|
|
return to_sample
|