370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
46 lines
1.8 KiB
Python
46 lines
1.8 KiB
Python
"""Dataset metadata (DatasetSpec) and declarative field mapping (FieldSpec)."""
|
|
|
|
from dataclasses import dataclass, field
|
|
from typing import List, Optional
|
|
|
|
|
|
@dataclass
|
|
class FieldSpec:
|
|
"""Declarative mapping: raw record field name -> Sample field name.
|
|
|
|
Use this when the raw records are already well-shaped; no custom
|
|
``record_to_sample`` function is needed then.
|
|
"""
|
|
|
|
input: str = 'input'
|
|
target: str = 'target'
|
|
choices: str = 'choices'
|
|
id: Optional[str] = None
|
|
metadata: List[str] = field(default_factory=list)
|
|
|
|
|
|
@dataclass
|
|
class DatasetSpec:
|
|
"""Everything the framework needs to know about a dataset *without*
|
|
loading it. Drives the cache key, the CLI listing, and (later) the
|
|
deployment-time dependency resolution via ``requires``.
|
|
"""
|
|
|
|
name: str
|
|
source: str # hub id ('AI-ModelScope/gsm8k') or local path
|
|
split: str = 'test'
|
|
subset: str = 'default'
|
|
version: Optional[str] = None
|
|
task_type: str = 'qa' # qa | mcq | math | coding | agent | vqa | fc
|
|
tags: List[str] = field(default_factory=list)
|
|
requires: List[str] = field(default_factory=list) # e.g. ['docker']
|
|
description: str = ''
|
|
params: dict = field(default_factory=dict) # extra load params, part of cache key
|
|
few_shot_split: Optional[str] = None # e.g. 'dev' (paper-faithful exemplars)
|
|
few_shot_num: int = 0 # paper default shots (mmlu=5, bbh=3, ...)
|
|
gen_config: dict = field(default_factory=dict) # per-bench generation params
|
|
prompt_suffix: str = ''
|
|
prompt_style: str = '' # ''=default; 'cot_letter'=CoT then ANSWER:X # appended to the question (e.g. boxed{} CoT directive)
|
|
# (temperature/max_tokens/top_p), consumed
|
|
# by run_eval unless overridden
|