Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Align with local evalscope run.py (full read-through): (1) shuffle+seed42 sample selection identical to es _shuffle_in_place -- same exact sample subset both frameworks; (2) official MATH_PROMPT_TEMPLATE CoT boxed suffix on 7 math benches via DatasetSpec.prompt_suffix; (3) per-bench gen_config (temp/max_tokens from dpv4 yaml) on DatasetSpec, default-merged into gen_kwargs

Browse Source
This commit is contained in:
sora 2026-08-27 11:25:17 +00:00
parent 7ce04b8ac6
commit da989ea665
19 changed files with 46 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
evalharness/data/datasets/aime24.py
View File

@ -13,6 +13,8 @@ from ..spec import DatasetSpec
name='aime24',
source='HuggingFaceH4/aime_2024', # https://huggingface.co/datasets/HuggingFaceH4/aime_2024
split='train', # the dataset ships a single split
gen_config={'temperature': 1.0, 'max_tokens': 32768},
prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
task_type='math',
tags=['math', 'competition'],
description='AIME 2024, 30 problems (integer answers 000-999).',

2
evalharness/data/datasets/aime25.py
View File

@ -10,6 +10,8 @@ from ..spec import DatasetSpec
name='aime25',
source='yentinglin/aime_2025', # https://huggingface.co/datasets/yentinglin/aime_2025
split='train',
gen_config={'temperature': 1.0, 'max_tokens': 32768},
prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
task_type='math',
tags=['math', 'competition'],
description='AIME 2025, 30 problems (integer answers 000-999).',

2
evalharness/data/datasets/aime26.py
View File

@ -10,6 +10,8 @@ from ..spec import DatasetSpec
name='aime26',
source='MathArena/aime_2026', # curated by MathArena (HuggingFace)
split='train', # the dataset ships a single split
gen_config={'temperature': 1.0, 'max_tokens': 32768},
prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
task_type='math',
tags=['math', 'competition'],
description='AIME 2026, 30 problems (integer answers 000-999).',

1
evalharness/data/datasets/bfcl_v3.py
View File

@ -19,6 +19,7 @@ from ..spec import DatasetSpec
name='bfcl_v3',
source='llamastack/bfcl_v3', # HF conversion of the official GitHub data
split='train', # the conversion ships a single split
gen_config={'temperature': 0.0, 'max_tokens': 4096},
task_type='fc',
tags=['function_calling', 'tool_use'],
description='BFCL v3 function calling (official content, HF conversion).',

1
evalharness/data/datasets/bigcodebench.py
View File

@ -10,6 +10,7 @@ from ..spec import DatasetSpec
name='bigcodebench',
source='bigcode/bigcodebench', # official: https://huggingface.co/datasets/bigcode/bigcodebench
split='v0.1.4', # BigCodeBench versions are published as splits
gen_config={'temperature': 0.0, 'max_tokens': 32768},
task_type='coding',
tags=['code'],
description='BigCodeBench: practical library-level function synthesis (official).',

2
evalharness/data/datasets/competition_math.py
View File

@ -29,6 +29,8 @@ def _extract_boxed(text: str) -> str:
split='test',
few_shot_split='train',
few_shot_num=4,
gen_config={'temperature': 0.0, 'max_tokens': 32768},
prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
task_type='math',
tags=['math'],
description='MATH competition problems (Hendrycks). Target = \\boxed answer.',

1
evalharness/data/datasets/drop.py
View File

@ -12,6 +12,7 @@ from ..spec import DatasetSpec
split='validation',
few_shot_split='train',
few_shot_num=3,
gen_config={'temperature': 0.0, 'max_tokens': 32768},
task_type='qa',
tags=['reading_comprehension'],
description='DROP reading comprehension; target = answer spans list.',

1
evalharness/data/datasets/general_fc.py
View File

@ -12,6 +12,7 @@ from ..spec import DatasetSpec
name='general_fc',
source='evalscope/GeneralFunctionCall-Test', # evalscope-native release (ModelScope)
split='test',
gen_config={'temperature': 0.0, 'max_tokens': 4096},
task_type='fc',
tags=['function_calling'],
description='Minimal function-calling test set (evalscope-native).',

1
evalharness/data/datasets/gpqa_diamond.py
View File

@ -19,6 +19,7 @@ from ..spec import DatasetSpec
name='gpqa_diamond',
source='nmayorga7/gpqa_diamond', # HF CSV export of the gated official
split='train',
gen_config={'temperature': 1.0, 'max_tokens': 8192},
task_type='mcq',
tags=['knowledge', 'science'],
description='GPQA diamond split, graduate-level science MCQ (official content).',

2
evalharness/data/datasets/gsm8k.py
View File

@ -17,6 +17,8 @@ from ..spec import DatasetSpec
split='test',
few_shot_split='train',
few_shot_num=4,
gen_config={'temperature': 0.0, 'max_tokens': 32768},
prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
task_type='math',
tags=['math', 'cot'],
description='Grade school math word problems (OpenAI, official).',

1
evalharness/data/datasets/hle.py
View File

@ -14,6 +14,7 @@ from ..spec import DatasetSpec
name='hle',
source='cais/hle', # ModelScope mirror of the gated HF original
split='test',
gen_config={'temperature': 0.0, 'max_tokens': 32768},
task_type='qa',
tags=['knowledge', 'frontier'],
description="Humanity's Last Exam. Some samples carry an image field (multimodal).",

2
evalharness/data/datasets/hmmt26.py
View File

@ -10,6 +10,8 @@ from ..spec import DatasetSpec
name='hmmt26',
source='MathArena/hmmt_feb_2026', # curated by MathArena (HuggingFace)
split='train', # the dataset ships a single split
gen_config={'temperature': 1.0, 'max_tokens': 32768},
prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
task_type='math',
tags=['math', 'competition'],
description='HMMT February 2026 (community-curated, no official upstream).',

1
evalharness/data/datasets/humaneval.py
View File

@ -10,6 +10,7 @@ from ..spec import DatasetSpec
name='humaneval',
source='openai/openai_humaneval', # official: https://huggingface.co/datasets/openai/openai_humaneval
split='test',
gen_config={'temperature': 1.0, 'max_tokens': 32768},
task_type='coding',
tags=['code'],
description='OpenAI HumanEval function synthesis (official). Tests in metadata for sandbox.',

2
evalharness/data/datasets/imo_answerbench.py
View File

@ -10,6 +10,8 @@ from ..spec import DatasetSpec
name='imo_answerbench',
source='OpenEvals/IMO-AnswerBench', # HF OpenEvals mirror of the community curation
split='train', # the dataset ships a single split
gen_config={'temperature': 1.0, 'max_tokens': 32768},
prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
task_type='math',
tags=['math', 'competition', 'imo'],
description='IMO-level answer bench (community-curated, no official upstream).',

1
evalharness/data/datasets/live_code_bench.py
View File

@ -12,6 +12,7 @@ from ..spec import DatasetSpec
# livecodebench/code_generation_lite is script-based (unloadable by datasets>=5)
subset='release_latest', # or release_v1..v6
split='test',
gen_config={'temperature': 1.0, 'max_tokens': 32768},
task_type='coding',
tags=['code'],
description='LiveCodeBench (lite) contest problems; version tags via DatasetSpec.version.',

1
evalharness/data/datasets/swe_bench_verified.py
View File

@ -10,6 +10,7 @@ from ..spec import DatasetSpec
name='swe_bench_verified',
source='princeton-nlp/SWE-bench_Verified', # official: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified
split='test',
gen_config={'temperature': 0.0, 'max_tokens': 32768},
task_type='agent',
tags=['code', 'agent', 'swe'],
requires=['docker'],

1
evalharness/data/datasets/tau2_bench.py
View File

@ -18,6 +18,7 @@ _DOMAINS = ('airline', 'retail', 'telecom', 'mock')
source='HuggingFaceH4/tau2-bench-data', # HF mirror of the official GitHub data
subset='airline', # or retail / telecom / mock; override with --subset
split='test',
gen_config={'temperature': 0.0, 'max_tokens': 16384},
task_type='agent',
tags=['agent', 'tool_use', 'dialog'],
description='tau2-bench agent-tool-dialog tasks (official content, HF mirror).',

4
evalharness/data/spec.py
View File

@ -38,3 +38,7 @@ class DatasetSpec:
params: dict = field(default_factory=dict) # extra load params, part of cache key
few_shot_split: Optional[str] = None # e.g. 'dev' (paper-faithful exemplars)
few_shot_num: int = 0 # paper default shots (mmlu=5, bbh=3, ...)
gen_config: dict = field(default_factory=dict) # per-bench generation params
prompt_suffix: str = '' # appended to the question (e.g. boxed{} CoT directive)
# (temperature/max_tokens/top_p), consumed
# by run_eval unless overridden

21
evalharness/model/runner.py
View File

@ -38,6 +38,7 @@ async def generate_predictions(
max_input_tokens: int = 0,
tokenizer_path: str = '',
attach_context_keys: tuple = ('passage', 'context'),
dataset_spec=None,
limit_per_task: Optional[int] = None,
checkpoint: Union[bool, str] = False,
dataset_name: str = 'adhoc',
@ -117,6 +118,9 @@ async def generate_predictions(
question = (f'{question}\n\n'
'End your reply with the final answer on its own last line '
'in the form "Answer: <answer>".')
ds_spec = dataset_spec
if ds_spec is not None and getattr(ds_spec, 'prompt_suffix', ''):
question = question + ds_spec.prompt_suffix
parts.append(question)
text = '\n\n'.join(parts)
if max_input_tokens:
@ -251,9 +255,18 @@ async def generate_predictions(
def _apply_limits(samples: List[Sample], total: Optional[int],
per_task: Optional[int], dataset=None) -> List[Sample]:
per_task: Optional[int], dataset=None,
shuffle: bool = True, seed: int = 42) -> List[Sample]:
"""total: cap the WHOLE run (ours semantics). per_task: cap each subset/
category (evalscope's --limit semantics) -- first N per group_key."""
category (evalscope's --limit semantics) -- first N per group_key.
shuffle+seed mirror evalscope run.py: dataset_args.shuffle=True with
--seed 42 -> random.Random(seed).shuffle BEFORE limiting, so both
frameworks evaluate the IDENTICAL sample subset."""
if shuffle:
import random
random.Random(seed).shuffle(samples)
if per_task:
seen: Dict[str, int] = {}
out = []
@ -374,9 +387,11 @@ async def run_eval(
try:
preds, _usages, usage = await generate_predictions(
adapter, samples, concurrency, progress=progress,
gen_kwargs=gen_kwargs, env_factory=env_factory,
gen_kwargs={**(getattr(spec, 'gen_config', {}) or {}), **(gen_kwargs or {})},
env_factory=env_factory,
system=system, max_turns=max_turns, max_input_chars=max_input_chars,
max_input_tokens=max_input_tokens,
dataset_spec=spec,
limit_per_task=limit_per_task,
checkpoint=checkpoint,
dataset_name=name,