Align with local evalscope run.py (full read-through): (1) shuffle+seed42 sample selection identical to es _shuffle_in_place -- same exact sample subset both frameworks; (2) official MATH_PROMPT_TEMPLATE CoT boxed suffix on 7 math benches via DatasetSpec.prompt_suffix; (3) per-bench gen_config (temp/max_tokens from dpv4 yaml) on DatasetSpec, default-merged into gen_kwargs

evalharness/data/datasets/aime24.py

@@ -13,6 +13,8 @@ from ..spec import DatasetSpec
         name='aime24',
         source='HuggingFaceH4/aime_2024',  # https://huggingface.co/datasets/HuggingFaceH4/aime_2024
         split='train',  # the dataset ships a single split
+        gen_config={'temperature': 1.0, 'max_tokens': 32768},
+        prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
         task_type='math',
         tags=['math', 'competition'],
         description='AIME 2024, 30 problems (integer answers 000-999).',

evalharness/data/datasets/aime25.py

@@ -10,6 +10,8 @@ from ..spec import DatasetSpec
         name='aime25',
         source='yentinglin/aime_2025',  # https://huggingface.co/datasets/yentinglin/aime_2025
         split='train',
+        gen_config={'temperature': 1.0, 'max_tokens': 32768},
+        prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
         task_type='math',
         tags=['math', 'competition'],
         description='AIME 2025, 30 problems (integer answers 000-999).',

evalharness/data/datasets/aime26.py

@@ -10,6 +10,8 @@ from ..spec import DatasetSpec
         name='aime26',
         source='MathArena/aime_2026',  # curated by MathArena (HuggingFace)
         split='train',  # the dataset ships a single split
+        gen_config={'temperature': 1.0, 'max_tokens': 32768},
+        prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
         task_type='math',
         tags=['math', 'competition'],
         description='AIME 2026, 30 problems (integer answers 000-999).',

evalharness/data/datasets/bfcl_v3.py

@@ -19,6 +19,7 @@ from ..spec import DatasetSpec
         name='bfcl_v3',
         source='llamastack/bfcl_v3',  # HF conversion of the official GitHub data
         split='train',  # the conversion ships a single split
+        gen_config={'temperature': 0.0, 'max_tokens': 4096},
         task_type='fc',
         tags=['function_calling', 'tool_use'],
         description='BFCL v3 function calling (official content, HF conversion).',

evalharness/data/datasets/bigcodebench.py

@@ -10,6 +10,7 @@ from ..spec import DatasetSpec
         name='bigcodebench',
         source='bigcode/bigcodebench',  # official: https://huggingface.co/datasets/bigcode/bigcodebench
         split='v0.1.4',  # BigCodeBench versions are published as splits
+        gen_config={'temperature': 0.0, 'max_tokens': 32768},
         task_type='coding',
         tags=['code'],
         description='BigCodeBench: practical library-level function synthesis (official).',

evalharness/data/datasets/competition_math.py

@@ -29,6 +29,8 @@ def _extract_boxed(text: str) -> str:
         split='test',
         few_shot_split='train',
         few_shot_num=4,
+        gen_config={'temperature': 0.0, 'max_tokens': 32768},
+        prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
         task_type='math',
         tags=['math'],
         description='MATH competition problems (Hendrycks). Target = \\boxed answer.',

evalharness/data/datasets/drop.py

@@ -12,6 +12,7 @@ from ..spec import DatasetSpec
         split='validation',
         few_shot_split='train',
         few_shot_num=3,
+        gen_config={'temperature': 0.0, 'max_tokens': 32768},
         task_type='qa',
         tags=['reading_comprehension'],
         description='DROP reading comprehension; target = answer spans list.',

evalharness/data/datasets/general_fc.py

@@ -12,6 +12,7 @@ from ..spec import DatasetSpec
         name='general_fc',
         source='evalscope/GeneralFunctionCall-Test',  # evalscope-native release (ModelScope)
         split='test',
+        gen_config={'temperature': 0.0, 'max_tokens': 4096},
         task_type='fc',
         tags=['function_calling'],
         description='Minimal function-calling test set (evalscope-native).',

evalharness/data/datasets/gpqa_diamond.py

@@ -19,6 +19,7 @@ from ..spec import DatasetSpec
         name='gpqa_diamond',
         source='nmayorga7/gpqa_diamond',  # HF CSV export of the gated official
         split='train',
+        gen_config={'temperature': 1.0, 'max_tokens': 8192},
         task_type='mcq',
         tags=['knowledge', 'science'],
         description='GPQA diamond split, graduate-level science MCQ (official content).',

evalharness/data/datasets/gsm8k.py

@@ -17,6 +17,8 @@ from ..spec import DatasetSpec
         split='test',
         few_shot_split='train',
         few_shot_num=4,
+        gen_config={'temperature': 0.0, 'max_tokens': 32768},
+        prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
         task_type='math',
         tags=['math', 'cot'],
         description='Grade school math word problems (OpenAI, official).',

evalharness/data/datasets/hle.py

@@ -14,6 +14,7 @@ from ..spec import DatasetSpec
         name='hle',
         source='cais/hle',  # ModelScope mirror of the gated HF original
         split='test',
+        gen_config={'temperature': 0.0, 'max_tokens': 32768},
         task_type='qa',
         tags=['knowledge', 'frontier'],
         description="Humanity's Last Exam. Some samples carry an image field (multimodal).",

evalharness/data/datasets/hmmt26.py

@@ -10,6 +10,8 @@ from ..spec import DatasetSpec
         name='hmmt26',
         source='MathArena/hmmt_feb_2026',  # curated by MathArena (HuggingFace)
         split='train',  # the dataset ships a single split
+        gen_config={'temperature': 1.0, 'max_tokens': 32768},
+        prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
         task_type='math',
         tags=['math', 'competition'],
         description='HMMT February 2026 (community-curated, no official upstream).',

evalharness/data/datasets/humaneval.py

@@ -10,6 +10,7 @@ from ..spec import DatasetSpec
         name='humaneval',
         source='openai/openai_humaneval',  # official: https://huggingface.co/datasets/openai/openai_humaneval
         split='test',
+        gen_config={'temperature': 1.0, 'max_tokens': 32768},
         task_type='coding',
         tags=['code'],
         description='OpenAI HumanEval function synthesis (official). Tests in metadata for sandbox.',

evalharness/data/datasets/imo_answerbench.py

@@ -10,6 +10,8 @@ from ..spec import DatasetSpec
         name='imo_answerbench',
         source='OpenEvals/IMO-AnswerBench',  # HF OpenEvals mirror of the community curation
         split='train',  # the dataset ships a single split
+        gen_config={'temperature': 1.0, 'max_tokens': 32768},
+        prompt_suffix="\nPlease reason step by step, and put your final answer within \\boxed{}",
         task_type='math',
         tags=['math', 'competition', 'imo'],
         description='IMO-level answer bench (community-curated, no official upstream).',

evalharness/data/datasets/live_code_bench.py

@@ -12,6 +12,7 @@ from ..spec import DatasetSpec
         # livecodebench/code_generation_lite is script-based (unloadable by datasets>=5)
         subset='release_latest',  # or release_v1..v6
         split='test',
+        gen_config={'temperature': 1.0, 'max_tokens': 32768},
         task_type='coding',
         tags=['code'],
         description='LiveCodeBench (lite) contest problems; version tags via DatasetSpec.version.',

evalharness/data/datasets/swe_bench_verified.py

@@ -10,6 +10,7 @@ from ..spec import DatasetSpec
         name='swe_bench_verified',
         source='princeton-nlp/SWE-bench_Verified',  # official: https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified
         split='test',
+        gen_config={'temperature': 0.0, 'max_tokens': 32768},
         task_type='agent',
         tags=['code', 'agent', 'swe'],
         requires=['docker'],

evalharness/data/datasets/tau2_bench.py

@@ -18,6 +18,7 @@ _DOMAINS = ('airline', 'retail', 'telecom', 'mock')
         source='HuggingFaceH4/tau2-bench-data',  # HF mirror of the official GitHub data
         subset='airline',  # or retail / telecom / mock; override with --subset
         split='test',
+        gen_config={'temperature': 0.0, 'max_tokens': 16384},
         task_type='agent',
         tags=['agent', 'tool_use', 'dialog'],
         description='tau2-bench agent-tool-dialog tasks (official content, HF mirror).',

evalharness/data/spec.py

@@ -38,3 +38,7 @@ class DatasetSpec:
     params: dict = field(default_factory=dict)  # extra load params, part of cache key
     few_shot_split: Optional[str] = None       # e.g. 'dev' (paper-faithful exemplars)
     few_shot_num: int = 0                      # paper default shots (mmlu=5, bbh=3, ...)
+    gen_config: dict = field(default_factory=dict)  # per-bench generation params
+    prompt_suffix: str = ''   # appended to the question (e.g. boxed{} CoT directive)
+                                             # (temperature/max_tokens/top_p), consumed
+                                             # by run_eval unless overridden

evalharness/model/runner.py

@@ -38,6 +38,7 @@ async def generate_predictions(
     max_input_tokens: int = 0,
     tokenizer_path: str = '',
     attach_context_keys: tuple = ('passage', 'context'),
+    dataset_spec=None,
     limit_per_task: Optional[int] = None,
     checkpoint: Union[bool, str] = False,
     dataset_name: str = 'adhoc',
@@ -117,6 +118,9 @@ async def generate_predictions(
                 question = (f'{question}\n\n'
                             'End your reply with the final answer on its own last line '
                             'in the form "Answer: <answer>".')
+        ds_spec = dataset_spec
+        if ds_spec is not None and getattr(ds_spec, 'prompt_suffix', ''):
+            question = question + ds_spec.prompt_suffix
         parts.append(question)
         text = '\n\n'.join(parts)
         if max_input_tokens:
@@ -251,9 +255,18 @@ async def generate_predictions(
 
 
 def _apply_limits(samples: List[Sample], total: Optional[int],
-                  per_task: Optional[int], dataset=None) -> List[Sample]:
+                  per_task: Optional[int], dataset=None,
+                  shuffle: bool = True, seed: int = 42) -> List[Sample]:
     """total: cap the WHOLE run (ours semantics). per_task: cap each subset/
-    category (evalscope's --limit semantics) -- first N per group_key."""
+    category (evalscope's --limit semantics) -- first N per group_key.
+
+    shuffle+seed mirror evalscope run.py: dataset_args.shuffle=True with
+    --seed 42 -> random.Random(seed).shuffle BEFORE limiting, so both
+    frameworks evaluate the IDENTICAL sample subset."""
+    if shuffle:
+        import random
+
+        random.Random(seed).shuffle(samples)
     if per_task:
         seen: Dict[str, int] = {}
         out = []
@@ -374,9 +387,11 @@ async def run_eval(
     try:
         preds, _usages, usage = await generate_predictions(
             adapter, samples, concurrency, progress=progress,
-            gen_kwargs=gen_kwargs, env_factory=env_factory,
+            gen_kwargs={**(getattr(spec, 'gen_config', {}) or {}), **(gen_kwargs or {})},
+ env_factory=env_factory,
             system=system, max_turns=max_turns, max_input_chars=max_input_chars,
             max_input_tokens=max_input_tokens,
+            dataset_spec=spec,
             limit_per_task=limit_per_task,
             checkpoint=checkpoint,
             dataset_name=name,