From 1da665fec4b94bd7b03e7642055db73f2897f950 Mon Sep 17 00:00:00 2001 From: sora Date: Tue, 25 Aug 2026 05:18:23 +0000 Subject: [PATCH] Add --limit-per-task (per-subset cap, evalscope --limit semantics; composable with --limit as intersection); execution-bench parity verified: official human-eval core 5/5 == our docker sandbox on same completions --- evalharness/cli.py | 6 +++++- evalharness/model/runner.py | 32 ++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index 26ac634..36ae4f9 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -148,6 +148,7 @@ def _cmd_eval_run(args) -> int: report = asyncio.run(run_eval( ds, args.model, concurrency=args.concurrency, limit=args.limit, + limit_per_task=args.limit_per_task, judge_spec=args.judge, env=args.env)) else: from evalharness.eval import evaluate @@ -262,7 +263,10 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument('--judge', default='', help='judge model spec for llm_judge recipes') p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump") p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)') - p.add_argument('--limit', type=int, help='evaluate only the first N samples') + p.add_argument('--limit', type=int, help='evaluate only the first N samples total') + p.add_argument('--limit-per-task', type=int, + help='first N samples PER subset/category (evalscope --limit semantics); ' + 'composable with --limit (intersection)') p.add_argument('--out', help='save the EvalReport json here (single dataset)') p.add_argument('--out-dir', help='save reports/.json + viz/.txt + summary.md ' 'here (multi-dataset runs)') diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index a098710..946d52b 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -35,6 +35,7 @@ async def generate_predictions( max_turns: int = 8, max_input_chars: int = 0, attach_context_keys: tuple = ('passage', 'context'), + limit_per_task: Optional[int] = None, few_shot_num: int = 0, few_shot_samples: Optional[List[Sample]] = None, few_shot_text: Optional[str] = None, @@ -57,7 +58,7 @@ async def generate_predictions( if few_shot_text: parts.append(few_shot_text.strip()) # official exemplars, verbatim elif few_shot_num and few_shot_samples: - letters_fs = 'ABCDEFGH' + letters_fs = 'ABCDEFGHIJ' for fs in few_shot_samples[:few_shot_num]: line = f'Question: {fs.input_text}' if fs.choices: @@ -73,7 +74,7 @@ async def generate_predictions( if sample.choices: if prompt_style in ('strict_letter', 'auto'): # evalscope/OpenAI-style contract: reply ONLY 'ANSWER: X' - letters = 'ABCDEFGH' + letters = 'ABCDEFGHIJ' opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(sample.choices) if i < len(letters)) question = (f'Answer the following multiple choice question. The entire ' @@ -81,7 +82,7 @@ async def generate_predictions( f"'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of " f'{letters[:len(sample.choices)]}.\n\n{question}\n\n{opts}') else: - letters = 'ABCDEFGH' + letters = 'ABCDEFGHIJ' opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(sample.choices) if i < len(letters)) question = (f'{question}\n\n{opts}\n\n' @@ -141,12 +142,33 @@ async def generate_predictions( _progress(progress, done_count, len(samples), t0, total_usage) return {'raw': text, 'usage': out.usage.model_dump()} - work = samples[:limit] if limit else samples + work = _apply_limits(samples, limit, limit_per_task) preds = list(await asyncio.gather(*(one(s) for s in work))) usages = [p.get('usage', {}) for p in preds] return preds, usages, total_usage +def _apply_limits(samples: List[Sample], total: Optional[int], + per_task: Optional[int], dataset=None) -> List[Sample]: + """total: cap the WHOLE run (ours semantics). per_task: cap each subset/ + category (evalscope's --limit semantics) -- first N per group_key.""" + if per_task: + seen: Dict[str, int] = {} + out = [] + for s in samples: + key = str((s.metadata or {}).get('category') + or (s.metadata or {}).get('subject') + or (s.metadata or {}).get('test_category') + or getattr(getattr(dataset, 'spec', None), 'subset', 'default')) + if seen.get(key, 0) < per_task: + seen[key] = seen.get(key, 0) + 1 + out.append(s) + samples = out + if total: + samples = samples[:total] + return samples + + def _progress(progress: bool, done: int, total: int, t0: float, usage: Usage) -> None: if progress and (done % 20 == 0 or done == total): rate = done / max(time.time() - t0, 1e-6) @@ -168,6 +190,7 @@ async def run_eval( system: str = '', max_turns: int = 8, max_input_chars: int = 0, + limit_per_task: Optional[int] = None, few_shot_num: int = -1, prompt_style: str = 'strict_letter', ) -> EvalReport: @@ -243,6 +266,7 @@ async def run_eval( adapter, samples, concurrency, progress=progress, gen_kwargs=gen_kwargs, env_factory=env_factory, system=system, max_turns=max_turns, max_input_chars=max_input_chars, + limit_per_task=limit_per_task, few_shot_num=few_shot_num, few_shot_samples=few_shot_samples, few_shot_text=few_shot_text, prompt_style=prompt_style)