Add --limit-per-task (per-subset cap, evalscope --limit semantics; composable with --limit as intersection); execution-bench parity verified: official human-eval core 5/5 == our docker sandbox on same completions

evalharness/cli.py

@@ -148,6 +148,7 @@ def _cmd_eval_run(args) -> int:
 
                 report = asyncio.run(run_eval(
                     ds, args.model, concurrency=args.concurrency, limit=args.limit,
+                    limit_per_task=args.limit_per_task,
                     judge_spec=args.judge, env=args.env))
             else:
                 from evalharness.eval import evaluate
@@ -262,7 +263,10 @@ def build_parser() -> argparse.ArgumentParser:
     p.add_argument('--judge', default='', help='judge model spec for llm_judge recipes')
     p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump")
     p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)')
-    p.add_argument('--limit', type=int, help='evaluate only the first N samples')
+    p.add_argument('--limit', type=int, help='evaluate only the first N samples total')
+    p.add_argument('--limit-per-task', type=int,
+                   help='first N samples PER subset/category (evalscope --limit semantics); '
+                        'composable with --limit (intersection)')
     p.add_argument('--out', help='save the EvalReport json here (single dataset)')
     p.add_argument('--out-dir', help='save reports/<name>.json + viz/<name>.txt + summary.md '
                                      'here (multi-dataset runs)')

evalharness/model/runner.py

@@ -35,6 +35,7 @@ async def generate_predictions(
     max_turns: int = 8,
     max_input_chars: int = 0,
     attach_context_keys: tuple = ('passage', 'context'),
+    limit_per_task: Optional[int] = None,
     few_shot_num: int = 0,
     few_shot_samples: Optional[List[Sample]] = None,
     few_shot_text: Optional[str] = None,
@@ -57,7 +58,7 @@ async def generate_predictions(
         if few_shot_text:
             parts.append(few_shot_text.strip())  # official exemplars, verbatim
         elif few_shot_num and few_shot_samples:
-            letters_fs = 'ABCDEFGH'
+            letters_fs = 'ABCDEFGHIJ'
             for fs in few_shot_samples[:few_shot_num]:
                 line = f'Question: {fs.input_text}'
                 if fs.choices:
@@ -73,7 +74,7 @@ async def generate_predictions(
         if sample.choices:
             if prompt_style in ('strict_letter', 'auto'):
                 # evalscope/OpenAI-style contract: reply ONLY 'ANSWER: X'
-                letters = 'ABCDEFGH'
+                letters = 'ABCDEFGHIJ'
                 opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(sample.choices)
                                  if i < len(letters))
                 question = (f'Answer the following multiple choice question. The entire '
@@ -81,7 +82,7 @@ async def generate_predictions(
                             f"'ANSWER: [LETTER]' (without quotes) where [LETTER] is one of "
                             f'{letters[:len(sample.choices)]}.\n\n{question}\n\n{opts}')
             else:
-                letters = 'ABCDEFGH'
+                letters = 'ABCDEFGHIJ'
                 opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(sample.choices)
                                  if i < len(letters))
                 question = (f'{question}\n\n{opts}\n\n'
@@ -141,12 +142,33 @@ async def generate_predictions(
         _progress(progress, done_count, len(samples), t0, total_usage)
         return {'raw': text, 'usage': out.usage.model_dump()}
 
-    work = samples[:limit] if limit else samples
+    work = _apply_limits(samples, limit, limit_per_task)
     preds = list(await asyncio.gather(*(one(s) for s in work)))
     usages = [p.get('usage', {}) for p in preds]
     return preds, usages, total_usage
 
 
+def _apply_limits(samples: List[Sample], total: Optional[int],
+                  per_task: Optional[int], dataset=None) -> List[Sample]:
+    """total: cap the WHOLE run (ours semantics). per_task: cap each subset/
+    category (evalscope's --limit semantics) -- first N per group_key."""
+    if per_task:
+        seen: Dict[str, int] = {}
+        out = []
+        for s in samples:
+            key = str((s.metadata or {}).get('category')
+                      or (s.metadata or {}).get('subject')
+                      or (s.metadata or {}).get('test_category')
+                      or getattr(getattr(dataset, 'spec', None), 'subset', 'default'))
+            if seen.get(key, 0) < per_task:
+                seen[key] = seen.get(key, 0) + 1
+                out.append(s)
+        samples = out
+    if total:
+        samples = samples[:total]
+    return samples
+
+
 def _progress(progress: bool, done: int, total: int, t0: float, usage: Usage) -> None:
     if progress and (done % 20 == 0 or done == total):
         rate = done / max(time.time() - t0, 1e-6)
@@ -168,6 +190,7 @@ async def run_eval(
     system: str = '',
     max_turns: int = 8,
     max_input_chars: int = 0,
+    limit_per_task: Optional[int] = None,
     few_shot_num: int = -1,
     prompt_style: str = 'strict_letter',
 ) -> EvalReport:
@@ -243,6 +266,7 @@ async def run_eval(
             adapter, samples, concurrency, progress=progress,
             gen_kwargs=gen_kwargs, env_factory=env_factory,
             system=system, max_turns=max_turns, max_input_chars=max_input_chars,
+            limit_per_task=limit_per_task,
             few_shot_num=few_shot_num,
             few_shot_samples=few_shot_samples, few_shot_text=few_shot_text,
             prompt_style=prompt_style)