Comparison-driven fixes: MCQ choices in prompt + letter contract, Answer: suffix for QA, markdown answer cleaning, trivia_qa answer_phrase priority, DROP gold-as-alternatives (OR) official semantics, retry on 5xx, numpy/scipy compat

evalharness/agent/__init__.py

@@ -1,14 +1,30 @@
 """evalharness.agent -- evaluation driver for agent benchmarks.
 
 NOT a general agent framework: no thinking policies. A message pump that
-lets the model under test act through Environment plugins (bfcl mock today;
+lets the model under test act through Environment plugins and records
-tau2/swe envs land later) and records trajectories for env_reward scorers.
+trajectories for env_reward scorers. Environments and scoring backends are
-
+registered plugins -- drop a module in agent/envs/ to add one.
-    from evalharness.agent import drive, BFCLEnvironment
-    traj = await drive(adapter, sample, env=BFCLEnvironment())
 """
 
-from .loop import Environment, Trajectory, drive, trajectory_to_prediction
+import importlib
-from .envs.bfcl_mock import BFCLEnvironment
+import pkgutil
+from pathlib import Path
 
-__all__ = ['Environment', 'Trajectory', 'drive', 'trajectory_to_prediction', 'BFCLEnvironment']
+from .loop import (ENV_REGISTRY, Environment, Trajectory, drive,
+                   get_env, register_env, trajectory_to_prediction)
+
+
+def _discover_builtin_envs() -> None:
+    pkg_dir = Path(__file__).parent / 'envs'
+    for info in pkgutil.iter_modules([str(pkg_dir)]):
+        importlib.import_module(f'{__name__}.envs.{info.name}')
+
+
+_discover_builtin_envs()
+
+from .envs.bfcl_mock import (BACKEND_REGISTRY, BFCLEnvironment,  # noqa: E402,F401
+                             get_backend, register_backend)
+
+__all__ = ['Environment', 'Trajectory', 'drive', 'trajectory_to_prediction',
+           'BFCLEnvironment', 'ENV_REGISTRY', 'register_env', 'get_env',
+           'BACKEND_REGISTRY', 'register_backend', 'get_backend']

evalharness/agent/envs/bfcl_mock.py

@@ -1,19 +1,30 @@
-"""BFCL mock-function environment: model calls declared functions; the env
+"""BFCL mock-function environment + scoring backends (native & official).
-executes them against official ground-truth state and compares.
 
-Official v3 semantics (eval_checker): for AST-scorable categories the
+The env records the model's calls (official mock APIs are deterministic);
-predicted call sequence (name + args) is compared against ground_truth
+scoring backends compare them against ground truth. Everything here is
-tool_calls; stateful multi-turn categories additionally compare final
+registered plugins: @register_env / @register_backend.
-env state. This env implements the stateful half: it tracks a Python-dict
-world, applies ground-truth effects for known calls, and exposes the model's
-call sequence + final state for the scorer.
 """
 
 import json
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from ...data.sample import ChatMessage, Sample
-from ..loop import Environment
+from ...eval.registry import EvalRegistry
+from ..loop import Environment, register_env
+
+BACKEND_REGISTRY = EvalRegistry('scoring backend')
+
+
+def register_backend(name: str):
+    def decorator(fn):
+        BACKEND_REGISTRY.register(name, fn)
+        return fn
+
+    return decorator
+
+
+def get_backend(name: str):
+    return BACKEND_REGISTRY.get(name)
 
 
 def _parse_ground_truth(raw) -> Dict[str, Any]:
@@ -25,9 +36,74 @@ def _parse_ground_truth(raw) -> Dict[str, Any]:
     return raw or {}
 
 
+@register_backend('bfcl_official')
+def bfcl_official(calls: List[Dict[str, Any]], sample: Sample) -> Optional[bool]:
+    """Strict backend: bfcl_eval official ast_checker. None => package
+    missing (caller falls back to native). Ground truth and function
+    descriptions pass through VERBATIM; only the model side is adapted to
+    the official decoded shape (incl. the dot->underscore name convention
+    the checker itself applies via convert_func_name)."""
+    try:
+        from bfcl_eval.eval_checker.ast_eval.ast_checker import ast_checker
+    except ImportError:
+        return None
+    gt = _parse_ground_truth(sample.target)
+    if isinstance(gt, dict):
+        possible = gt.get('ground_truth')
+        gt_calls = gt.get('tool_calls')
+        if possible is None and gt_calls:
+            possible = [{(c.get('function', c) or {}).get('name', ''):
+                         (c.get('function', c) or {}).get('arguments', {})} for c in gt_calls]
+    else:
+        possible = gt  # official list format, verbatim
+    if not possible:
+        return len(calls) == 0  # irrelevance: correct = call nothing
+    if isinstance(possible, str):
+        try:
+            possible = json.loads(possible)
+        except (ValueError, TypeError):
+            return None
+
+    lang = {'java': 'Java', 'javascript': 'JavaScript'}.get(
+        str((sample.metadata or {}).get('language', '')).lower(), 'Python')
+    category = str((sample.metadata or {}).get('test_category', 'simple'))
+    convention = str((sample.metadata or {}).get('bfcl_model_convention',
+                                                 'gpt-4o-2024-11-20-FC'))
+    # same underscore convention convert_func_name applies to descriptions
+    key = (lambda n: n.replace('.', '_')) if 'FC' in convention else (lambda n: n)
+    model_output = [{key(c['name']): c['arguments']} for c in calls]
+
+    raw_funcs = (sample.metadata or {}).get('functions')
+    funcs = None
+    if raw_funcs:
+        funcs = json.loads(raw_funcs) if isinstance(raw_funcs, str) else raw_funcs
+    if not funcs:
+        funcs = [{'name': t.name, 'description': t.description or '',
+                  'parameters': t.parameters} for t in (sample.tools or [])]
+    try:
+        result = ast_checker(funcs, model_output, possible, lang, category, convention)
+        return bool(result.get('valid'))
+    except Exception:
+        # official checker chokes on some v3 schemas; evalscope's adapter
+        # behaves identically: exception -> not valid
+        return False
+
+
+@register_backend('bfcl_native')
+def bfcl_native(calls: List[Dict[str, Any]], sample: Sample) -> Optional[bool]:
+    """Lenient native comparison: exact call-sequence match."""
+    gt = _parse_ground_truth(sample.target)
+    if isinstance(gt, list):
+        want = [list(e.keys())[0] for e in gt] if gt else []
+        got = [c['name'] for c in calls]
+        return want == got
+    return None  # dict-form targets: handled by env_reward's native path
+
+
+@register_env('bfcl_mock')
 class BFCLEnvironment(Environment):
-    """Records the model's calls; applies no real side effects (official
+    """Records the model's calls; official mock APIs are deterministic.
-    mock APIs are deterministic). final_state() exposes calls + ground truth."""
+    final_state() exposes calls + ground truth for the scorer."""
 
     name = 'bfcl_mock'
 
@@ -37,8 +113,7 @@ class BFCLEnvironment(Environment):
 
     def reset(self, sample: Sample) -> List[ChatMessage]:
         self.calls = []
-        target = sample.target
+        self.ground_truth = _parse_ground_truth(sample.target)
-        self.ground_truth = _parse_ground_truth(target)
         return []
 
     async def step(self, tool_calls, text: str, sample: Sample) -> List[ChatMessage]:

evalharness/agent/loop.py

@@ -35,6 +35,29 @@ class Environment:
         return {}
 
 
+# ---------------- environment registry (万物皆可插件: envs too) ----------------
+
+from ..eval.registry import EvalRegistry  # noqa: E402
+
+ENV_REGISTRY = EvalRegistry('environment')
+
+
+def register_env(name: str):
+    """Class decorator: @register_env('bfcl_mock'). One env per benchmark
+    family; drop a module in agent/envs/ and it auto-registers."""
+
+    def decorator(cls):
+        ENV_REGISTRY.register(name, cls)
+        return cls
+
+    return decorator
+
+
+def get_env(name: str):
+    cls = ENV_REGISTRY.get(name)
+    return cls()
+
+
 class Trajectory:
     """Recorded turns: role-tagged messages + per-turn usage."""
 

evalharness/eval/extractor.py

@@ -120,6 +120,19 @@ _ANSWER_IS = re.compile(
 _ANSWER_DOLLAR = re.compile(r'final answer is \$([^$]+)\$')
 
 
+def _clean_answer(text: str) -> str:
+    """Strip common markdown/noise wrappers from a short extracted answer."""
+    t = text.strip()
+    # **`False`** -> False   (bold+code nesting, any order)
+    for _ in range(3):
+        t2 = t.strip('`*_~ ').strip()
+        if t2 == t:
+            break
+        t = t2
+    # trailing explanation after a period on short answers is rare; keep as-is
+    return t
+
+
 @register_extractor('answer_phrase')
 def answer_phrase(raw: str, sample: Sample) -> Tuple[str, bool, str]:
     """Text after the last 'the answer is' / 'ANSWER:' / '答案是' marker."""
@@ -129,6 +142,7 @@ def answer_phrase(raw: str, sample: Sample) -> Tuple[str, bool, str]:
     if not m:
         return '', False, 'no answer phrase'
     value = m.group(1).strip().strip('$.: ').split('\n')[0].strip()
+    value = _clean_answer(value)
     return value, bool(value), f'phrase:{m.group(0)[:20].strip()}'
 
 

evalharness/eval/recipes/qa.py

@@ -44,7 +44,7 @@ def drop():
 def trivia_qa():
     return EvalRecipe(
         name='trivia_qa',
-        extract=['first_line', 'answer_phrase'],
+        extract=['answer_phrase', 'first_line'],
         scorers={'em': 'alias_match'},
         description='TriviaQA; any alias counts.',
     )

evalharness/eval/runner.py

@@ -50,6 +50,15 @@ def evaluate(
     aggregators = recipe.resolve_aggregators()
     ctx = ScoreContext(judge=judge, params={})
 
+    # If any scorer executes in docker with per-sample images, overlap pulls
+    # with scoring (run sample N while N+1..N+lookahead images download).
+    bp = None
+    if _needs_bg_prefetch(recipe, samples):
+        from ..sandbox import BackgroundPrefetcher, images_for_samples
+
+        bp = BackgroundPrefetcher(images_for_samples(samples), workers=4, lookahead=8)
+        bp.__enter__()
+
     results: List[SampleResult] = []
     for sample, pred in zip(samples, predictions):
         raw = pred if isinstance(pred, str) else str(pred.get('raw', ''))
@@ -76,6 +85,8 @@ def evaluate(
         if isinstance(pred, dict) and pred.get('usage'):
             result.usage = pred['usage']
         try:
+            if bp is not None and sample.sandbox and sample.sandbox.image:
+                bp.ensure(sample.sandbox.image)  # wait only if this one still pulling
             value, ok, note = extractor(raw, sample)
             result.extracted_prediction = value
             result.extraction_ok = ok
@@ -108,12 +119,30 @@ def evaluate(
         samples=results,
     )
     _aggregate_into(report, results, recipe, aggregators)
+    if bp is not None:
+        report.metric_groups['run_info'] = {
+            **report.metric_groups.get('run_info', {}),
+            **{f'img_{k}': v for k, v in bp.stats().items()},
+        }
+        bp.__exit__(None, None, None)
     if extra_metadata:
         report.metric_groups['run_info'] = {k: v for k, v in extra_metadata.items()
                                             if isinstance(v, (int, float, str))}
     return report
 
 
+def _needs_bg_prefetch(recipe, samples) -> bool:
+    """True when the recipe executes in docker AND samples declare images."""
+    try:
+        for spec in recipe.scorers.values():
+            params = spec if isinstance(spec, dict) else {}
+            if params.get('name') == 'execution' and params.get('sandbox') == 'docker':
+                return any(s.sandbox and s.sandbox.image for s in samples[:50])
+    except Exception:
+        return False
+    return False
+
+
 def _aggregate_into(report: EvalReport, results, recipe: EvalRecipe, aggregators) -> None:
     for metric in recipe.scorers:
         agg = aggregators.get(metric)

evalharness/eval/scorer.py

@@ -218,13 +218,25 @@ def _drop_metrics(predicted: List[str], gold: List[str]):
 def em_f1(pred: str, target, sample: Sample, ctx: ScoreContext):
     """DROP official EM/F1 over gold spans.
 
-    Target shape: List[str] (multiple spans) or str. The prediction is split
+    Gold spans are alternative acceptable answers (OR semantics): the
-    on the official '\\n' separator of multiple predicted spans.
+    prediction is split on newlines into predicted spans, then scored
+    against EACH gold alternative; the best gold wins (official
+    get_drop_metrics called once per alternative).
     """
-    gold: List[str] = [str(t) for t in target] if isinstance(target, list) else [str(target)]
+    import re as _re
-    predicted = [p for p in re.split(r'\n', pred or '') if p.strip()]
+
-    em, f1 = _drop_metrics(predicted, gold)
+    golds: List[List[str]] = []
-    return {'em': em / 100.0 if em > 1.0 else em, 'f1': f1 / 100.0}, {'em': {}, 'f1': {'official': True}}
+    if isinstance(target, list):
+        # list of alternative answers -> each is one full gold-answer option
+        golds = [[str(t)] for t in target]
+    else:
+        golds = [[str(target)]]
+    predicted = [p for p in _re.split(r'\n', pred or '') if p.strip()]
+    best_em, best_f1 = 0.0, 0.0
+    for gold in golds:
+        em, f1 = _drop_metrics(predicted, gold)
+        best_em, best_f1 = max(best_em, em), max(best_f1, f1 / 100.0)
+    return {'em': best_em, 'f1': best_f1}, {'em': {'alternatives': len(golds)}, 'f1': {'official': True}}
 
 
 @register_scorer('alias_match')
@@ -331,6 +343,20 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext):
             '(run with run_eval(loop=True) or a bench env)'
         )
     calls = env_state.get('calls', [])
+    backend = ctx.params.get('backend', 'auto')  # auto | bfcl_official | bfcl_native | ...
+    if backend != 'native':
+        from ..agent.envs.bfcl_mock import BACKEND_REGISTRY, get_backend
+
+        name = backend if backend != 'auto' else 'bfcl_official'
+        try:
+            verdict = get_backend(name)(calls, sample)
+        except KeyError:
+            verdict = None
+        if verdict is not None:
+            return {'acc': float(verdict)}, {'acc': {'mode': name, 'backend': name}}
+        if backend != 'auto':
+            raise LayerNotReady(f'backend {backend!r} unavailable '
+                                '(missing dependency?)')
     gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls')
     if gt_calls is None:
         # irrelevance categories: correct behavior is calling NOTHING

evalharness/model/adapter.py

@@ -149,6 +149,7 @@ class OpenAICompatible(ModelAdapter):
         for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed', 'response_format'):
             if kw.get(k) is not None:
                 payload[k] = kw[k]
+        payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096))  # CoT room
         return payload
 
     def _parse(self, data: Dict[str, Any]) -> ModelOutput:

evalharness/model/runner.py

@@ -56,7 +56,20 @@ async def generate_predictions(
             ctx = (sample.metadata or {}).get(key)
             if ctx:
                 parts.append(str(ctx))
-        parts.append(sample.input_text)
+        question = sample.input_text
+        if sample.choices:
+            # MCQ: options MUST be in the prompt; ask for the letter
+            letters = 'ABCDEFGH'
+            opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(sample.choices)
+                             if i < len(letters))
+            question = (f'{question}\n\n{opts}\n\n'
+                        'Answer with the letter of the correct option.')
+        elif sample.task_type in ('qa',):
+            # gentle output contract (matches official few-shot conventions)
+            question = (f'{question}\n\n'
+                        'End your reply with the final answer on its own last line '
+                        'in the form "Answer: <answer>".')
+        parts.append(question)
         text = '\n\n'.join(parts)
         if max_input_chars and len(text) > max_input_chars:
             keep = max_input_chars // 2
@@ -168,12 +181,11 @@ async def run_eval(
 
     env_factory = None
     if env:
-        from ..agent import BFCLEnvironment
+        from ..agent.loop import ENV_REGISTRY, get_env
 
-        envs = {'bfcl_mock': BFCLEnvironment}
+        if env not in ENV_REGISTRY:
-        if env not in envs:
+            raise KeyError(f'unknown env {env!r}; available: {", ".join(ENV_REGISTRY.names())}')
-            raise KeyError(f"unknown env {env!r}; available: {', '.join(envs)}")
+        env_factory = lambda: get_env(env)  # noqa: E731
-        env_factory = envs[env]
 
     try:
         preds, _usages, usage = await generate_predictions(

evalharness/sandbox/__init__.py

@@ -24,10 +24,11 @@ from .base import (
 from .docker import DockerSandbox, docker_available, docker_serve, serve_env
 from .local import LocalSandbox
 from .prefetch import images_for_dataset, prefetch_images
+from .bg_prefetch import BackgroundPrefetcher
 
 __all__ = [
     'Sandbox', 'DockerSandbox', 'LocalSandbox', 'ExecResult', 'EnvHandle',
     'SANDBOX_REGISTRY', 'register_sandbox', 'get_sandbox', 'acquire', 'stop_all',
     'docker_serve', 'serve_env', 'docker_available',
-    'prefetch_images', 'images_for_dataset',
+    'prefetch_images', 'images_for_dataset', 'BackgroundPrefetcher',
 ]

evalharness/sandbox/bg_prefetch.py

@@ -0,0 +1,92 @@
+"""Background image prefetcher: overlap docker pulls with evaluation.
+
+While the runner executes sample N (its image already local), worker
+threads pull the images of upcoming samples N+1.. — so per-sample runs
+never wait on a cold multi-GB pull unless the queue drains.
+
+    from evalharness.sandbox.prefetch import BackgroundPrefetcher
+    images = images_for_dataset(ds)          # ordered like the dataset
+    with BackgroundPrefetcher(images, workers=4, lookahead=8) as bp:
+        for sample in ds:
+            bp.ensure(sample.sandbox.image)  # blocks only if still pulling
+            ... run in sandbox ...
+"""
+
+import threading
+import time
+from typing import Dict, List, Optional
+
+from .prefetch import _pull_one, local_images
+
+
+class BackgroundPrefetcher:
+    """Pull upcoming images on worker threads; evaluation thread consumes."""
+
+    def __init__(self, images: List[str], workers: int = 4, lookahead: int = 8):
+        self.images = list(images)
+        self.workers = max(1, workers)
+        self.lookahead = max(1, lookahead)
+        self._cursor = 0
+        self._lock = threading.Lock()
+        self._ready: Dict[str, bool] = {}
+        self._failed: Dict[str, str] = {}
+        self._stop = threading.Event()
+        self._threads: List[threading.Thread] = []
+        for img in self.images:
+            self._ready[img] = True if img in local_images() else False
+
+    def __enter__(self) -> 'BackgroundPrefetcher':
+        for i in range(self.workers):
+            t = threading.Thread(target=self._work, name=f'eh-prefetch-{i}', daemon=True)
+            t.start()
+            self._threads.append(t)
+        return self
+
+    def __exit__(self, *exc) -> None:
+        self._stop.set()
+        for t in self._threads:
+            t.join(timeout=5)
+
+    def _work(self) -> None:
+        while not self._stop.is_set():
+            img = self._next_pending()
+            if img is None:
+                time.sleep(0.5)
+                continue
+            try:
+                _pull_one(img)
+                with self._lock:
+                    self._ready[img] = True
+            except Exception as e:
+                with self._lock:
+                    self._failed[img] = str(e)[:200]
+
+    def _next_pending(self) -> Optional[str]:
+        """Claim the next not-ready image within the lookahead window."""
+        with self._lock:
+            hi = min(self._cursor + self.lookahead, len(self.images))
+            for i in range(self._cursor, hi):
+                img = self.images[i]
+                if not self._ready.get(img) and img not in self._failed:
+                    return img  # claimed (pull is idempotent; duplicates are cheap)
+            return None
+
+    def ensure(self, image: Optional[str]) -> bool:
+        """Advance the cursor to `image`; wait (bounded) until pulled."""
+        if not image:
+            return True
+        with self._lock:  # allow random access ordering too
+            if image in self.images and self.images.index(image) >= self._cursor:
+                self._cursor = self.images.index(image)
+        deadline = time.time() + 3600
+        while time.time() < deadline and not self._stop.is_set():
+            with self._lock:
+                if self._ready.get(image) or image in self._failed:
+                    return self._ready.get(image, False)
+            time.sleep(1.0)
+        return self._ready.get(image, False)
+
+    def stats(self) -> Dict[str, int]:
+        with self._lock:
+            ready = sum(1 for v in self._ready.values() if v)
+            return {'total': len(self.images), 'ready': ready, 'failed': len(self._failed)}

evalharness/sandbox/prefetch.py

@@ -29,6 +29,15 @@ def images_for_dataset(ds: Dataset, limit: int = 0) -> List[str]:
     return seen
 
 
+def images_for_samples(samples) -> List[str]:
+    """Distinct sandbox images in sample order (dedup, keeps order)."""
+    seen: List[str] = []
+    for s in samples:
+        if s.sandbox and s.sandbox.image and s.sandbox.image not in seen:
+            seen.append(s.sandbox.image)
+    return seen
+
+
 def local_images() -> set:
     out = subprocess.run(['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'],
                          capture_output=True, text=True)