From a32d902e94142979353b6ccccb8a3498506c6608 Mon Sep 17 00:00:00 2001 From: sora Date: Mon, 24 Aug 2026 15:48:42 +0000 Subject: [PATCH] Comparison-driven fixes: MCQ choices in prompt + letter contract, Answer: suffix for QA, markdown answer cleaning, trivia_qa answer_phrase priority, DROP gold-as-alternatives (OR) official semantics, retry on 5xx, numpy/scipy compat --- evalharness/agent/__init__.py | 32 ++++++--- evalharness/agent/envs/bfcl_mock.py | 103 ++++++++++++++++++++++++---- evalharness/agent/loop.py | 23 +++++++ evalharness/eval/extractor.py | 14 ++++ evalharness/eval/recipes/qa.py | 2 +- evalharness/eval/runner.py | 29 ++++++++ evalharness/eval/scorer.py | 38 ++++++++-- evalharness/model/adapter.py | 1 + evalharness/model/runner.py | 24 +++++-- evalharness/sandbox/__init__.py | 3 +- evalharness/sandbox/bg_prefetch.py | 92 +++++++++++++++++++++++++ evalharness/sandbox/prefetch.py | 9 +++ 12 files changed, 334 insertions(+), 36 deletions(-) create mode 100644 evalharness/sandbox/bg_prefetch.py diff --git a/evalharness/agent/__init__.py b/evalharness/agent/__init__.py index 7cc3be1..af612c0 100644 --- a/evalharness/agent/__init__.py +++ b/evalharness/agent/__init__.py @@ -1,14 +1,30 @@ """evalharness.agent -- evaluation driver for agent benchmarks. NOT a general agent framework: no thinking policies. A message pump that -lets the model under test act through Environment plugins (bfcl mock today; -tau2/swe envs land later) and records trajectories for env_reward scorers. - - from evalharness.agent import drive, BFCLEnvironment - traj = await drive(adapter, sample, env=BFCLEnvironment()) +lets the model under test act through Environment plugins and records +trajectories for env_reward scorers. Environments and scoring backends are +registered plugins -- drop a module in agent/envs/ to add one. """ -from .loop import Environment, Trajectory, drive, trajectory_to_prediction -from .envs.bfcl_mock import BFCLEnvironment +import importlib +import pkgutil +from pathlib import Path -__all__ = ['Environment', 'Trajectory', 'drive', 'trajectory_to_prediction', 'BFCLEnvironment'] +from .loop import (ENV_REGISTRY, Environment, Trajectory, drive, + get_env, register_env, trajectory_to_prediction) + + +def _discover_builtin_envs() -> None: + pkg_dir = Path(__file__).parent / 'envs' + for info in pkgutil.iter_modules([str(pkg_dir)]): + importlib.import_module(f'{__name__}.envs.{info.name}') + + +_discover_builtin_envs() + +from .envs.bfcl_mock import (BACKEND_REGISTRY, BFCLEnvironment, # noqa: E402,F401 + get_backend, register_backend) + +__all__ = ['Environment', 'Trajectory', 'drive', 'trajectory_to_prediction', + 'BFCLEnvironment', 'ENV_REGISTRY', 'register_env', 'get_env', + 'BACKEND_REGISTRY', 'register_backend', 'get_backend'] diff --git a/evalharness/agent/envs/bfcl_mock.py b/evalharness/agent/envs/bfcl_mock.py index 955ad94..4c6f943 100644 --- a/evalharness/agent/envs/bfcl_mock.py +++ b/evalharness/agent/envs/bfcl_mock.py @@ -1,19 +1,30 @@ -"""BFCL mock-function environment: model calls declared functions; the env -executes them against official ground-truth state and compares. +"""BFCL mock-function environment + scoring backends (native & official). -Official v3 semantics (eval_checker): for AST-scorable categories the -predicted call sequence (name + args) is compared against ground_truth -tool_calls; stateful multi-turn categories additionally compare final -env state. This env implements the stateful half: it tracks a Python-dict -world, applies ground-truth effects for known calls, and exposes the model's -call sequence + final state for the scorer. +The env records the model's calls (official mock APIs are deterministic); +scoring backends compare them against ground truth. Everything here is +registered plugins: @register_env / @register_backend. """ import json -from typing import Any, Dict, List +from typing import Any, Dict, List, Optional from ...data.sample import ChatMessage, Sample -from ..loop import Environment +from ...eval.registry import EvalRegistry +from ..loop import Environment, register_env + +BACKEND_REGISTRY = EvalRegistry('scoring backend') + + +def register_backend(name: str): + def decorator(fn): + BACKEND_REGISTRY.register(name, fn) + return fn + + return decorator + + +def get_backend(name: str): + return BACKEND_REGISTRY.get(name) def _parse_ground_truth(raw) -> Dict[str, Any]: @@ -25,9 +36,74 @@ def _parse_ground_truth(raw) -> Dict[str, Any]: return raw or {} +@register_backend('bfcl_official') +def bfcl_official(calls: List[Dict[str, Any]], sample: Sample) -> Optional[bool]: + """Strict backend: bfcl_eval official ast_checker. None => package + missing (caller falls back to native). Ground truth and function + descriptions pass through VERBATIM; only the model side is adapted to + the official decoded shape (incl. the dot->underscore name convention + the checker itself applies via convert_func_name).""" + try: + from bfcl_eval.eval_checker.ast_eval.ast_checker import ast_checker + except ImportError: + return None + gt = _parse_ground_truth(sample.target) + if isinstance(gt, dict): + possible = gt.get('ground_truth') + gt_calls = gt.get('tool_calls') + if possible is None and gt_calls: + possible = [{(c.get('function', c) or {}).get('name', ''): + (c.get('function', c) or {}).get('arguments', {})} for c in gt_calls] + else: + possible = gt # official list format, verbatim + if not possible: + return len(calls) == 0 # irrelevance: correct = call nothing + if isinstance(possible, str): + try: + possible = json.loads(possible) + except (ValueError, TypeError): + return None + + lang = {'java': 'Java', 'javascript': 'JavaScript'}.get( + str((sample.metadata or {}).get('language', '')).lower(), 'Python') + category = str((sample.metadata or {}).get('test_category', 'simple')) + convention = str((sample.metadata or {}).get('bfcl_model_convention', + 'gpt-4o-2024-11-20-FC')) + # same underscore convention convert_func_name applies to descriptions + key = (lambda n: n.replace('.', '_')) if 'FC' in convention else (lambda n: n) + model_output = [{key(c['name']): c['arguments']} for c in calls] + + raw_funcs = (sample.metadata or {}).get('functions') + funcs = None + if raw_funcs: + funcs = json.loads(raw_funcs) if isinstance(raw_funcs, str) else raw_funcs + if not funcs: + funcs = [{'name': t.name, 'description': t.description or '', + 'parameters': t.parameters} for t in (sample.tools or [])] + try: + result = ast_checker(funcs, model_output, possible, lang, category, convention) + return bool(result.get('valid')) + except Exception: + # official checker chokes on some v3 schemas; evalscope's adapter + # behaves identically: exception -> not valid + return False + + +@register_backend('bfcl_native') +def bfcl_native(calls: List[Dict[str, Any]], sample: Sample) -> Optional[bool]: + """Lenient native comparison: exact call-sequence match.""" + gt = _parse_ground_truth(sample.target) + if isinstance(gt, list): + want = [list(e.keys())[0] for e in gt] if gt else [] + got = [c['name'] for c in calls] + return want == got + return None # dict-form targets: handled by env_reward's native path + + +@register_env('bfcl_mock') class BFCLEnvironment(Environment): - """Records the model's calls; applies no real side effects (official - mock APIs are deterministic). final_state() exposes calls + ground truth.""" + """Records the model's calls; official mock APIs are deterministic. + final_state() exposes calls + ground truth for the scorer.""" name = 'bfcl_mock' @@ -37,8 +113,7 @@ class BFCLEnvironment(Environment): def reset(self, sample: Sample) -> List[ChatMessage]: self.calls = [] - target = sample.target - self.ground_truth = _parse_ground_truth(target) + self.ground_truth = _parse_ground_truth(sample.target) return [] async def step(self, tool_calls, text: str, sample: Sample) -> List[ChatMessage]: diff --git a/evalharness/agent/loop.py b/evalharness/agent/loop.py index dd842c8..ddc39fc 100644 --- a/evalharness/agent/loop.py +++ b/evalharness/agent/loop.py @@ -35,6 +35,29 @@ class Environment: return {} +# ---------------- environment registry (万物皆可插件: envs too) ---------------- + +from ..eval.registry import EvalRegistry # noqa: E402 + +ENV_REGISTRY = EvalRegistry('environment') + + +def register_env(name: str): + """Class decorator: @register_env('bfcl_mock'). One env per benchmark + family; drop a module in agent/envs/ and it auto-registers.""" + + def decorator(cls): + ENV_REGISTRY.register(name, cls) + return cls + + return decorator + + +def get_env(name: str): + cls = ENV_REGISTRY.get(name) + return cls() + + class Trajectory: """Recorded turns: role-tagged messages + per-turn usage.""" diff --git a/evalharness/eval/extractor.py b/evalharness/eval/extractor.py index edc59a7..a2c4378 100644 --- a/evalharness/eval/extractor.py +++ b/evalharness/eval/extractor.py @@ -120,6 +120,19 @@ _ANSWER_IS = re.compile( _ANSWER_DOLLAR = re.compile(r'final answer is \$([^$]+)\$') +def _clean_answer(text: str) -> str: + """Strip common markdown/noise wrappers from a short extracted answer.""" + t = text.strip() + # **`False`** -> False (bold+code nesting, any order) + for _ in range(3): + t2 = t.strip('`*_~ ').strip() + if t2 == t: + break + t = t2 + # trailing explanation after a period on short answers is rare; keep as-is + return t + + @register_extractor('answer_phrase') def answer_phrase(raw: str, sample: Sample) -> Tuple[str, bool, str]: """Text after the last 'the answer is' / 'ANSWER:' / '答案是' marker.""" @@ -129,6 +142,7 @@ def answer_phrase(raw: str, sample: Sample) -> Tuple[str, bool, str]: if not m: return '', False, 'no answer phrase' value = m.group(1).strip().strip('$.: ').split('\n')[0].strip() + value = _clean_answer(value) return value, bool(value), f'phrase:{m.group(0)[:20].strip()}' diff --git a/evalharness/eval/recipes/qa.py b/evalharness/eval/recipes/qa.py index 7b7dff5..1173257 100644 --- a/evalharness/eval/recipes/qa.py +++ b/evalharness/eval/recipes/qa.py @@ -44,7 +44,7 @@ def drop(): def trivia_qa(): return EvalRecipe( name='trivia_qa', - extract=['first_line', 'answer_phrase'], + extract=['answer_phrase', 'first_line'], scorers={'em': 'alias_match'}, description='TriviaQA; any alias counts.', ) diff --git a/evalharness/eval/runner.py b/evalharness/eval/runner.py index 147bbba..ebaa0ee 100644 --- a/evalharness/eval/runner.py +++ b/evalharness/eval/runner.py @@ -50,6 +50,15 @@ def evaluate( aggregators = recipe.resolve_aggregators() ctx = ScoreContext(judge=judge, params={}) + # If any scorer executes in docker with per-sample images, overlap pulls + # with scoring (run sample N while N+1..N+lookahead images download). + bp = None + if _needs_bg_prefetch(recipe, samples): + from ..sandbox import BackgroundPrefetcher, images_for_samples + + bp = BackgroundPrefetcher(images_for_samples(samples), workers=4, lookahead=8) + bp.__enter__() + results: List[SampleResult] = [] for sample, pred in zip(samples, predictions): raw = pred if isinstance(pred, str) else str(pred.get('raw', '')) @@ -76,6 +85,8 @@ def evaluate( if isinstance(pred, dict) and pred.get('usage'): result.usage = pred['usage'] try: + if bp is not None and sample.sandbox and sample.sandbox.image: + bp.ensure(sample.sandbox.image) # wait only if this one still pulling value, ok, note = extractor(raw, sample) result.extracted_prediction = value result.extraction_ok = ok @@ -108,12 +119,30 @@ def evaluate( samples=results, ) _aggregate_into(report, results, recipe, aggregators) + if bp is not None: + report.metric_groups['run_info'] = { + **report.metric_groups.get('run_info', {}), + **{f'img_{k}': v for k, v in bp.stats().items()}, + } + bp.__exit__(None, None, None) if extra_metadata: report.metric_groups['run_info'] = {k: v for k, v in extra_metadata.items() if isinstance(v, (int, float, str))} return report +def _needs_bg_prefetch(recipe, samples) -> bool: + """True when the recipe executes in docker AND samples declare images.""" + try: + for spec in recipe.scorers.values(): + params = spec if isinstance(spec, dict) else {} + if params.get('name') == 'execution' and params.get('sandbox') == 'docker': + return any(s.sandbox and s.sandbox.image for s in samples[:50]) + except Exception: + return False + return False + + def _aggregate_into(report: EvalReport, results, recipe: EvalRecipe, aggregators) -> None: for metric in recipe.scorers: agg = aggregators.get(metric) diff --git a/evalharness/eval/scorer.py b/evalharness/eval/scorer.py index c9a9eac..4af5dfe 100644 --- a/evalharness/eval/scorer.py +++ b/evalharness/eval/scorer.py @@ -218,13 +218,25 @@ def _drop_metrics(predicted: List[str], gold: List[str]): def em_f1(pred: str, target, sample: Sample, ctx: ScoreContext): """DROP official EM/F1 over gold spans. - Target shape: List[str] (multiple spans) or str. The prediction is split - on the official '\\n' separator of multiple predicted spans. + Gold spans are alternative acceptable answers (OR semantics): the + prediction is split on newlines into predicted spans, then scored + against EACH gold alternative; the best gold wins (official + get_drop_metrics called once per alternative). """ - gold: List[str] = [str(t) for t in target] if isinstance(target, list) else [str(target)] - predicted = [p for p in re.split(r'\n', pred or '') if p.strip()] - em, f1 = _drop_metrics(predicted, gold) - return {'em': em / 100.0 if em > 1.0 else em, 'f1': f1 / 100.0}, {'em': {}, 'f1': {'official': True}} + import re as _re + + golds: List[List[str]] = [] + if isinstance(target, list): + # list of alternative answers -> each is one full gold-answer option + golds = [[str(t)] for t in target] + else: + golds = [[str(target)]] + predicted = [p for p in _re.split(r'\n', pred or '') if p.strip()] + best_em, best_f1 = 0.0, 0.0 + for gold in golds: + em, f1 = _drop_metrics(predicted, gold) + best_em, best_f1 = max(best_em, em), max(best_f1, f1 / 100.0) + return {'em': best_em, 'f1': best_f1}, {'em': {'alternatives': len(golds)}, 'f1': {'official': True}} @register_scorer('alias_match') @@ -331,6 +343,20 @@ def env_reward(pred: str, target, sample: Sample, ctx: ScoreContext): '(run with run_eval(loop=True) or a bench env)' ) calls = env_state.get('calls', []) + backend = ctx.params.get('backend', 'auto') # auto | bfcl_official | bfcl_native | ... + if backend != 'native': + from ..agent.envs.bfcl_mock import BACKEND_REGISTRY, get_backend + + name = backend if backend != 'auto' else 'bfcl_official' + try: + verdict = get_backend(name)(calls, sample) + except KeyError: + verdict = None + if verdict is not None: + return {'acc': float(verdict)}, {'acc': {'mode': name, 'backend': name}} + if backend != 'auto': + raise LayerNotReady(f'backend {backend!r} unavailable ' + '(missing dependency?)') gt_calls = (env_state.get('ground_truth') or {}).get('tool_calls') if gt_calls is None: # irrelevance categories: correct behavior is calling NOTHING diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index e853a9c..c49d7c2 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -149,6 +149,7 @@ class OpenAICompatible(ModelAdapter): for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed', 'response_format'): if kw.get(k) is not None: payload[k] = kw[k] + payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096)) # CoT room return payload def _parse(self, data: Dict[str, Any]) -> ModelOutput: diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 332dbc1..a4cab1c 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -56,7 +56,20 @@ async def generate_predictions( ctx = (sample.metadata or {}).get(key) if ctx: parts.append(str(ctx)) - parts.append(sample.input_text) + question = sample.input_text + if sample.choices: + # MCQ: options MUST be in the prompt; ask for the letter + letters = 'ABCDEFGH' + opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(sample.choices) + if i < len(letters)) + question = (f'{question}\n\n{opts}\n\n' + 'Answer with the letter of the correct option.') + elif sample.task_type in ('qa',): + # gentle output contract (matches official few-shot conventions) + question = (f'{question}\n\n' + 'End your reply with the final answer on its own last line ' + 'in the form "Answer: ".') + parts.append(question) text = '\n\n'.join(parts) if max_input_chars and len(text) > max_input_chars: keep = max_input_chars // 2 @@ -168,12 +181,11 @@ async def run_eval( env_factory = None if env: - from ..agent import BFCLEnvironment + from ..agent.loop import ENV_REGISTRY, get_env - envs = {'bfcl_mock': BFCLEnvironment} - if env not in envs: - raise KeyError(f"unknown env {env!r}; available: {', '.join(envs)}") - env_factory = envs[env] + if env not in ENV_REGISTRY: + raise KeyError(f'unknown env {env!r}; available: {", ".join(ENV_REGISTRY.names())}') + env_factory = lambda: get_env(env) # noqa: E731 try: preds, _usages, usage = await generate_predictions( diff --git a/evalharness/sandbox/__init__.py b/evalharness/sandbox/__init__.py index 2325025..3f21ac3 100644 --- a/evalharness/sandbox/__init__.py +++ b/evalharness/sandbox/__init__.py @@ -24,10 +24,11 @@ from .base import ( from .docker import DockerSandbox, docker_available, docker_serve, serve_env from .local import LocalSandbox from .prefetch import images_for_dataset, prefetch_images +from .bg_prefetch import BackgroundPrefetcher __all__ = [ 'Sandbox', 'DockerSandbox', 'LocalSandbox', 'ExecResult', 'EnvHandle', 'SANDBOX_REGISTRY', 'register_sandbox', 'get_sandbox', 'acquire', 'stop_all', 'docker_serve', 'serve_env', 'docker_available', - 'prefetch_images', 'images_for_dataset', + 'prefetch_images', 'images_for_dataset', 'BackgroundPrefetcher', ] diff --git a/evalharness/sandbox/bg_prefetch.py b/evalharness/sandbox/bg_prefetch.py new file mode 100644 index 0000000..fa992e9 --- /dev/null +++ b/evalharness/sandbox/bg_prefetch.py @@ -0,0 +1,92 @@ +"""Background image prefetcher: overlap docker pulls with evaluation. + +While the runner executes sample N (its image already local), worker +threads pull the images of upcoming samples N+1.. — so per-sample runs +never wait on a cold multi-GB pull unless the queue drains. + + from evalharness.sandbox.prefetch import BackgroundPrefetcher + images = images_for_dataset(ds) # ordered like the dataset + with BackgroundPrefetcher(images, workers=4, lookahead=8) as bp: + for sample in ds: + bp.ensure(sample.sandbox.image) # blocks only if still pulling + ... run in sandbox ... +""" + +import threading +import time +from typing import Dict, List, Optional + +from .prefetch import _pull_one, local_images + + +class BackgroundPrefetcher: + """Pull upcoming images on worker threads; evaluation thread consumes.""" + + def __init__(self, images: List[str], workers: int = 4, lookahead: int = 8): + self.images = list(images) + self.workers = max(1, workers) + self.lookahead = max(1, lookahead) + self._cursor = 0 + self._lock = threading.Lock() + self._ready: Dict[str, bool] = {} + self._failed: Dict[str, str] = {} + self._stop = threading.Event() + self._threads: List[threading.Thread] = [] + for img in self.images: + self._ready[img] = True if img in local_images() else False + + def __enter__(self) -> 'BackgroundPrefetcher': + for i in range(self.workers): + t = threading.Thread(target=self._work, name=f'eh-prefetch-{i}', daemon=True) + t.start() + self._threads.append(t) + return self + + def __exit__(self, *exc) -> None: + self._stop.set() + for t in self._threads: + t.join(timeout=5) + + def _work(self) -> None: + while not self._stop.is_set(): + img = self._next_pending() + if img is None: + time.sleep(0.5) + continue + try: + _pull_one(img) + with self._lock: + self._ready[img] = True + except Exception as e: + with self._lock: + self._failed[img] = str(e)[:200] + + def _next_pending(self) -> Optional[str]: + """Claim the next not-ready image within the lookahead window.""" + with self._lock: + hi = min(self._cursor + self.lookahead, len(self.images)) + for i in range(self._cursor, hi): + img = self.images[i] + if not self._ready.get(img) and img not in self._failed: + return img # claimed (pull is idempotent; duplicates are cheap) + return None + + def ensure(self, image: Optional[str]) -> bool: + """Advance the cursor to `image`; wait (bounded) until pulled.""" + if not image: + return True + with self._lock: # allow random access ordering too + if image in self.images and self.images.index(image) >= self._cursor: + self._cursor = self.images.index(image) + deadline = time.time() + 3600 + while time.time() < deadline and not self._stop.is_set(): + with self._lock: + if self._ready.get(image) or image in self._failed: + return self._ready.get(image, False) + time.sleep(1.0) + return self._ready.get(image, False) + + def stats(self) -> Dict[str, int]: + with self._lock: + ready = sum(1 for v in self._ready.values() if v) + return {'total': len(self.images), 'ready': ready, 'failed': len(self._failed)} diff --git a/evalharness/sandbox/prefetch.py b/evalharness/sandbox/prefetch.py index b4aca65..9666fee 100644 --- a/evalharness/sandbox/prefetch.py +++ b/evalharness/sandbox/prefetch.py @@ -29,6 +29,15 @@ def images_for_dataset(ds: Dataset, limit: int = 0) -> List[str]: return seen +def images_for_samples(samples) -> List[str]: + """Distinct sandbox images in sample order (dedup, keeps order).""" + seen: List[str] = [] + for s in samples: + if s.sandbox and s.sandbox.image and s.sandbox.image not in seen: + seen.append(s.sandbox.image) + return seen + + def local_images() -> set: out = subprocess.run(['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'], capture_output=True, text=True)