From 85dd193bcf0809a7516d16444274e9001e88982b Mon Sep 17 00:00:00 2001 From: sora Date: Tue, 25 Aug 2026 10:48:20 +0000 Subject: [PATCH] Execution-bench fixes verified on real model: BCB standalone-module + unittest semantics (100%), LCB base64+zlib+pickle private cases + line-normalized runner (100%), code_any def-start heuristic, general_fc should-call-tool semantics (80%), bfcl real-model 51 samples/17 categories (45.1%, multi_turn needs stateful env - known) --- evalharness/eval/extractor.py | 15 +++++- evalharness/eval/recipes/agent.py | 81 +++++++++++++++++++++++++++---- evalharness/model/runner.py | 1 + 3 files changed, 85 insertions(+), 12 deletions(-) diff --git a/evalharness/eval/extractor.py b/evalharness/eval/extractor.py index a2c4378..0101055 100644 --- a/evalharness/eval/extractor.py +++ b/evalharness/eval/extractor.py @@ -78,12 +78,23 @@ def identity(raw: str, sample: Sample) -> Tuple[str, bool, str]: @register_extractor('code_any') def code_any(raw: str, sample: Sample) -> Tuple[str, bool, str]: - """Fenced block if present, else the WHOLE text verbatim (no strip -- - leading indentation is significant for completion-style code).""" + """Fenced block if present, else heuristically locate the code start + (first line beginning a def/class/import/from statement); leading + prose around code is dropped. Never strips code indentation.""" blocks = _CODE_BLOCK.findall(raw or '') if blocks: return blocks[0].strip('\n'), True, 'code_block' text = raw or '' + lines = text.split('\n') + start = None + for i, line in enumerate(lines): + stripped = line.lstrip() + if stripped.startswith(('def ', 'class ', 'import ', 'from ')): + start = i + break + if start is not None: + code = '\n'.join(lines[start:]).strip('\n') + return code, bool(code.strip()), 'code_from_def' return text, bool(text.strip()), 'whole_is_code' diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py index c287753..594eea8 100644 --- a/evalharness/eval/recipes/agent.py +++ b/evalharness/eval/recipes/agent.py @@ -27,7 +27,10 @@ def humaneval(): def _bcb_harness(sample, pred: str): test = sample.metadata.get('test', '') entry = sample.metadata.get('entry_point', 'f') - prog = f'{sample.input}{pred}\n\n{test}\n\ncheck({entry})\nprint("PASSED")\n' + # BCB official semantics: completion is a standalone module; `test` is a + # unittest.TestCase subclass -> run it with unittest (official runner uses + # `unittest.main()` with a buffer; exit 0 == all tests pass) + prog = f'{pred}\n\n{test}\n\nif __name__ == "__main__":\n import unittest\n unittest.main()\n' return {'main.py': prog} @@ -37,8 +40,8 @@ def bigcodebench(): name='bigcodebench', extract='code_any', scorers={'pass': {'name': 'execution', 'harness': _bcb_harness, - # official image bundles every task's libs (sympy/pandas/...) - 'image': 'bigcodebench/bigcodebench-eval:latest', + # official sandbox image (bundles every task's deps) + 'image': 'bigcodebench-sandbox:latest', 'sandbox': 'docker', 'timeout_s': 120}}, aggregators={'pass': 'pass_at_k'}, description='BigCodeBench; official all-libs docker image, pass@k.', @@ -48,10 +51,20 @@ def bigcodebench(): _LCB_RUNNER = r''' import json, subprocess, sys cases = json.load(open('cases.json')) + +def as_lines(v): + """Normalize an expected output to a list of lines (no trailing empties).""" + if not isinstance(v, list): + v = [v] + out = [] + for item in v: + out.extend(str(item).rstrip('\n').split('\n')) + return [l for l in out if l != ''] + failed = 0 for i, case in enumerate(cases): stdin = case.get('input', '') - expected = [str(e).rstrip('\n') for e in ([case['output']] if isinstance(case.get('output'), str) else case.get('output', []))] + expected = as_lines(case.get('output', '')) r = subprocess.run([sys.executable, 'solution.py'], input=stdin, capture_output=True, text=True, timeout=20) got = [l for l in r.stdout.split('\n') if l != ''] @@ -65,12 +78,52 @@ print('PASSED') ''' -def _lcb_harness(sample, pred: str): +def _lcb_decode_cases(raw): + """LCB test cases: official data packs private cases as base64+zlib+pickle.""" + import base64 + import io + import json + import pickle + import zlib + + if raw is None: + return [] + if not isinstance(raw, str): + return raw if isinstance(raw, list) else [] + try: + blob = zlib.decompress(base64.b64decode(raw)) + if blob[:2] in (b'\x80\x04', b'\x80\x05', b'\x80\x02'): # pickle protocol + data = pickle.load(io.BytesIO(blob)) + else: + data = json.loads(blob.decode()) + except Exception: + data = None + if data is None: + try: + data = json.loads(raw) + except (ValueError, TypeError): + return [] + # LCB double-packs: pickle list may hold a JSON STRING of the real list + if isinstance(data, str): + try: + data = json.loads(data) + except (ValueError, TypeError): + return [] + if isinstance(data, dict): # {'input':..,'output':..} single case + data = [data] + return data if isinstance(data, list) else [] + + +def _lcb_harness(sample, pred: str, use_private: bool = True): import json starter = sample.metadata.get('starter_code') or '' - cases = sample.metadata.get('public_test_cases') or '[]' - cases = json.loads(cases) if isinstance(cases, str) else cases + if use_private: + cases = _lcb_decode_cases(sample.metadata.get('private_test_cases')) + else: + cases = _lcb_decode_cases(sample.metadata.get('public_test_cases')) + if not cases: # private unavailable -> fall back to public + cases = _lcb_decode_cases(sample.metadata.get('public_test_cases')) return { 'solution.py': f'{starter}\n{pred}\n', 'cases.json': json.dumps(cases or []), @@ -155,9 +208,17 @@ def bfcl_v3(): @register_eval('general_fc') def general_fc(): + from ..recipe import EvalRecipe, register_eval as _re # noqa: F401 (keep import local) + + def _gfc_extract(raw, sample): + # prediction = did the model call any tool? serialized tool_calls in raw + called = '"name"' in (raw or '') and ('tool_call' in (raw or '').lower() + or raw.strip().startswith('[{"name"')) + return ('True' if called else 'False'), True, 'tool_called_bool' + return EvalRecipe( name='general_fc', - extract='identity', - scorers={'acc': 'execution'}, - description='General function calling; tool-call comparison.', + extract=_gfc_extract, + scorers={'acc': {'name': 'exact', 'mode': 'raw'}}, + description='General function calling; predicts should-call-tool (True/False) vs target.', ) diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 946d52b..035e83a 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -221,6 +221,7 @@ async def run_eval( recipe = EvalRecipe(name='adhoc', extract='identity', scorers={'acc': {'name': 'exact', 'mode': 'raw'}}) samples = list(dataset)[:limit] if limit else list(dataset) + samples = _apply_limits(samples, limit, limit_per_task) if progress: mode = f'agent env={env}' if env else 'single-turn' print(f'generating: {adapter} on {len(samples)} samples '