Execution-bench fixes verified on real model: BCB standalone-module + unittest semantics (100%), LCB base64+zlib+pickle private cases + line-normalized runner (100%), code_any def-start heuristic, general_fc should-call-tool semantics (80%), bfcl real-model 51 samples/17 categories (45.1%, multi_turn needs stateful env - known)

evalharness/eval/extractor.py

@@ -78,12 +78,23 @@ def identity(raw: str, sample: Sample) -> Tuple[str, bool, str]:
 
 @register_extractor('code_any')
 def code_any(raw: str, sample: Sample) -> Tuple[str, bool, str]:
-    """Fenced block if present, else the WHOLE text verbatim (no strip --
+    """Fenced block if present, else heuristically locate the code start
-    leading indentation is significant for completion-style code)."""
+    (first line beginning a def/class/import/from statement); leading
+    prose around code is dropped. Never strips code indentation."""
     blocks = _CODE_BLOCK.findall(raw or '')
     if blocks:
         return blocks[0].strip('\n'), True, 'code_block'
     text = raw or ''
+    lines = text.split('\n')
+    start = None
+    for i, line in enumerate(lines):
+        stripped = line.lstrip()
+        if stripped.startswith(('def ', 'class ', 'import ', 'from ')):
+            start = i
+            break
+    if start is not None:
+        code = '\n'.join(lines[start:]).strip('\n')
+        return code, bool(code.strip()), 'code_from_def'
     return text, bool(text.strip()), 'whole_is_code'
 
 

evalharness/eval/recipes/agent.py

@@ -27,7 +27,10 @@ def humaneval():
 def _bcb_harness(sample, pred: str):
     test = sample.metadata.get('test', '')
     entry = sample.metadata.get('entry_point', 'f')
-    prog = f'{sample.input}{pred}\n\n{test}\n\ncheck({entry})\nprint("PASSED")\n'
+    # BCB official semantics: completion is a standalone module; `test` is a
+    # unittest.TestCase subclass -> run it with unittest (official runner uses
+    # `unittest.main()` with a buffer; exit 0 == all tests pass)
+    prog = f'{pred}\n\n{test}\n\nif __name__ == "__main__":\n    import unittest\n    unittest.main()\n'
     return {'main.py': prog}
 
 
@@ -37,8 +40,8 @@ def bigcodebench():
         name='bigcodebench',
         extract='code_any',
         scorers={'pass': {'name': 'execution', 'harness': _bcb_harness,
-                          # official image bundles every task's libs (sympy/pandas/...)
+                          # official sandbox image (bundles every task's deps)
-                          'image': 'bigcodebench/bigcodebench-eval:latest',
+                          'image': 'bigcodebench-sandbox:latest',
                           'sandbox': 'docker', 'timeout_s': 120}},
         aggregators={'pass': 'pass_at_k'},
         description='BigCodeBench; official all-libs docker image, pass@k.',
@@ -48,10 +51,20 @@ def bigcodebench():
 _LCB_RUNNER = r'''
 import json, subprocess, sys
 cases = json.load(open('cases.json'))
+
+def as_lines(v):
+    """Normalize an expected output to a list of lines (no trailing empties)."""
+    if not isinstance(v, list):
+        v = [v]
+    out = []
+    for item in v:
+        out.extend(str(item).rstrip('\n').split('\n'))
+    return [l for l in out if l != '']
+
 failed = 0
 for i, case in enumerate(cases):
     stdin = case.get('input', '')
-    expected = [str(e).rstrip('\n') for e in ([case['output']] if isinstance(case.get('output'), str) else case.get('output', []))]
+    expected = as_lines(case.get('output', ''))
     r = subprocess.run([sys.executable, 'solution.py'], input=stdin,
                        capture_output=True, text=True, timeout=20)
     got = [l for l in r.stdout.split('\n') if l != '']
@@ -65,12 +78,52 @@ print('PASSED')
 '''
 
 
-def _lcb_harness(sample, pred: str):
+def _lcb_decode_cases(raw):
+    """LCB test cases: official data packs private cases as base64+zlib+pickle."""
+    import base64
+    import io
+    import json
+    import pickle
+    import zlib
+
+    if raw is None:
+        return []
+    if not isinstance(raw, str):
+        return raw if isinstance(raw, list) else []
+    try:
+        blob = zlib.decompress(base64.b64decode(raw))
+        if blob[:2] in (b'\x80\x04', b'\x80\x05', b'\x80\x02'):  # pickle protocol
+            data = pickle.load(io.BytesIO(blob))
+        else:
+            data = json.loads(blob.decode())
+    except Exception:
+        data = None
+    if data is None:
+        try:
+            data = json.loads(raw)
+        except (ValueError, TypeError):
+            return []
+    # LCB double-packs: pickle list may hold a JSON STRING of the real list
+    if isinstance(data, str):
+        try:
+            data = json.loads(data)
+        except (ValueError, TypeError):
+            return []
+    if isinstance(data, dict):  # {'input':..,'output':..} single case
+        data = [data]
+    return data if isinstance(data, list) else []
+
+
+def _lcb_harness(sample, pred: str, use_private: bool = True):
     import json
 
     starter = sample.metadata.get('starter_code') or ''
-    cases = sample.metadata.get('public_test_cases') or '[]'
+    if use_private:
-    cases = json.loads(cases) if isinstance(cases, str) else cases
+        cases = _lcb_decode_cases(sample.metadata.get('private_test_cases'))
+    else:
+        cases = _lcb_decode_cases(sample.metadata.get('public_test_cases'))
+    if not cases:  # private unavailable -> fall back to public
+        cases = _lcb_decode_cases(sample.metadata.get('public_test_cases'))
     return {
         'solution.py': f'{starter}\n{pred}\n',
         'cases.json': json.dumps(cases or []),
@@ -155,9 +208,17 @@ def bfcl_v3():
 
 @register_eval('general_fc')
 def general_fc():
+    from ..recipe import EvalRecipe, register_eval as _re  # noqa: F401  (keep import local)
+
+    def _gfc_extract(raw, sample):
+        # prediction = did the model call any tool? serialized tool_calls in raw
+        called = '"name"' in (raw or '') and ('tool_call' in (raw or '').lower()
+                                             or raw.strip().startswith('[{"name"'))
+        return ('True' if called else 'False'), True, 'tool_called_bool'
+
     return EvalRecipe(
         name='general_fc',
-        extract='identity',
+        extract=_gfc_extract,
-        scorers={'acc': 'execution'},
+        scorers={'acc': {'name': 'exact', 'mode': 'raw'}},
-        description='General function calling; tool-call comparison.',
+        description='General function calling; predicts should-call-tool (True/False) vs target.',
     )

evalharness/model/runner.py

@@ -221,6 +221,7 @@ async def run_eval(
             recipe = EvalRecipe(name='adhoc', extract='identity',
                                 scorers={'acc': {'name': 'exact', 'mode': 'raw'}})
     samples = list(dataset)[:limit] if limit else list(dataset)
+    samples = _apply_limits(samples, limit, limit_per_task)
     if progress:
         mode = f'agent env={env}' if env else 'single-turn'
         print(f'generating: {adapter} on {len(samples)} samples '