Execution-bench fixes verified on real model: BCB standalone-module + unittest semantics (100%), LCB base64+zlib+pickle private cases + line-normalized runner (100%), code_any def-start heuristic, general_fc should-call-tool semantics (80%), bfcl real-model 51 samples/17 categories (45.1%, multi_turn needs stateful env - known)
This commit is contained in:
parent
1da665fec4
commit
85dd193bcf
@ -78,12 +78,23 @@ def identity(raw: str, sample: Sample) -> Tuple[str, bool, str]:
|
||||
|
||||
@register_extractor('code_any')
|
||||
def code_any(raw: str, sample: Sample) -> Tuple[str, bool, str]:
|
||||
"""Fenced block if present, else the WHOLE text verbatim (no strip --
|
||||
leading indentation is significant for completion-style code)."""
|
||||
"""Fenced block if present, else heuristically locate the code start
|
||||
(first line beginning a def/class/import/from statement); leading
|
||||
prose around code is dropped. Never strips code indentation."""
|
||||
blocks = _CODE_BLOCK.findall(raw or '')
|
||||
if blocks:
|
||||
return blocks[0].strip('\n'), True, 'code_block'
|
||||
text = raw or ''
|
||||
lines = text.split('\n')
|
||||
start = None
|
||||
for i, line in enumerate(lines):
|
||||
stripped = line.lstrip()
|
||||
if stripped.startswith(('def ', 'class ', 'import ', 'from ')):
|
||||
start = i
|
||||
break
|
||||
if start is not None:
|
||||
code = '\n'.join(lines[start:]).strip('\n')
|
||||
return code, bool(code.strip()), 'code_from_def'
|
||||
return text, bool(text.strip()), 'whole_is_code'
|
||||
|
||||
|
||||
|
||||
@ -27,7 +27,10 @@ def humaneval():
|
||||
def _bcb_harness(sample, pred: str):
|
||||
test = sample.metadata.get('test', '')
|
||||
entry = sample.metadata.get('entry_point', 'f')
|
||||
prog = f'{sample.input}{pred}\n\n{test}\n\ncheck({entry})\nprint("PASSED")\n'
|
||||
# BCB official semantics: completion is a standalone module; `test` is a
|
||||
# unittest.TestCase subclass -> run it with unittest (official runner uses
|
||||
# `unittest.main()` with a buffer; exit 0 == all tests pass)
|
||||
prog = f'{pred}\n\n{test}\n\nif __name__ == "__main__":\n import unittest\n unittest.main()\n'
|
||||
return {'main.py': prog}
|
||||
|
||||
|
||||
@ -37,8 +40,8 @@ def bigcodebench():
|
||||
name='bigcodebench',
|
||||
extract='code_any',
|
||||
scorers={'pass': {'name': 'execution', 'harness': _bcb_harness,
|
||||
# official image bundles every task's libs (sympy/pandas/...)
|
||||
'image': 'bigcodebench/bigcodebench-eval:latest',
|
||||
# official sandbox image (bundles every task's deps)
|
||||
'image': 'bigcodebench-sandbox:latest',
|
||||
'sandbox': 'docker', 'timeout_s': 120}},
|
||||
aggregators={'pass': 'pass_at_k'},
|
||||
description='BigCodeBench; official all-libs docker image, pass@k.',
|
||||
@ -48,10 +51,20 @@ def bigcodebench():
|
||||
_LCB_RUNNER = r'''
|
||||
import json, subprocess, sys
|
||||
cases = json.load(open('cases.json'))
|
||||
|
||||
def as_lines(v):
|
||||
"""Normalize an expected output to a list of lines (no trailing empties)."""
|
||||
if not isinstance(v, list):
|
||||
v = [v]
|
||||
out = []
|
||||
for item in v:
|
||||
out.extend(str(item).rstrip('\n').split('\n'))
|
||||
return [l for l in out if l != '']
|
||||
|
||||
failed = 0
|
||||
for i, case in enumerate(cases):
|
||||
stdin = case.get('input', '')
|
||||
expected = [str(e).rstrip('\n') for e in ([case['output']] if isinstance(case.get('output'), str) else case.get('output', []))]
|
||||
expected = as_lines(case.get('output', ''))
|
||||
r = subprocess.run([sys.executable, 'solution.py'], input=stdin,
|
||||
capture_output=True, text=True, timeout=20)
|
||||
got = [l for l in r.stdout.split('\n') if l != '']
|
||||
@ -65,12 +78,52 @@ print('PASSED')
|
||||
'''
|
||||
|
||||
|
||||
def _lcb_harness(sample, pred: str):
|
||||
def _lcb_decode_cases(raw):
|
||||
"""LCB test cases: official data packs private cases as base64+zlib+pickle."""
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import pickle
|
||||
import zlib
|
||||
|
||||
if raw is None:
|
||||
return []
|
||||
if not isinstance(raw, str):
|
||||
return raw if isinstance(raw, list) else []
|
||||
try:
|
||||
blob = zlib.decompress(base64.b64decode(raw))
|
||||
if blob[:2] in (b'\x80\x04', b'\x80\x05', b'\x80\x02'): # pickle protocol
|
||||
data = pickle.load(io.BytesIO(blob))
|
||||
else:
|
||||
data = json.loads(blob.decode())
|
||||
except Exception:
|
||||
data = None
|
||||
if data is None:
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
# LCB double-packs: pickle list may hold a JSON STRING of the real list
|
||||
if isinstance(data, str):
|
||||
try:
|
||||
data = json.loads(data)
|
||||
except (ValueError, TypeError):
|
||||
return []
|
||||
if isinstance(data, dict): # {'input':..,'output':..} single case
|
||||
data = [data]
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
|
||||
def _lcb_harness(sample, pred: str, use_private: bool = True):
|
||||
import json
|
||||
|
||||
starter = sample.metadata.get('starter_code') or ''
|
||||
cases = sample.metadata.get('public_test_cases') or '[]'
|
||||
cases = json.loads(cases) if isinstance(cases, str) else cases
|
||||
if use_private:
|
||||
cases = _lcb_decode_cases(sample.metadata.get('private_test_cases'))
|
||||
else:
|
||||
cases = _lcb_decode_cases(sample.metadata.get('public_test_cases'))
|
||||
if not cases: # private unavailable -> fall back to public
|
||||
cases = _lcb_decode_cases(sample.metadata.get('public_test_cases'))
|
||||
return {
|
||||
'solution.py': f'{starter}\n{pred}\n',
|
||||
'cases.json': json.dumps(cases or []),
|
||||
@ -155,9 +208,17 @@ def bfcl_v3():
|
||||
|
||||
@register_eval('general_fc')
|
||||
def general_fc():
|
||||
from ..recipe import EvalRecipe, register_eval as _re # noqa: F401 (keep import local)
|
||||
|
||||
def _gfc_extract(raw, sample):
|
||||
# prediction = did the model call any tool? serialized tool_calls in raw
|
||||
called = '"name"' in (raw or '') and ('tool_call' in (raw or '').lower()
|
||||
or raw.strip().startswith('[{"name"'))
|
||||
return ('True' if called else 'False'), True, 'tool_called_bool'
|
||||
|
||||
return EvalRecipe(
|
||||
name='general_fc',
|
||||
extract='identity',
|
||||
scorers={'acc': 'execution'},
|
||||
description='General function calling; tool-call comparison.',
|
||||
extract=_gfc_extract,
|
||||
scorers={'acc': {'name': 'exact', 'mode': 'raw'}},
|
||||
description='General function calling; predicts should-call-tool (True/False) vs target.',
|
||||
)
|
||||
|
||||
@ -221,6 +221,7 @@ async def run_eval(
|
||||
recipe = EvalRecipe(name='adhoc', extract='identity',
|
||||
scorers={'acc': {'name': 'exact', 'mode': 'raw'}})
|
||||
samples = list(dataset)[:limit] if limit else list(dataset)
|
||||
samples = _apply_limits(samples, limit, limit_per_task)
|
||||
if progress:
|
||||
mode = f'agent env={env}' if env else 'single-turn'
|
||||
print(f'generating: {adapter} on {len(samples)} samples '
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user