133 lines
4.3 KiB
Python
133 lines
4.3 KiB
Python
"""Execution / agent benchmarks. Execution recipes build a runnable program
|
|
(completion + tests + checker) via a harness closure and run it in a sandbox;
|
|
agent recipes wait for the agent layer (env_reward slot)."""
|
|
|
|
from ..recipe import EvalRecipe, register_eval
|
|
|
|
|
|
def _humaneval_harness(sample, pred: str):
|
|
test = sample.metadata.get('test', '')
|
|
entry = sample.metadata.get('entry_point', 'f')
|
|
prog = f'{sample.input}{pred}\n\n{test}\n\ncheck({entry})\nprint("PASSED")\n'
|
|
return {'main.py': prog}
|
|
|
|
|
|
@register_eval('humaneval')
|
|
def humaneval():
|
|
return EvalRecipe(
|
|
name='humaneval',
|
|
extract='code_any',
|
|
scorers={'pass': {'name': 'execution', 'harness': _humaneval_harness,
|
|
'sandbox': 'local', 'timeout_s': 30}},
|
|
aggregators={'pass': 'pass_at_k'},
|
|
description='HumanEval; completion + official tests in a sandbox, pass@k.',
|
|
)
|
|
|
|
|
|
def _bcb_harness_factory(requirements: str):
|
|
def harness(sample, pred: str):
|
|
test = sample.metadata.get('test', '')
|
|
entry = sample.metadata.get('entry_point', 'f')
|
|
prog = f'{sample.input}{pred}\n\n{test}\n\ncheck({entry})\nprint("PASSED")\n'
|
|
return {'main.py': prog}
|
|
|
|
return harness
|
|
|
|
|
|
@register_eval('bigcodebench')
|
|
def bigcodebench():
|
|
return EvalRecipe(
|
|
name='bigcodebench',
|
|
extract='code_any',
|
|
scorers={'pass': {'name': 'execution', 'harness': _bcb_harness_factory('libs'),
|
|
'sandbox': 'docker', 'timeout_s': 120}},
|
|
aggregators={'pass': 'pass_at_k'},
|
|
description='BigCodeBench; library-level tasks need the docker sandbox (pip deps).',
|
|
)
|
|
|
|
|
|
_LCB_RUNNER = r'''
|
|
import json, subprocess, sys
|
|
cases = json.load(open('cases.json'))
|
|
failed = 0
|
|
for i, case in enumerate(cases):
|
|
stdin = case.get('input', '')
|
|
expected = [str(e).rstrip('\n') for e in ([case['output']] if isinstance(case.get('output'), str) else case.get('output', []))]
|
|
r = subprocess.run([sys.executable, 'solution.py'], input=stdin,
|
|
capture_output=True, text=True, timeout=20)
|
|
got = [l for l in r.stdout.split('\n') if l != '']
|
|
if got != expected:
|
|
failed += 1
|
|
print(f'case {i}: expected {expected!r} got {got!r}', file=sys.stderr)
|
|
if failed:
|
|
print(f'{failed}/{len(cases)} cases failed', file=sys.stderr)
|
|
sys.exit(1)
|
|
print('PASSED')
|
|
'''
|
|
|
|
|
|
def _lcb_harness(sample, pred: str):
|
|
import json
|
|
|
|
starter = sample.metadata.get('starter_code') or ''
|
|
cases = sample.metadata.get('public_test_cases') or '[]'
|
|
cases = json.loads(cases) if isinstance(cases, str) else cases
|
|
return {
|
|
'solution.py': f'{starter}\n{pred}\n',
|
|
'cases.json': json.dumps(cases or []),
|
|
'runner.py': _LCB_RUNNER,
|
|
}
|
|
|
|
|
|
@register_eval('live_code_bench')
|
|
def live_code_bench():
|
|
return EvalRecipe(
|
|
name='live_code_bench',
|
|
extract='code_any',
|
|
scorers={'pass': {'name': 'execution', 'harness': _lcb_harness,
|
|
'entry': 'runner.py', 'sandbox': 'local', 'timeout_s': 60}},
|
|
aggregators={'pass': 'pass_at_k'},
|
|
description='LiveCodeBench; stdin/stdout public-case runner in sandbox.',
|
|
)
|
|
|
|
|
|
@register_eval('swe_bench_verified')
|
|
def swe_bench_verified():
|
|
return EvalRecipe(
|
|
name='swe_bench_verified',
|
|
extract='identity', # a patch, not an answer
|
|
scorers={'resolved': 'env_reward'},
|
|
description='SWE-bench Verified; docker env, FAIL_TO_PASS/PASS_TO_PASS.',
|
|
)
|
|
|
|
|
|
@register_eval('tau2_bench')
|
|
def tau2_bench():
|
|
return EvalRecipe(
|
|
name='tau2_bench',
|
|
extract='identity',
|
|
scorers={'acc': 'env_reward'},
|
|
description='tau2-bench; user-simulated dialog, environment reward.',
|
|
)
|
|
|
|
|
|
@register_eval('bfcl_v3')
|
|
def bfcl_v3():
|
|
return EvalRecipe(
|
|
name='bfcl_v3',
|
|
extract='identity',
|
|
scorers={'acc': 'env_reward'}, # call-sequence vs ground truth (bfcl_mock env)
|
|
aggregators={'acc': 'weighted_group_avg'}, # group_key = test_category
|
|
description='BFCL v3; run with env=bfcl_mock (agent pump), official call-sequence scoring.',
|
|
)
|
|
|
|
|
|
@register_eval('general_fc')
|
|
def general_fc():
|
|
return EvalRecipe(
|
|
name='general_fc',
|
|
extract='identity',
|
|
scorers={'acc': 'execution'},
|
|
description='General function calling; tool-call comparison.',
|
|
)
|