79 lines
2.2 KiB
Python
79 lines
2.2 KiB
Python
"""Execution / agent benchmarks. Recipes exist now; scorers raise LayerNotReady
|
|
until the sandbox & agent layers land (interfaces are stable)."""
|
|
|
|
from ..recipe import EvalRecipe, register_eval
|
|
|
|
|
|
@register_eval('humaneval')
|
|
def humaneval():
|
|
return EvalRecipe(
|
|
name='humaneval',
|
|
extract='code_block',
|
|
scorers={'pass@1': 'execution'},
|
|
aggregators={'pass@1': 'pass_at_k'},
|
|
description='HumanEval; sandbox test execution, pass@k.',
|
|
)
|
|
|
|
|
|
@register_eval('bigcodebench')
|
|
def bigcodebench():
|
|
return EvalRecipe(
|
|
name='bigcodebench',
|
|
extract='code_block',
|
|
scorers={'pass@1': 'execution'},
|
|
aggregators={'pass@1': 'pass_at_k'},
|
|
description='BigCodeBench; sandbox test execution with libs, pass@k.',
|
|
)
|
|
|
|
|
|
@register_eval('live_code_bench')
|
|
def live_code_bench():
|
|
return EvalRecipe(
|
|
name='live_code_bench',
|
|
extract='code_block',
|
|
scorers={'pass@1': 'execution'},
|
|
aggregators={'pass@1': 'pass_at_k'},
|
|
description='LiveCodeBench; hidden tests, pass@k.',
|
|
)
|
|
|
|
|
|
@register_eval('swe_bench_verified')
|
|
def swe_bench_verified():
|
|
return EvalRecipe(
|
|
name='swe_bench_verified',
|
|
extract='identity', # a patch, not an answer
|
|
scorers={'resolved': 'env_reward'},
|
|
description='SWE-bench Verified; docker env, FAIL_TO_PASS/PASS_TO_PASS.',
|
|
)
|
|
|
|
|
|
@register_eval('tau2_bench')
|
|
def tau2_bench():
|
|
return EvalRecipe(
|
|
name='tau2_bench',
|
|
extract='identity',
|
|
scorers={'acc': 'env_reward'},
|
|
description='tau2-bench; user-simulated dialog, environment reward.',
|
|
)
|
|
|
|
|
|
@register_eval('bfcl_v3')
|
|
def bfcl_v3():
|
|
return EvalRecipe(
|
|
name='bfcl_v3',
|
|
extract='identity',
|
|
scorers={'acc': 'execution'}, # AST check for most categories; exec for executable ones
|
|
aggregators={'acc': 'weighted_group_avg'}, # group_key = test_category
|
|
description='BFCL v3; AST/exec per category, weighted category average.',
|
|
)
|
|
|
|
|
|
@register_eval('general_fc')
|
|
def general_fc():
|
|
return EvalRecipe(
|
|
name='general_fc',
|
|
extract='identity',
|
|
scorers={'acc': 'execution'},
|
|
description='General function calling; tool-call comparison.',
|
|
)
|