diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py index 9e0993f..73e1d7d 100644 --- a/evalharness/eval/recipes/agent.py +++ b/evalharness/eval/recipes/agent.py @@ -55,8 +55,11 @@ def bigcodebench(): name='bigcodebench', extract='code_any', scorers={'pass': {'name': 'execution', 'harness': _bcb_harness, - # official sandbox image (bundles every task's deps) - 'image': 'bigcodebench/bigcodebench-evaluate:latest', # official hub image, same as evalscope + # official sandbox image (bundles every task's deps); + # its ENTRYPOINT is the official evaluate CLI which + # swallows our runner -> override with plain python3 + 'image': 'bigcodebench/bigcodebench-evaluate:latest', + 'entrypoint': 'python3', 'sandbox': 'docker', 'timeout_s': 120}}, aggregators={'pass': 'pass_at_k'}, exec_workers=12, diff --git a/evalharness/eval/scorer.py b/evalharness/eval/scorer.py index e8d4ed3..2d8040e 100644 --- a/evalharness/eval/scorer.py +++ b/evalharness/eval/scorer.py @@ -342,7 +342,8 @@ def execution(pred: str, target, sample: Sample, ctx: ScoreContext): files = harness(sample, pred or '') result = sbx.exec(files, entry=ctx.params.get('entry', 'main.py'), timeout_s=ctx.params.get('timeout_s', 30), - image=ctx.params.get('image', '')) + image=ctx.params.get('image', ''), + entrypoint=ctx.params.get('entrypoint', '')) ok = result.ok return ({'pass': 1.0} if ok else {'pass': 0.0}), {'pass': { 'exit_code': result.exit_code, diff --git a/evalharness/sandbox/docker.py b/evalharness/sandbox/docker.py index 3a647fc..8b40e72 100644 --- a/evalharness/sandbox/docker.py +++ b/evalharness/sandbox/docker.py @@ -65,6 +65,7 @@ class DockerSandbox(Sandbox): mounts: Optional[Dict[str, str]] = None, timeout_s: int = 60, image: str = '', + entrypoint: str = '', ) -> ExecResult: img = image or self.DEFAULT_EXEC_IMAGE with tempfile.TemporaryDirectory(prefix='eh-sbx-') as host_dir: @@ -92,6 +93,11 @@ class DockerSandbox(Sandbox): cmd += ['-v', f'{out_host}:{cpath}:rw'] runner = ['python', f'/work/{entry}'] if entry.endswith('.py') \ else ['sh', f'/work/{entry}'] + if entrypoint: + # images with an official-evaluator ENTRYPOINT (bigcodebench: + # python3 -m bigcodebench.evaluate) swallow our runner as + # CLI args -> override it so main.py actually executes + cmd += ['--entrypoint', entrypoint] # Named + retried runs. A timed-out/killed `docker run` only kills # the CLI client -- the container lives on (--rm fires on EXIT) # and leaked containers slowly drown the daemon; the name gives