From fd7e1af3a43d2c687ce480d3618184737c7f6641 Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Thu, 17 Sep 2026 03:05:13 +0000
Subject: [PATCH] bigcodebench: override the image ENTRYPOINT so our runner
 executes

The official image's ENTRYPOINT is 'python3 -m bigcodebench.evaluate'
-- our 'python /work/main.py' runner was swallowed as CLI args and the
official evaluator died on 'No samples provided'. execution scorer now
accepts entrypoint= and docker exec passes --entrypoint; the BCB recipe
sets entrypoint python3. Also swept 7 leaked eh-exec containers (the
--rm path never fires when our timeout kills the CLI first).

Co-Authored-By: Claude <noreply@anthropic.com>
---
 evalharness/eval/recipes/agent.py | 7 +++++--
 evalharness/eval/scorer.py        | 3 ++-
 evalharness/sandbox/docker.py     | 6 ++++++
 3 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py
index 9e0993f..73e1d7d 100644
--- a/evalharness/eval/recipes/agent.py
+++ b/evalharness/eval/recipes/agent.py
@@ -55,8 +55,11 @@ def bigcodebench():
         name='bigcodebench',
         extract='code_any',
         scorers={'pass': {'name': 'execution', 'harness': _bcb_harness,
-                          # official sandbox image (bundles every task's deps)
-                          'image': 'bigcodebench/bigcodebench-evaluate:latest',  # official hub image, same as evalscope
+                          # official sandbox image (bundles every task's deps);
+                          # its ENTRYPOINT is the official evaluate CLI which
+                          # swallows our runner -> override with plain python3
+                          'image': 'bigcodebench/bigcodebench-evaluate:latest',
+                          'entrypoint': 'python3',
                           'sandbox': 'docker', 'timeout_s': 120}},
         aggregators={'pass': 'pass_at_k'},
         exec_workers=12,
diff --git a/evalharness/eval/scorer.py b/evalharness/eval/scorer.py
index e8d4ed3..2d8040e 100644
--- a/evalharness/eval/scorer.py
+++ b/evalharness/eval/scorer.py
@@ -342,7 +342,8 @@ def execution(pred: str, target, sample: Sample, ctx: ScoreContext):
     files = harness(sample, pred or '')
     result = sbx.exec(files, entry=ctx.params.get('entry', 'main.py'),
                       timeout_s=ctx.params.get('timeout_s', 30),
-                      image=ctx.params.get('image', ''))
+                      image=ctx.params.get('image', ''),
+                      entrypoint=ctx.params.get('entrypoint', ''))
     ok = result.ok
     return ({'pass': 1.0} if ok else {'pass': 0.0}), {'pass': {
         'exit_code': result.exit_code,
diff --git a/evalharness/sandbox/docker.py b/evalharness/sandbox/docker.py
index 3a647fc..8b40e72 100644
--- a/evalharness/sandbox/docker.py
+++ b/evalharness/sandbox/docker.py
@@ -65,6 +65,7 @@ class DockerSandbox(Sandbox):
         mounts: Optional[Dict[str, str]] = None,
         timeout_s: int = 60,
         image: str = '',
+        entrypoint: str = '',
     ) -> ExecResult:
         img = image or self.DEFAULT_EXEC_IMAGE
         with tempfile.TemporaryDirectory(prefix='eh-sbx-') as host_dir:
@@ -92,6 +93,11 @@ class DockerSandbox(Sandbox):
                     cmd += ['-v', f'{out_host}:{cpath}:rw']
             runner = ['python', f'/work/{entry}'] if entry.endswith('.py') \
                 else ['sh', f'/work/{entry}']
+            if entrypoint:
+                # images with an official-evaluator ENTRYPOINT (bigcodebench:
+                # python3 -m bigcodebench.evaluate) swallow our runner as
+                # CLI args -> override it so main.py actually executes
+                cmd += ['--entrypoint', entrypoint]
             # Named + retried runs. A timed-out/killed `docker run` only kills
             # the CLI client -- the container lives on (--rm fires on EXIT)
             # and leaked containers slowly drown the daemon; the name gives