From 92b36c5e6cca139cf132c9fea8b9c3438d6b2496 Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Mon, 14 Sep 2026 03:26:35 +0000
Subject: [PATCH] Sandbox: reap timed-out containers, retry daemon-side
 failures; scoring milestones

- docker exec: named containers; a timed-out/killed 'docker run' only
  kills the CLI client while the container lives on (--rm fires on
  EXIT) -- rm -f the name on timeout/interrupt so runs stop leaking
- exit 125 = daemon-side failure, not model failure: retry up to 2x
  (a bloated daemon was turning healthy samples into pass=0)
- scoring milestones: first completion logs immediately, then every
  ~5% (10% was too sparse when docker is slow: minutes of silence
  right after the 'scoring' phase starts, looks hung)

Co-Authored-By: Claude <noreply@anthropic.com>
---
 evalharness/cli.py            |  9 ++++++---
 evalharness/sandbox/docker.py | 31 +++++++++++++++++++++++++------
 2 files changed, 31 insertions(+), 9 deletions(-)

diff --git a/evalharness/cli.py b/evalharness/cli.py
index 1024fab..40dfd34 100644
--- a/evalharness/cli.py
+++ b/evalharness/cli.py
@@ -721,9 +721,12 @@ def _cmd_eval_run(args) -> int:
                     'generating 100%' and looks hung)."""
                     if _reporter is not None:
                         _reporter.set_phase(f'scoring {done}/{total_s}')
-                    # milestone lines: pipes/logs without a live bar see movement
-                    if _cb and total_s and (done % max(1, total_s // 10) == 0
-                                            or done == total_s):
+                    # milestone lines: FIRST completion reports immediately
+                    # (docker-slow runs otherwise look frozen for minutes),
+                    # then every ~5%
+                    step = max(1, total_s // 20) if total_s else 1
+                    if _cb and (done == 1 or done % step == 0
+                                or done == total_s):
                         _cb(f'Scoring {done}/{total_s} samples')
                 _gen_kw = {**bench_cfg, **(getattr(args, '_gen_override', {}) or {})}
                 # max_input_tokens must be a SEPARATE run_eval param (it drives
diff --git a/evalharness/sandbox/docker.py b/evalharness/sandbox/docker.py
index e68e600..1f60fb2 100644
--- a/evalharness/sandbox/docker.py
+++ b/evalharness/sandbox/docker.py
@@ -66,13 +66,32 @@ class DockerSandbox(Sandbox):
                     cmd += ['-v', f'{out_host}:{cpath}:rw']
             runner = ['python', f'/work/{entry}'] if entry.endswith('.py') \
                 else ['sh', f'/work/{entry}']
-            cmd += [img, *runner]
+            # Named + retried runs. A timed-out/killed `docker run` only kills
+            # the CLI client -- the container lives on (--rm fires on EXIT)
+            # and leaked containers slowly drown the daemon; the name gives
+            # us something to rm -f. Exit 125 is a DAEMON-side failure (shim
+            # error etc.), not the model's code failing -- scoring it pass=0
+            # would corrupt results, so retry it.
+            import uuid
+
             t0 = time.time()
-            try:
-                proc = _run(cmd, timeout=timeout_s + 30)
-            except subprocess.TimeoutExpired:
-                return ExecResult(exit_code=-1, timed_out=True, duration_s=timeout_s,
-                                  error=f'sandbox timeout after {timeout_s}s')
+            proc = None
+            for attempt in range(3):
+                cname = f'eh-exec-{uuid.uuid4().hex[:10]}'
+                full = cmd + ['--name', cname, img, *runner]
+                try:
+                    proc = _run(full, timeout=timeout_s + 30)
+                except subprocess.TimeoutExpired:
+                    _run(['docker', 'rm', '-f', cname])  # CLI died, container didn't
+                    return ExecResult(exit_code=-1, timed_out=True, duration_s=timeout_s,
+                                      error=f'sandbox timeout after {timeout_s}s')
+                except BaseException:  # Ctrl+C / kill: reap, then propagate
+                    _run(['docker', 'rm', '-f', cname])
+                    raise
+                if proc.returncode != 125 or attempt == 2:
+                    break
+                _run(['docker', 'rm', '-f', cname])  # clear any husk; fresh name next try
+                time.sleep(2 * (attempt + 1))        # give the daemon a beat
             return ExecResult(
                 exit_code=proc.returncode,
                 stdout=proc.stdout,