diff --git a/evalharness/sandbox/docker.py b/evalharness/sandbox/docker.py index 1f60fb2..08cb35e 100644 --- a/evalharness/sandbox/docker.py +++ b/evalharness/sandbox/docker.py @@ -82,15 +82,18 @@ class DockerSandbox(Sandbox): try: proc = _run(full, timeout=timeout_s + 30) except subprocess.TimeoutExpired: - _run(['docker', 'rm', '-f', cname]) # CLI died, container didn't + _run(['docker', 'rm', '-f', cname], timeout=60) # CLI died, container didn't return ExecResult(exit_code=-1, timed_out=True, duration_s=timeout_s, error=f'sandbox timeout after {timeout_s}s') except BaseException: # Ctrl+C / kill: reap, then propagate - _run(['docker', 'rm', '-f', cname]) + _run(['docker', 'rm', '-f', cname], timeout=60) raise if proc.returncode != 125 or attempt == 2: break - _run(['docker', 'rm', '-f', cname]) # clear any husk; fresh name next try + # clear any husk; fresh name next try. Bounded: an rm against + # a bloated daemon can hang for minutes and silently eat the + # whole worker pool (7 of 8 workers were observed stuck here) + _run(['docker', 'rm', '-f', cname], timeout=60) time.sleep(2 * (attempt + 1)) # give the daemon a beat return ExecResult( exit_code=proc.returncode,