diff --git a/evalharness/eval/runner.py b/evalharness/eval/runner.py index 97da2ba..ba96821 100644 --- a/evalharness/eval/runner.py +++ b/evalharness/eval/runner.py @@ -120,6 +120,15 @@ def evaluate( each sample gets its own container/workdir).""" result = _shell(sample, pred) raw = result.raw_prediction + # hybrid-thinking backends sometimes inline the reasoning channel + # into content wrapped in ... (or leave a stray + # closer): extractors then fish answers out of reasoning text + # ('3Let me analyze...'). Strip the blocks before extract. + if '' in raw or '' in raw: + import re as _re0 + + raw = _re0.sub(r'.*?', '', raw, flags=_re0.S) + raw = raw.replace('', '') try: if bp is not None and sample.sandbox and sample.sandbox.image: bp.ensure(sample.sandbox.image) # wait only if this one still pulling