From 9f765fc5cee3b5eed3998b12a28f1607fc27480b Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Thu, 17 Sep 2026 07:41:33 +0000
Subject: [PATCH] Strip blocks from raw predictions before extraction
Hybrid-thinking gateways inline the reasoning channel into content
wrapped in ..; extractors then produced answers like
'3Let me analyze...' (drop em 0.0 with f1 0.5 on samples the
model actually answered). Blocks and stray closers removed pre-extract.
Co-Authored-By: Claude
---
evalharness/eval/runner.py | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/evalharness/eval/runner.py b/evalharness/eval/runner.py
index 97da2ba..ba96821 100644
--- a/evalharness/eval/runner.py
+++ b/evalharness/eval/runner.py
@@ -120,6 +120,15 @@ def evaluate(
each sample gets its own container/workdir)."""
result = _shell(sample, pred)
raw = result.raw_prediction
+ # hybrid-thinking backends sometimes inline the reasoning channel
+ # into content wrapped in ... (or leave a stray
+ # closer): extractors then fish answers out of reasoning text
+ # ('3Let me analyze...'). Strip the blocks before extract.
+ if '' in raw or '' in raw:
+ import re as _re0
+
+ raw = _re0.sub(r'.*?', '', raw, flags=_re0.S)
+ raw = raw.replace('', '')
try:
if bp is not None and sample.sandbox and sample.sandbox.image:
bp.ensure(sample.sandbox.image) # wait only if this one still pulling