From 3be48addfc492461bfe17422750e318c35e6a2d0 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Tue, 15 Sep 2026 02:26:15 +0000 Subject: [PATCH] Thread the truncation tokenize: inline encode froze the event loop assemble() runs the max_input_tokens truncation tokenizer inline in the coroutine; longbench_v2's 2M-token docs take seconds of CPU per encode, and each one BLOCKED the whole loop -- heartbeat frozen, gate probes dead, zero HTTP while the process sat at 100% single-core. encode now runs in a worker thread (loop stays live, encodes parallelize). Co-Authored-By: Claude --- evalharness/model/runner.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 1e9ca83..c81df9f 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -317,7 +317,15 @@ async def generate_predictions( _progress(progress, done_count, len(samples), t0, total_usage) return pred - messages = ([ChatMessage(role='user', content=assemble(sample))] + # assemble() tokenizes for the max_input_tokens truncation -- on + # long-context benches that is SECONDS of CPU per sample (2M-token + # docs), and running it inline FROZE the whole event loop: heartbeat, + # gate probes and every other request serialized behind one encode. + # Thread it: the loop stays live and encodes parallelize (the Rust + # fast tokenizer releases the GIL). + text = await asyncio.to_thread(assemble, sample) \ + if isinstance(sample.input, str) else None + messages = ([ChatMessage(role='user', content=text)] if isinstance(sample.input, str) else list(sample.input)) if not system and extra_system[0] and isinstance(sample.input, str): # renderer-provided SYSTEM contract (es lcb expert-programmer)