Multi-endpoint model pool (openai-pool/{lo..hi} round-robin with failover), nothink/textools spec flags (chat_template_kwargs enable_thinking=false; text-protocol tool calls for backends without --enable-auto-tool-choice), global cross-bench queue in ladder runs

evalharness/data/datasets/trivia_qa.py

@@ -1,4 +1,9 @@
-"""TriviaQA (official source: mandarjoshi/trivia_qa, rc.nocontext config)."""
+"""TriviaQA (official source: mandarjoshi/trivia_qa, rc.wikipedia config).
+
+rc.wikipedia = reading-comprehension WITH the Wikipedia evidence document
+(open-book, aligned with evalscope's default); rc.nocontext is the
+closed-book variant (pass --subset rc.nocontext).
+"""
 
 from ..sample import Sample
 from ..registry import register_dataset
@@ -9,21 +14,32 @@ from ..spec import DatasetSpec
     DatasetSpec(
         name='trivia_qa',
         source='mandarjoshi/trivia_qa',  # official: https://huggingface.co/datasets/mandarjoshi/trivia_qa
-        subset='rc.nocontext',
+        subset='rc.wikipedia',           # open-book (evalscope parity); --subset rc.nocontext for closed
         split='validation',
         task_type='qa',
         tags=['knowledge', 'openqa'],
-        description='TriviaQA open-domain QA without context (official).',
+        description='TriviaQA with Wikipedia evidence (open-book); any alias counts.',
     )
 )
 def trivia_qa():
     def to_sample(record: dict) -> Sample:
         answer = record['answer']  # {'value': ..., 'aliases': [...], ...}
         targets = [answer['value']] + list(answer.get('aliases') or [])
+        # open-book: the Wikipedia evidence document (runner prepends it via
+        # metadata['context'] when assembling the prompt)
+        wiki = ''
+        entity = record.get('entity_pages') or {}
+        for doc in (entity.get('wiki_content') or [])[:1]:
+            wiki = doc or ''
+            break
+        search = record.get('search_results') or {}
+        if not wiki:
+            wiki = '\n'.join((search.get('search_context') or [])[:2])
         return Sample(
             input=record['question'],
             target=targets,  # multi-target: any alias counts
-            metadata={'question_id': record.get('question_id')},
+            metadata={'question_id': record.get('question_id'),
+                      'context': wiki or None},
         )
 
     return to_sample

evalharness/model/adapter.py

@@ -106,6 +106,25 @@ def _key_for(api_base: str) -> str:
 # --------------------------- openai_compatible ---------------------------
 
 
+
+def _parse_text_tool_calls(text: str) -> list:
+    """Extract [{"name":..,"arguments":{..}}] JSON from a text-protocol reply."""
+    import re as _re
+
+    candidates = _re.findall(r'\[[\s\S]*?\]', text) or []
+    for cand in candidates:
+        try:
+            arr = json.loads(cand)
+            if isinstance(arr, list) and arr and all(isinstance(x, dict) and 'name' in x for x in arr):
+                return [{'id': '', 'type': 'function',
+                         'function': {'name': x['name'],
+                                      'arguments': json.dumps(x.get('arguments', {}))}}
+                        for x in arr]
+        except (ValueError, TypeError):
+            continue
+    return []
+
+
 @register_adapter('openai')
 class OpenAICompatible(ModelAdapter):
     """Async OpenAI chat-completions client with zero hard dependencies.
@@ -124,7 +143,7 @@ class OpenAICompatible(ModelAdapter):
         headers = {'Content-Type': 'application/json'}
         if self.api_key:
             headers['Authorization'] = f'Bearer {self.api_key}'
-        retries = self.extra.get('retries', 3)
+        retries = self.extra.get('retries', 6)
         last_exc: Exception = None
         for attempt in range(retries + 1):
             try:
@@ -141,27 +160,45 @@ class OpenAICompatible(ModelAdapter):
                     raise
                 import asyncio
 
-                await asyncio.sleep(min(2 ** attempt * 2, 30))
+                await asyncio.sleep(min(2 ** attempt * 3, 120))
         raise last_exc  # unreachable
 
     def _payload(self, messages, tools, kw) -> Dict[str, Any]:
         msgs = [{'role': m.role, 'content': m.content} for m in messages]
         payload: Dict[str, Any] = {'model': self.model, 'messages': msgs}
         if tools:
-            payload['tools'] = [
+            if self.extra.get('tools_mode', 'api') == 'text':
-                {'type': 'function', 'function': t} if 'function' not in t else t for t in tools
+                # TEXT-PROTOCOL fallback for backends without --enable-auto-tool-choice:
-            ]
+                # declare tools in the prompt; the model outputs JSON calls as text
-        for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed', 'response_format'):
+                decl = '\n\n'.join(
+                    f"- {t.get('name')}: {t.get('description', '')} args={t.get('parameters', {})}"
+                    for t in tools)
+                payload['messages'][-1]['content'] += (
+                    '\n\nYou can call these functions:\n' + decl
+                    + '\nTo call, output ONLY a JSON array like '
+                      '[{"name": "...", "arguments": {...}}] and nothing else.')
+            else:
+                payload['tools'] = [
+                    {'type': 'function', 'function': t} if 'function' not in t else t for t in tools
+                ]
+                payload.pop('chat_template_kwargs', None)
+        for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed',
+                  'response_format', 'chat_template_kwargs'):
             if kw.get(k) is not None:
                 payload[k] = kw[k]
         payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096))  # CoT room
+        if self.extra.get('no_think'):  # Qwen3-style: disable thinking at the adapter level
+            payload.setdefault('chat_template_kwargs', {'enable_thinking': False})
         return payload
 
     def _parse(self, data: Dict[str, Any]) -> ModelOutput:
         choice = (data.get('choices') or [{}])[0]
         msg = choice.get('message') or {}
         calls = []
-        for c in msg.get('tool_calls') or []:
+        raw_calls = list(msg.get('tool_calls') or [])
+        if not raw_calls and self.extra.get('tools_mode') == 'text':
+            raw_calls = _parse_text_tool_calls(msg.get('content') or '')
+        for c in raw_calls:
             fn = c.get('function') or {}
             args = fn.get('arguments') or '{}'
             try:

evalharness/model/pool.py

@@ -0,0 +1,58 @@
+"""Multi-endpoint load balancing: one logical model, N local ports.
+
+    from evalharness.model.pool import PooledAdapter
+    from evalharness.model.adapter import resolve_adapter
+
+    base = resolve_adapter('openai/http://127.0.0.1:8123/v1?Qwen3-8B')
+    pool = PooledAdapter([resolve_adapter(f'openai/http://127.0.0.1:{p}/v1?Qwen3-8B')
+                          for p in range(8123, 8131)])
+    out = await pool.generate(...)   # round-robin over instances
+"""
+
+import itertools
+from typing import List, Optional
+
+from ..data.sample import ChatMessage
+from .adapter import ModelAdapter
+from .output import ModelOutput, Usage
+
+
+class PooledAdapter(ModelAdapter):
+    """Round-robin over N equivalent backend instances."""
+
+    name = 'pool'
+
+    def __init__(self, adapters: List[ModelAdapter]):
+        if not adapters:
+            raise ValueError('PooledAdapter needs at least one backend')
+        super().__init__(model=adapters[0].model, api_base=adapters[0].api_base)
+        self.adapters = adapters
+        self._cycle = itertools.cycle(range(len(adapters)))
+        self.usage = Usage()
+
+    def _next(self) -> ModelAdapter:
+        return self.adapters[next(self._cycle)]
+
+    async def generate(self, messages: List[ChatMessage],
+                       tools: Optional[list] = None, **kw) -> ModelOutput:
+        last_exc = None
+        for _ in range(len(self.adapters)):  # try each instance once
+            adapter = self._next()
+            try:
+                out = await adapter.generate(messages, tools=tools, **kw)
+                self.usage = self.usage + out.usage
+                return out
+            except Exception as e:  # dead instance -> next
+                last_exc = e
+        raise last_exc
+
+    async def close(self) -> None:
+        for a in self.adapters:
+            await a.close()
+
+
+def pooled(specs: List[str]) -> PooledAdapter:
+    """['openai/http://127.0.0.1:8123/v1?M', ...] -> PooledAdapter."""
+    from .adapter import resolve_adapter
+
+    return PooledAdapter([resolve_adapter(s) for s in specs])

evalharness/model/runner.py

@@ -91,9 +91,26 @@ async def generate_predictions(
                 question = (f'{question}\n\n{opts}\n\n'
                             'Answer with the letter of the correct option.')
         elif sample.task_type in ('qa',):
-            question = (f'{question}\n\n'
+            # hle OFFICIAL protocol: answer_type-specific system contract
-                        'End your reply with the final answer on its own last line '
+            at = (sample.metadata or {}).get('answer_type')
-                        'in the form "Answer: <answer>".')
+            if at == 'exactMatch':
+                question = (
+                    'Your response should be in the following format:\n'
+                    'Explanation: {your explanation for your final answer}\n'
+                    'Exact Answer: {your succinct, final answer}\n'
+                    'Confidence: {your confidence score between 0% and 100% for your answer}\n\n'
+                    f'{question}')
+            elif at == 'multipleChoice':
+                question = (
+                    'Your response should be in the following format:\n'
+                    'Explanation: {your explanation for your answer choice}\n'
+                    'Answer: {your chosen answer}\n'
+                    'Confidence: {your confidence score between 0% and 100% for your answer}\n\n'
+                    f'{question}')
+            else:
+                question = (f'{question}\n\n'
+                            'End your reply with the final answer on its own last line '
+                            'in the form "Answer: <answer>".')
         parts.append(question)
         text = '\n\n'.join(parts)
         if max_input_chars and len(text) > max_input_chars:
@@ -360,18 +377,47 @@ async def run_eval(
 
 
 def _make_adapter(spec: str) -> ModelAdapter:
-    """'mock:boxed' -> MockAdapter(mode='boxed'); else resolve_adapter().
+    """Model spec forms:
-
+    - 'mock[:mode]'                     offline adapter
-    The colon-mode syntax exists ONLY for 'mock': adapter names contain no
+    - 'openai-pool/<base-url-template>?model' with {port} placeholder:
-    scheme/colon, so 'mock:xxx' is safe while URLs ('openai/http://...')
+      e.g. 'openai-pool/http://127.0.0.1:{8123..8130}/v1?Qwen3-8B' -> N ports
-    must never be split on ':'.
+    - else resolve_adapter(spec)        single endpoint
     """
-    base, sep, mode = spec.partition(':')
+    opts = {}
-    if sep and '/' not in base and base == 'mock':
+    while True:
+        for f in ('!nothink', '!textools'):
+            if spec.endswith(f):
+                spec = spec[:-len(f)]
+                opts[f] = True
+                break
+        else:
+            break
+    if spec.startswith('openai-pool/'):
+        from .pool import pooled
+
+        rest = spec[len('openai-pool/'):]
+        m = __import__('re').search(r'\{(\d+)\.\.(\d+)\}', rest)
+        if not m:
+            raise ValueError("openai-pool needs a {start..end} port range")
+        lo, hi = int(m.group(1)), int(m.group(2))
+        base_url, _, model = rest.partition('?')
+        specs = []
+        for port in range(lo, hi + 1):
+            specs.append(f'openai/{base_url.replace(m.group(0), str(port))}?{model}')
+        adapter = pooled(specs)
+    elif spec.partition(':')[0] == 'mock' and ':' in spec and '/' not in spec.partition(':')[0]:
         adapter = resolve_adapter('mock')
-        adapter.extra['mode'] = mode or 'echo'
+        adapter.extra['mode'] = spec.partition(':')[2] or 'echo'
         return adapter
-    return resolve_adapter(spec)
+    else:
+        adapter = resolve_adapter(spec)
+    members = adapter.adapters if hasattr(adapter, 'adapters') else [adapter]
+    for a in members:
+        if opts.get('!nothink'):
+            a.extra['no_think'] = True
+        if opts.get('!textools'):
+            a.extra['tools_mode'] = 'text'
+    return adapter
 
 
 def _judge_callable(judge_adapter: ModelAdapter):

evalharness/sandbox/prefetch.py

@@ -16,6 +16,17 @@ from typing import Iterable, List
 
 from ..data.dataset import Dataset
 
+# CN mirrors tried in order before/alongside the daemon's configured mirrors.
+# Some namespaces (e.g. swebench/*) are blocked by individual CN mirrors, so we
+# fall through: daemon default -> 1ms.run -> baidubce -> sjtug.
+_CN_MIRROR_FALLBACKS = [
+    '{img}',  # daemon default (uses its own registry-mirrors config)
+    'docker.1ms.run/{img}',
+    'mirror.baidubce.com/{img}',
+    'docker.mirrors.sjtug.sjtu.edu.cn/{img}',
+    'hub.rat.dev/{img}',
+]
+
 
 def images_for_dataset(ds: Dataset, limit: int = 0) -> List[str]:
     """Collect distinct sandbox images declared by a dataset's samples."""
@@ -82,6 +93,16 @@ def prefetch_images(images: Iterable[str], workers: int = 8) -> List[str]:
 
 
 def _pull_one(image: str) -> None:
-    r = subprocess.run(['docker', 'pull', image], capture_output=True, text=True, timeout=3600)
+    """Pull via CN-mirror fallback chain; retag to the canonical name on hit."""
-    if r.returncode != 0:
+    last_err = None
-        raise RuntimeError(r.stderr.strip()[:200])
+    for template in _CN_MIRROR_FALLBACKS:
+        ref = template.format(img=image)
+        r = subprocess.run(['docker', 'pull', ref], capture_output=True, text=True,
+                           timeout=3600)
+        if r.returncode == 0:
+            if ref != image:  # retag the mirrored pull to the canonical name
+                subprocess.run(['docker', 'tag', ref, image], check=False)
+                subprocess.run(['docker', 'rmi', ref], check=False)
+            return
+        last_err = (ref, (r.stderr or '').strip()[:150])
+    raise RuntimeError(f'all mirrors failed for {image}: {last_err}')