Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Multi-endpoint model pool (openai-pool/{lo..hi} round-robin with failover), nothink/textools spec flags (chat_template_kwargs enable_thinking=false; text-protocol tool calls for backends without --enable-auto-tool-choice), global cross-bench queue in ladder runs

Browse Source
This commit is contained in:
sora 2026-08-26 06:26:52 +00:00
parent f3514d7c08
commit 7114564301
5 changed files with 204 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
evalharness/data/datasets/trivia_qa.py
View File

@ -1,4 +1,9 @@
"""TriviaQA (official source: mandarjoshi/trivia_qa, rc.nocontext config)."""
"""TriviaQA (official source: mandarjoshi/trivia_qa, rc.wikipedia config).
rc.wikipedia = reading-comprehension WITH the Wikipedia evidence document
(open-book, aligned with evalscope's default); rc.nocontext is the
closed-book variant (pass --subset rc.nocontext).
"""
from ..sample import Sample
from ..registry import register_dataset
@ -9,21 +14,32 @@ from ..spec import DatasetSpec
DatasetSpec(
name='trivia_qa',
source='mandarjoshi/trivia_qa', # official: https://huggingface.co/datasets/mandarjoshi/trivia_qa
subset='rc.nocontext',
subset='rc.wikipedia', # open-book (evalscope parity); --subset rc.nocontext for closed
split='validation',
task_type='qa',
tags=['knowledge', 'openqa'],
description='TriviaQA open-domain QA without context (official).',
description='TriviaQA with Wikipedia evidence (open-book); any alias counts.',
)
)
def trivia_qa():
def to_sample(record: dict) -> Sample:
answer = record['answer'] # {'value': ..., 'aliases': [...], ...}
targets = [answer['value']] + list(answer.get('aliases') or [])
# open-book: the Wikipedia evidence document (runner prepends it via
# metadata['context'] when assembling the prompt)
wiki = ''
entity = record.get('entity_pages') or {}
for doc in (entity.get('wiki_content') or [])[:1]:
wiki = doc or ''
break
search = record.get('search_results') or {}
if not wiki:
wiki = '\n'.join((search.get('search_context') or [])[:2])
return Sample(
input=record['question'],
target=targets, # multi-target: any alias counts
metadata={'question_id': record.get('question_id')},
metadata={'question_id': record.get('question_id'),
'context': wiki or None},
)
return to_sample

45
evalharness/model/adapter.py
View File

@ -106,6 +106,25 @@ def _key_for(api_base: str) -> str:
# --------------------------- openai_compatible ---------------------------
def _parse_text_tool_calls(text: str) -> list:
"""Extract [{"name":..,"arguments":{..}}] JSON from a text-protocol reply."""
import re as _re
candidates = _re.findall(r'\[[\s\S]*?\]', text) or []
for cand in candidates:
try:
arr = json.loads(cand)
if isinstance(arr, list) and arr and all(isinstance(x, dict) and 'name' in x for x in arr):
return [{'id': '', 'type': 'function',
'function': {'name': x['name'],
'arguments': json.dumps(x.get('arguments', {}))}}
for x in arr]
except (ValueError, TypeError):
continue
return []
@register_adapter('openai')
class OpenAICompatible(ModelAdapter):
"""Async OpenAI chat-completions client with zero hard dependencies.
@ -124,7 +143,7 @@ class OpenAICompatible(ModelAdapter):
headers = {'Content-Type': 'application/json'}
if self.api_key:
headers['Authorization'] = f'Bearer {self.api_key}'
retries = self.extra.get('retries', 3)
retries = self.extra.get('retries', 6)
last_exc: Exception = None
for attempt in range(retries + 1):
try:
@ -141,27 +160,45 @@ class OpenAICompatible(ModelAdapter):
raise
import asyncio
await asyncio.sleep(min(2 ** attempt * 2, 30))
await asyncio.sleep(min(2 ** attempt * 3, 120))
raise last_exc # unreachable
def _payload(self, messages, tools, kw) -> Dict[str, Any]:
msgs = [{'role': m.role, 'content': m.content} for m in messages]
payload: Dict[str, Any] = {'model': self.model, 'messages': msgs}
if tools:
if self.extra.get('tools_mode', 'api') == 'text':
# TEXT-PROTOCOL fallback for backends without --enable-auto-tool-choice:
# declare tools in the prompt; the model outputs JSON calls as text
decl = '\n\n'.join(
f"- {t.get('name')}: {t.get('description', '')} args={t.get('parameters', {})}"
for t in tools)
payload['messages'][-1]['content'] += (
'\n\nYou can call these functions:\n' + decl
+ '\nTo call, output ONLY a JSON array like '
'[{"name": "...", "arguments": {...}}] and nothing else.')
else:
payload['tools'] = [
{'type': 'function', 'function': t} if 'function' not in t else t for t in tools
]
for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed', 'response_format'):
payload.pop('chat_template_kwargs', None)
for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed',
'response_format', 'chat_template_kwargs'):
if kw.get(k) is not None:
payload[k] = kw[k]
payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096)) # CoT room
if self.extra.get('no_think'): # Qwen3-style: disable thinking at the adapter level
payload.setdefault('chat_template_kwargs', {'enable_thinking': False})
return payload
def _parse(self, data: Dict[str, Any]) -> ModelOutput:
choice = (data.get('choices') or [{}])[0]
msg = choice.get('message') or {}
calls = []
for c in msg.get('tool_calls') or []:
raw_calls = list(msg.get('tool_calls') or [])
if not raw_calls and self.extra.get('tools_mode') == 'text':
raw_calls = _parse_text_tool_calls(msg.get('content') or '')
for c in raw_calls:
fn = c.get('function') or {}
args = fn.get('arguments') or '{}'
try:

58
evalharness/model/pool.py Normal file
View File

@ -0,0 +1,58 @@
"""Multi-endpoint load balancing: one logical model, N local ports.
from evalharness.model.pool import PooledAdapter
from evalharness.model.adapter import resolve_adapter
base = resolve_adapter('openai/http://127.0.0.1:8123/v1?Qwen3-8B')
pool = PooledAdapter([resolve_adapter(f'openai/http://127.0.0.1:{p}/v1?Qwen3-8B')
for p in range(8123, 8131)])
out = await pool.generate(...) # round-robin over instances
"""
import itertools
from typing import List, Optional
from ..data.sample import ChatMessage
from .adapter import ModelAdapter
from .output import ModelOutput, Usage
class PooledAdapter(ModelAdapter):
"""Round-robin over N equivalent backend instances."""
name = 'pool'
def __init__(self, adapters: List[ModelAdapter]):
if not adapters:
raise ValueError('PooledAdapter needs at least one backend')
super().__init__(model=adapters[0].model, api_base=adapters[0].api_base)
self.adapters = adapters
self._cycle = itertools.cycle(range(len(adapters)))
self.usage = Usage()
def _next(self) -> ModelAdapter:
return self.adapters[next(self._cycle)]
async def generate(self, messages: List[ChatMessage],
tools: Optional[list] = None, **kw) -> ModelOutput:
last_exc = None
for _ in range(len(self.adapters)): # try each instance once
adapter = self._next()
try:
out = await adapter.generate(messages, tools=tools, **kw)
self.usage = self.usage + out.usage
return out
except Exception as e: # dead instance -> next
last_exc = e
raise last_exc
async def close(self) -> None:
for a in self.adapters:
await a.close()
def pooled(specs: List[str]) -> PooledAdapter:
"""['openai/http://127.0.0.1:8123/v1?M', ...] -> PooledAdapter."""
from .adapter import resolve_adapter
return PooledAdapter([resolve_adapter(s) for s in specs])

64
evalharness/model/runner.py
View File

@ -91,6 +91,23 @@ async def generate_predictions(
question = (f'{question}\n\n{opts}\n\n'
'Answer with the letter of the correct option.')
elif sample.task_type in ('qa',):
# hle OFFICIAL protocol: answer_type-specific system contract
at = (sample.metadata or {}).get('answer_type')
if at == 'exactMatch':
question = (
'Your response should be in the following format:\n'
'Explanation: {your explanation for your final answer}\n'
'Exact Answer: {your succinct, final answer}\n'
'Confidence: {your confidence score between 0% and 100% for your answer}\n\n'
f'{question}')
elif at == 'multipleChoice':
question = (
'Your response should be in the following format:\n'
'Explanation: {your explanation for your answer choice}\n'
'Answer: {your chosen answer}\n'
'Confidence: {your confidence score between 0% and 100% for your answer}\n\n'
f'{question}')
else:
question = (f'{question}\n\n'
'End your reply with the final answer on its own last line '
'in the form "Answer: <answer>".')
@ -360,18 +377,47 @@ async def run_eval(
def _make_adapter(spec: str) -> ModelAdapter:
"""'mock:boxed' -> MockAdapter(mode='boxed'); else resolve_adapter().
The colon-mode syntax exists ONLY for 'mock': adapter names contain no
scheme/colon, so 'mock:xxx' is safe while URLs ('openai/http://...')
must never be split on ':'.
"""Model spec forms:
- 'mock[:mode]' offline adapter
- 'openai-pool/<base-url-template>?model' with {port} placeholder:
e.g. 'openai-pool/http://127.0.0.1:{8123..8130}/v1?Qwen3-8B' -> N ports
- else resolve_adapter(spec) single endpoint
"""
base, sep, mode = spec.partition(':')
if sep and '/' not in base and base == 'mock':
opts = {}
while True:
for f in ('!nothink', '!textools'):
if spec.endswith(f):
spec = spec[:-len(f)]
opts[f] = True
break
else:
break
if spec.startswith('openai-pool/'):
from .pool import pooled
rest = spec[len('openai-pool/'):]
m = __import__('re').search(r'\{(\d+)\.\.(\d+)\}', rest)
if not m:
raise ValueError("openai-pool needs a {start..end} port range")
lo, hi = int(m.group(1)), int(m.group(2))
base_url, _, model = rest.partition('?')
specs = []
for port in range(lo, hi + 1):
specs.append(f'openai/{base_url.replace(m.group(0), str(port))}?{model}')
adapter = pooled(specs)
elif spec.partition(':')[0] == 'mock' and ':' in spec and '/' not in spec.partition(':')[0]:
adapter = resolve_adapter('mock')
adapter.extra['mode'] = mode or 'echo'
adapter.extra['mode'] = spec.partition(':')[2] or 'echo'
return adapter
else:
adapter = resolve_adapter(spec)
members = adapter.adapters if hasattr(adapter, 'adapters') else [adapter]
for a in members:
if opts.get('!nothink'):
a.extra['no_think'] = True
if opts.get('!textools'):
a.extra['tools_mode'] = 'text'
return adapter
return resolve_adapter(spec)
def _judge_callable(judge_adapter: ModelAdapter):

27
evalharness/sandbox/prefetch.py
View File

@ -16,6 +16,17 @@ from typing import Iterable, List
from ..data.dataset import Dataset
# CN mirrors tried in order before/alongside the daemon's configured mirrors.
# Some namespaces (e.g. swebench/*) are blocked by individual CN mirrors, so we
# fall through: daemon default -> 1ms.run -> baidubce -> sjtug.
_CN_MIRROR_FALLBACKS = [
'{img}', # daemon default (uses its own registry-mirrors config)
'docker.1ms.run/{img}',
'mirror.baidubce.com/{img}',
'docker.mirrors.sjtug.sjtu.edu.cn/{img}',
'hub.rat.dev/{img}',
]
def images_for_dataset(ds: Dataset, limit: int = 0) -> List[str]:
"""Collect distinct sandbox images declared by a dataset's samples."""
@ -82,6 +93,16 @@ def prefetch_images(images: Iterable[str], workers: int = 8) -> List[str]:
def _pull_one(image: str) -> None:
r = subprocess.run(['docker', 'pull', image], capture_output=True, text=True, timeout=3600)
if r.returncode != 0:
raise RuntimeError(r.stderr.strip()[:200])
"""Pull via CN-mirror fallback chain; retag to the canonical name on hit."""
last_err = None
for template in _CN_MIRROR_FALLBACKS:
ref = template.format(img=image)
r = subprocess.run(['docker', 'pull', ref], capture_output=True, text=True,
timeout=3600)
if r.returncode == 0:
if ref != image: # retag the mirrored pull to the canonical name
subprocess.run(['docker', 'tag', ref, image], check=False)
subprocess.run(['docker', 'rmi', ref], check=False)
return
last_err = (ref, (r.stderr or '').strip()[:150])
raise RuntimeError(f'all mirrors failed for {image}: {last_err}')