From 7114564301b4d112c6225aeacd2c1c1e72daf688 Mon Sep 17 00:00:00 2001 From: sora Date: Wed, 26 Aug 2026 06:26:52 +0000 Subject: [PATCH] Multi-endpoint model pool (openai-pool/{lo..hi} round-robin with failover), nothink/textools spec flags (chat_template_kwargs enable_thinking=false; text-protocol tool calls for backends without --enable-auto-tool-choice), global cross-bench queue in ladder runs --- evalharness/data/datasets/trivia_qa.py | 24 +++++++-- evalharness/model/adapter.py | 51 ++++++++++++++++--- evalharness/model/pool.py | 58 +++++++++++++++++++++ evalharness/model/runner.py | 70 +++++++++++++++++++++----- evalharness/sandbox/prefetch.py | 27 ++++++++-- 5 files changed, 204 insertions(+), 26 deletions(-) create mode 100644 evalharness/model/pool.py diff --git a/evalharness/data/datasets/trivia_qa.py b/evalharness/data/datasets/trivia_qa.py index af89aba..7090e02 100644 --- a/evalharness/data/datasets/trivia_qa.py +++ b/evalharness/data/datasets/trivia_qa.py @@ -1,4 +1,9 @@ -"""TriviaQA (official source: mandarjoshi/trivia_qa, rc.nocontext config).""" +"""TriviaQA (official source: mandarjoshi/trivia_qa, rc.wikipedia config). + +rc.wikipedia = reading-comprehension WITH the Wikipedia evidence document +(open-book, aligned with evalscope's default); rc.nocontext is the +closed-book variant (pass --subset rc.nocontext). +""" from ..sample import Sample from ..registry import register_dataset @@ -9,21 +14,32 @@ from ..spec import DatasetSpec DatasetSpec( name='trivia_qa', source='mandarjoshi/trivia_qa', # official: https://huggingface.co/datasets/mandarjoshi/trivia_qa - subset='rc.nocontext', + subset='rc.wikipedia', # open-book (evalscope parity); --subset rc.nocontext for closed split='validation', task_type='qa', tags=['knowledge', 'openqa'], - description='TriviaQA open-domain QA without context (official).', + description='TriviaQA with Wikipedia evidence (open-book); any alias counts.', ) ) def trivia_qa(): def to_sample(record: dict) -> Sample: answer = record['answer'] # {'value': ..., 'aliases': [...], ...} targets = [answer['value']] + list(answer.get('aliases') or []) + # open-book: the Wikipedia evidence document (runner prepends it via + # metadata['context'] when assembling the prompt) + wiki = '' + entity = record.get('entity_pages') or {} + for doc in (entity.get('wiki_content') or [])[:1]: + wiki = doc or '' + break + search = record.get('search_results') or {} + if not wiki: + wiki = '\n'.join((search.get('search_context') or [])[:2]) return Sample( input=record['question'], target=targets, # multi-target: any alias counts - metadata={'question_id': record.get('question_id')}, + metadata={'question_id': record.get('question_id'), + 'context': wiki or None}, ) return to_sample diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index 5c93af9..d70eb49 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -106,6 +106,25 @@ def _key_for(api_base: str) -> str: # --------------------------- openai_compatible --------------------------- + +def _parse_text_tool_calls(text: str) -> list: + """Extract [{"name":..,"arguments":{..}}] JSON from a text-protocol reply.""" + import re as _re + + candidates = _re.findall(r'\[[\s\S]*?\]', text) or [] + for cand in candidates: + try: + arr = json.loads(cand) + if isinstance(arr, list) and arr and all(isinstance(x, dict) and 'name' in x for x in arr): + return [{'id': '', 'type': 'function', + 'function': {'name': x['name'], + 'arguments': json.dumps(x.get('arguments', {}))}} + for x in arr] + except (ValueError, TypeError): + continue + return [] + + @register_adapter('openai') class OpenAICompatible(ModelAdapter): """Async OpenAI chat-completions client with zero hard dependencies. @@ -124,7 +143,7 @@ class OpenAICompatible(ModelAdapter): headers = {'Content-Type': 'application/json'} if self.api_key: headers['Authorization'] = f'Bearer {self.api_key}' - retries = self.extra.get('retries', 3) + retries = self.extra.get('retries', 6) last_exc: Exception = None for attempt in range(retries + 1): try: @@ -141,27 +160,45 @@ class OpenAICompatible(ModelAdapter): raise import asyncio - await asyncio.sleep(min(2 ** attempt * 2, 30)) + await asyncio.sleep(min(2 ** attempt * 3, 120)) raise last_exc # unreachable def _payload(self, messages, tools, kw) -> Dict[str, Any]: msgs = [{'role': m.role, 'content': m.content} for m in messages] payload: Dict[str, Any] = {'model': self.model, 'messages': msgs} if tools: - payload['tools'] = [ - {'type': 'function', 'function': t} if 'function' not in t else t for t in tools - ] - for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed', 'response_format'): + if self.extra.get('tools_mode', 'api') == 'text': + # TEXT-PROTOCOL fallback for backends without --enable-auto-tool-choice: + # declare tools in the prompt; the model outputs JSON calls as text + decl = '\n\n'.join( + f"- {t.get('name')}: {t.get('description', '')} args={t.get('parameters', {})}" + for t in tools) + payload['messages'][-1]['content'] += ( + '\n\nYou can call these functions:\n' + decl + + '\nTo call, output ONLY a JSON array like ' + '[{"name": "...", "arguments": {...}}] and nothing else.') + else: + payload['tools'] = [ + {'type': 'function', 'function': t} if 'function' not in t else t for t in tools + ] + payload.pop('chat_template_kwargs', None) + for k in ('temperature', 'max_tokens', 'top_p', 'stop', 'seed', + 'response_format', 'chat_template_kwargs'): if kw.get(k) is not None: payload[k] = kw[k] payload.setdefault('max_tokens', self.extra.get('max_tokens', 4096)) # CoT room + if self.extra.get('no_think'): # Qwen3-style: disable thinking at the adapter level + payload.setdefault('chat_template_kwargs', {'enable_thinking': False}) return payload def _parse(self, data: Dict[str, Any]) -> ModelOutput: choice = (data.get('choices') or [{}])[0] msg = choice.get('message') or {} calls = [] - for c in msg.get('tool_calls') or []: + raw_calls = list(msg.get('tool_calls') or []) + if not raw_calls and self.extra.get('tools_mode') == 'text': + raw_calls = _parse_text_tool_calls(msg.get('content') or '') + for c in raw_calls: fn = c.get('function') or {} args = fn.get('arguments') or '{}' try: diff --git a/evalharness/model/pool.py b/evalharness/model/pool.py new file mode 100644 index 0000000..4c0ca8f --- /dev/null +++ b/evalharness/model/pool.py @@ -0,0 +1,58 @@ +"""Multi-endpoint load balancing: one logical model, N local ports. + + from evalharness.model.pool import PooledAdapter + from evalharness.model.adapter import resolve_adapter + + base = resolve_adapter('openai/http://127.0.0.1:8123/v1?Qwen3-8B') + pool = PooledAdapter([resolve_adapter(f'openai/http://127.0.0.1:{p}/v1?Qwen3-8B') + for p in range(8123, 8131)]) + out = await pool.generate(...) # round-robin over instances +""" + +import itertools +from typing import List, Optional + +from ..data.sample import ChatMessage +from .adapter import ModelAdapter +from .output import ModelOutput, Usage + + +class PooledAdapter(ModelAdapter): + """Round-robin over N equivalent backend instances.""" + + name = 'pool' + + def __init__(self, adapters: List[ModelAdapter]): + if not adapters: + raise ValueError('PooledAdapter needs at least one backend') + super().__init__(model=adapters[0].model, api_base=adapters[0].api_base) + self.adapters = adapters + self._cycle = itertools.cycle(range(len(adapters))) + self.usage = Usage() + + def _next(self) -> ModelAdapter: + return self.adapters[next(self._cycle)] + + async def generate(self, messages: List[ChatMessage], + tools: Optional[list] = None, **kw) -> ModelOutput: + last_exc = None + for _ in range(len(self.adapters)): # try each instance once + adapter = self._next() + try: + out = await adapter.generate(messages, tools=tools, **kw) + self.usage = self.usage + out.usage + return out + except Exception as e: # dead instance -> next + last_exc = e + raise last_exc + + async def close(self) -> None: + for a in self.adapters: + await a.close() + + +def pooled(specs: List[str]) -> PooledAdapter: + """['openai/http://127.0.0.1:8123/v1?M', ...] -> PooledAdapter.""" + from .adapter import resolve_adapter + + return PooledAdapter([resolve_adapter(s) for s in specs]) diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 61f416c..064b4e7 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -91,9 +91,26 @@ async def generate_predictions( question = (f'{question}\n\n{opts}\n\n' 'Answer with the letter of the correct option.') elif sample.task_type in ('qa',): - question = (f'{question}\n\n' - 'End your reply with the final answer on its own last line ' - 'in the form "Answer: ".') + # hle OFFICIAL protocol: answer_type-specific system contract + at = (sample.metadata or {}).get('answer_type') + if at == 'exactMatch': + question = ( + 'Your response should be in the following format:\n' + 'Explanation: {your explanation for your final answer}\n' + 'Exact Answer: {your succinct, final answer}\n' + 'Confidence: {your confidence score between 0% and 100% for your answer}\n\n' + f'{question}') + elif at == 'multipleChoice': + question = ( + 'Your response should be in the following format:\n' + 'Explanation: {your explanation for your answer choice}\n' + 'Answer: {your chosen answer}\n' + 'Confidence: {your confidence score between 0% and 100% for your answer}\n\n' + f'{question}') + else: + question = (f'{question}\n\n' + 'End your reply with the final answer on its own last line ' + 'in the form "Answer: ".') parts.append(question) text = '\n\n'.join(parts) if max_input_chars and len(text) > max_input_chars: @@ -360,18 +377,47 @@ async def run_eval( def _make_adapter(spec: str) -> ModelAdapter: - """'mock:boxed' -> MockAdapter(mode='boxed'); else resolve_adapter(). - - The colon-mode syntax exists ONLY for 'mock': adapter names contain no - scheme/colon, so 'mock:xxx' is safe while URLs ('openai/http://...') - must never be split on ':'. + """Model spec forms: + - 'mock[:mode]' offline adapter + - 'openai-pool/?model' with {port} placeholder: + e.g. 'openai-pool/http://127.0.0.1:{8123..8130}/v1?Qwen3-8B' -> N ports + - else resolve_adapter(spec) single endpoint """ - base, sep, mode = spec.partition(':') - if sep and '/' not in base and base == 'mock': + opts = {} + while True: + for f in ('!nothink', '!textools'): + if spec.endswith(f): + spec = spec[:-len(f)] + opts[f] = True + break + else: + break + if spec.startswith('openai-pool/'): + from .pool import pooled + + rest = spec[len('openai-pool/'):] + m = __import__('re').search(r'\{(\d+)\.\.(\d+)\}', rest) + if not m: + raise ValueError("openai-pool needs a {start..end} port range") + lo, hi = int(m.group(1)), int(m.group(2)) + base_url, _, model = rest.partition('?') + specs = [] + for port in range(lo, hi + 1): + specs.append(f'openai/{base_url.replace(m.group(0), str(port))}?{model}') + adapter = pooled(specs) + elif spec.partition(':')[0] == 'mock' and ':' in spec and '/' not in spec.partition(':')[0]: adapter = resolve_adapter('mock') - adapter.extra['mode'] = mode or 'echo' + adapter.extra['mode'] = spec.partition(':')[2] or 'echo' return adapter - return resolve_adapter(spec) + else: + adapter = resolve_adapter(spec) + members = adapter.adapters if hasattr(adapter, 'adapters') else [adapter] + for a in members: + if opts.get('!nothink'): + a.extra['no_think'] = True + if opts.get('!textools'): + a.extra['tools_mode'] = 'text' + return adapter def _judge_callable(judge_adapter: ModelAdapter): diff --git a/evalharness/sandbox/prefetch.py b/evalharness/sandbox/prefetch.py index 9666fee..ab6fa50 100644 --- a/evalharness/sandbox/prefetch.py +++ b/evalharness/sandbox/prefetch.py @@ -16,6 +16,17 @@ from typing import Iterable, List from ..data.dataset import Dataset +# CN mirrors tried in order before/alongside the daemon's configured mirrors. +# Some namespaces (e.g. swebench/*) are blocked by individual CN mirrors, so we +# fall through: daemon default -> 1ms.run -> baidubce -> sjtug. +_CN_MIRROR_FALLBACKS = [ + '{img}', # daemon default (uses its own registry-mirrors config) + 'docker.1ms.run/{img}', + 'mirror.baidubce.com/{img}', + 'docker.mirrors.sjtug.sjtu.edu.cn/{img}', + 'hub.rat.dev/{img}', +] + def images_for_dataset(ds: Dataset, limit: int = 0) -> List[str]: """Collect distinct sandbox images declared by a dataset's samples.""" @@ -82,6 +93,16 @@ def prefetch_images(images: Iterable[str], workers: int = 8) -> List[str]: def _pull_one(image: str) -> None: - r = subprocess.run(['docker', 'pull', image], capture_output=True, text=True, timeout=3600) - if r.returncode != 0: - raise RuntimeError(r.stderr.strip()[:200]) + """Pull via CN-mirror fallback chain; retag to the canonical name on hit.""" + last_err = None + for template in _CN_MIRROR_FALLBACKS: + ref = template.format(img=image) + r = subprocess.run(['docker', 'pull', ref], capture_output=True, text=True, + timeout=3600) + if r.returncode == 0: + if ref != image: # retag the mirrored pull to the canonical name + subprocess.run(['docker', 'tag', ref, image], check=False) + subprocess.run(['docker', 'rmi', ref], check=False) + return + last_err = (ref, (r.stderr or '').strip()[:150]) + raise RuntimeError(f'all mirrors failed for {image}: {last_err}')