diff --git a/README.md b/README.md index 4fc3c9f..33aa2c9 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ pip install . 离线验证安装(不需要模型、不联网): ```bash -evalharness eval run gsm8k --model mock:boxed --limit 8 +evalharness eval run gsm8k --model mock-boxed --limit 8 # acc 100% —— mock 适配器直接输出金标答案,证明 # 数据 → prompt → 生成 → 判分 → 报告 全链路可用 ``` @@ -68,7 +68,7 @@ evalharness eval run gsm8k \ | `--api-url URL` | OpenAI 兼容端点;与 `--model` 搭配使用(不用手拼 spec 字符串) | | `--model NAME` | 服务端模型名(配合 `--api-url`);或直接给完整 spec:`openai/http://h:8000/v1?qwen3-8b` | | `--provider {openai-chat,openai-pool}` | 协议/提供方,默认 `openai-chat`;端点池用 `openai-pool` | -| `--judge SPEC` | LLM-judge 模型 spec(hle / simple_qa / imo 等 judge 类 recipe 需要) | +| `--judge NAME` + `--judge-api-url URL` | LLM-judge 模型(hle / simple_qa / imo 需要);也可单给完整 spec `--judge openai/http://...?m` | | `--profile NAME` | 命名生成参数集(内置 `dp4-nothink`、`qwen3-es-parity`、`t1-short`,或任意 `@register_gen_profile` 名);优先级:插件默认 < profile 默认 < profile 单 bench 覆盖 < 显式参数 | | `--disable-thinking` | 发送 `enable_thinking=false`(Qwen3 类思考模型推荐;带 tools 的请求自动退回模板安全的软开关) | | `--textools` | 工具以文本形式随 prompt 下发,而非原生 tool_calls | @@ -157,7 +157,7 @@ evalharness viz show report.json --style errors # 失败样本下钻 | `openai/?` | 单端点(vLLM、SGLang、lmdeploy、ollama、云 API) | | `openai-pool/?` | 端点池:轮询 + 自适应并发 + failover | | `deploy:/` | 经 Deployer 解析(钉版本的推理环境) | -| `mock` / `mock:boxed` / `mock:fc` | 离线适配器(管线自检) | +| `mock` / `mock-boxed` / `mock-fc` | 离线适配器(管线自检:回声 / 回放金标 / 回放工具调用) | | `!nothink` `!perf` `!textools` 后缀 | spec 内联开关(与 CLI 参数等价) | API key 按端点从环境变量读取(`OPENAI_API_KEY`、`ANTHROPIC_API_KEY` …)。 @@ -180,7 +180,7 @@ evalharness eval list # 全部判分 recipe 各族注意事项: -- **LLM-judge 类**(`hle`、`simple_qa`、`imo_answerbench`):传 `--judge `;judge 走官方协议 +- **LLM-judge 类**(`hle`、`simple_qa`、`imo_answerbench`):传 `--judge <模型名> --judge-api-url <端点>`(与主模型同款分离参数风格);judge 走官方协议 (如 SimpleQA 的分级正确性 + NOT_ATTEMPTED 兜底)。 - **代码执行类**:模型生成的代码在硬隔离 Docker 中运行(`--network none`、cgroup 上限、只读 rootfs); swe 的逐实例 `sweb.eval.*` 镜像用 `evalharness sandbox prefetch swe_bench_verified` 预取。 diff --git a/evalharness/cli.py b/evalharness/cli.py index 1374a45..a306740 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -228,6 +228,26 @@ def _compose_model_spec(args): return _model_with_flags(model, args) +def _compose_judge_spec(args): + """--judge accepts a bare model name (with --judge-api-url) or a full + legacy spec; keep both working like the main model flags.""" + judge = getattr(args, 'judge', '') or '' + url = getattr(args, 'judge_api_url', '') or '' + if url and judge and '/' not in judge: + judge = f'openai/{url.rstrip("/")}?{judge}' + return judge or None + + +def _compose_judge_spec(args): + """--judge accepts a bare model name (with --judge-api-url) or a full + legacy spec; both keep working, mirroring the main model flags.""" + judge = getattr(args, 'judge', '') or '' + url = getattr(args, 'judge_api_url', '') or '' + if url and judge and '/' not in judge: + judge = f'openai/{url.rstrip("/")}?{judge}' + return judge or None + + def _cmd_eval_run(args) -> int: import asyncio import time as _time @@ -283,7 +303,7 @@ def _cmd_eval_run(args) -> int: ds, model_spec, concurrency=args.concurrency, limit=args.limit, limit_per_task=args.limit_per_task, checkpoint=args.resume, - judge_spec=args.judge, env=args.env, + judge_spec=_compose_judge_spec(args), env=args.env, gen_profile=getattr(args, 'profile', ''), progress_reporter=progress_reporter, status_callback=status_callback)) @@ -496,7 +516,10 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument('--provider', default='openai-chat', choices=('openai-chat', 'openai-pool'), help='API protocol/provider (default: openai-chat)') - p.add_argument('--judge', default='', help='judge model spec for llm_judge recipes') + p.add_argument('--judge', default='', + help='judge model name with --judge-api-url, or full spec') + p.add_argument('--judge-api-url', default='', + help='judge API base URL when --judge is only the model name') p.add_argument('--profile', default='', help='named gen-params profile (dp4-nothink | qwen3-es-parity | t1-short ' 'or any @register_gen_profile name); layers: plugin default < ' diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 968369c..1fb04bf 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -12,6 +12,7 @@ raw strings to the sync evaluate(). import asyncio import os +import re import time from typing import Any, Dict, List, Optional, Union @@ -743,9 +744,15 @@ def _make_adapter(spec: str) -> ModelAdapter: elif seg: specs.append(f'openai/{seg}?{model}') adapter = pooled(specs) - elif spec.partition(':')[0] == 'mock' and ':' in spec and '/' not in spec.partition(':')[0]: - adapter = resolve_adapter('mock') - adapter.extra['mode'] = spec.partition(':')[2] or 'echo' + elif re.fullmatch(r'mock[-:](boxed|oracle|fc|tool|echo|const)?', spec): + # mock-boxed (preferred) == legacy mock:boxed; bare 'mock' == echo. + # NEVER reuse the cached singleton: resolve_adapter memoizes and a + # shared instance would leak this run's mode into the next one + mode = re.fullmatch(r'mock[-:]?(.*)', spec).group(1) or 'echo' + from .adapter import ADAPTER_REGISTRY + + adapter = ADAPTER_REGISTRY.get('mock')(model='mock', api_base='') + adapter.extra['mode'] = mode return adapter else: adapter = resolve_adapter(spec)