diff --git a/README.md b/README.md index ec59054..e5077a2 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ evalharness eval run gsm8k \ | `--model NAME` | 服务端模型名(配合 `--api-url`);或直接给完整 spec:`openai/http://h:8000/v1?qwen3-8b` | | `--provider {openai-chat,openai-pool}` | 协议/提供方,默认 `openai-chat`;端点池用 `openai-pool` | | `--judge NAME` + `--judge-api-url URL` | LLM-judge 模型(hle / simple_qa / imo 需要);也可单给完整 spec `--judge openai/http://...?m` | +| `--api-key KEY` | 显式指定主模型端点的 key(优先于环境变量推断;只进请求头,不写入 spec/报告) | | `--judge-provider {openai-chat,openai-pool}` | judge 协议;多 judge 端点负载均衡用 `openai-pool` | +| `--judge-api-key KEY` | 显式指定 judge 端点的 key | | `--profile NAME` | 命名生成参数集(内置 `dp4-nothink`、`qwen3-es-parity`、`t1-short`,或任意 `@register_gen_profile` 名);优先级:插件默认 < profile 默认 < profile 单 bench 覆盖 < 显式参数 | | `--disable-thinking` | 发送 `enable_thinking=false`(Qwen3 类思考模型推荐;带 tools 的请求自动退回模板安全的软开关) | | `--textools` | 工具以文本形式随 prompt 下发,而非原生 tool_calls | @@ -164,6 +166,8 @@ evalharness viz show report.json --style errors # 失败样本下钻 API key(含 judge)按端点域名自动从环境变量读取:`api.openai.com`→`OPENAI_API_KEY`、 `anthropic.com`→`ANTHROPIC_API_KEY`、`dashscope`→`DASHSCOPE_API_KEY`、`bigmodel`→`ZAI_API_KEY`; 其余域名回退 `OPENAI_API_KEY`。自建端点无鉴权时无需设置。 +同一服务有多个 key 时用 `--api-key` 显式指定(跑两次各用一个 key 即可对比;key 只进请求头, +不会出现在 spec、报告或日志里)。 ## 内置 benchmark diff --git a/evalharness/cli.py b/evalharness/cli.py index 5a27a44..c194bf3 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -308,6 +308,8 @@ def _cmd_eval_run(args) -> int: limit_per_task=args.limit_per_task, checkpoint=args.resume, judge_spec=_compose_judge_spec(args), env=args.env, + api_key=getattr(args, 'api_key', ''), + judge_api_key=getattr(args, 'judge_api_key', ''), gen_profile=getattr(args, 'profile', ''), progress_reporter=progress_reporter, status_callback=status_callback)) @@ -524,6 +526,11 @@ def build_parser() -> argparse.ArgumentParser: help='judge model name with --judge-api-url, or full spec') p.add_argument('--judge-api-url', default='', help='judge API base URL when --judge is only the model name') + p.add_argument('--api-key', default='', + help='explicit API key for the model endpoint (overrides ' + 'env-based resolution; never written into reports)') + p.add_argument('--judge-api-key', default='', + help='explicit API key for the judge endpoint') p.add_argument('--judge-provider', default='openai-chat', help='judge protocol/provider (default openai-chat; ' 'openai-pool for multi-endpoint judges)') diff --git a/evalharness/model/pool.py b/evalharness/model/pool.py index 0d85f13..09af0c8 100644 --- a/evalharness/model/pool.py +++ b/evalharness/model/pool.py @@ -245,8 +245,15 @@ class AdaptiveGate: **{f'gate_{k}': v for k, v in self.stats.items()}} -def pooled(specs: List[str]) -> PooledAdapter: - """['openai/http://127.0.0.1:8123/v1?M', ...] -> PooledAdapter.""" +def pooled(specs: List[str], api_key: str = '') -> PooledAdapter: + """['openai/http://127.0.0.1:8123/v1?M', ...] -> PooledAdapter. + + api_key: explicit key applied to EVERY member (two-key setups should + build two pools, or use env resolution per host).""" from .adapter import resolve_adapter - return PooledAdapter([resolve_adapter(s) for s in specs]) + members = [resolve_adapter(s) for s in specs] + if api_key: + for m in members: + m.api_key = api_key + return PooledAdapter(members) diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 1fb04bf..d880ca1 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -502,6 +502,8 @@ async def run_eval( concurrency: int = 32, limit: Optional[int] = None, gen_kwargs: Optional[Dict[str, Any]] = None, + api_key: str = '', + judge_api_key: str = '', judge_spec: Optional[str] = None, judge: Optional[Any] = None, progress: bool = True, @@ -535,7 +537,7 @@ async def run_eval( spec = getattr(dataset, 'spec', None) if few_shot_num < 0: few_shot_num = (spec.few_shot_num if spec is not None else 0) - adapter = _make_adapter(model_spec) + adapter = _make_adapter(model_spec, api_key=api_key) # reports carry a string model label: pre-built adapter objects need one model_spec = model_spec if isinstance(model_spec, str) \ else (getattr(model_spec, 'model', '') or repr(model_spec)) @@ -650,7 +652,7 @@ async def run_eval( if judge is None and judge_spec: if status_callback: status_callback('loading judge model') - judge_adapter = _make_adapter(judge_spec) + judge_adapter = _make_adapter(judge_spec, api_key=judge_api_key or api_key) judge = _judge_callable(judge_adapter) if status_callback: @@ -693,7 +695,7 @@ def _env_user_adapter(spec: str): _ENV_USER_CACHE = {} -def _make_adapter(spec: str) -> ModelAdapter: +def _make_adapter(spec: str, api_key: str = '') -> ModelAdapter: """Model spec forms: - 'mock[:mode]' offline adapter - 'openai-pool/?model' with {port} placeholder: @@ -743,7 +745,7 @@ def _make_adapter(spec: str) -> ModelAdapter: specs.append(f'openai/{u}?{model}') elif seg: specs.append(f'openai/{seg}?{model}') - adapter = pooled(specs) + adapter = pooled(specs, api_key=api_key) if api_key else pooled(specs) elif re.fullmatch(r'mock[-:](boxed|oracle|fc|tool|echo|const)?', spec): # mock-boxed (preferred) == legacy mock:boxed; bare 'mock' == echo. # NEVER reuse the cached singleton: resolve_adapter memoizes and a