From 5c14d118527a7137964f8e5f4834b7509491cb42 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 17 Sep 2026 08:33:08 +0000 Subject: [PATCH] env per-bench from YAML config; swe image naming to official repo-base - config env entries: bfcl_v3->bfcl_mock, tau2_bench->tau2_official; the eval loop previously POPPED the env key and discarded it -- now it feeds args.env (CLI --env still works as default/fallback) - swe_bench_verified images: per-instance names are never published; the official docker.io layout is repo-level BASE images (swebench/sweb.eval.x86_64.{repo}) with per-instance images built on top -- naming corrected (note: the swebench/* namespace is currently blocked on every reachable CN mirror all the same) Co-Authored-By: Claude --- evalharness/cli.py | 6 +++++- evalharness/config/default.yaml | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/evalharness/cli.py b/evalharness/cli.py index 000ede1..a5df6ae 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -788,8 +788,12 @@ def _cmd_eval_run(args) -> int: # YAML config: per-bench generation params, AUTO-LOADED # (single .yaml in config/ = the default; --config overrides) bench_cfg = _load_bench_cfg(args, name) + # env: per-bench from YAML, CLI --env as fallback/default + _env_cfg = bench_cfg.pop('env', '') or '' + if _env_cfg: + args.env = _env_cfg # strip non-generation keys (they go to run_eval kwargs) - for k in ('judge', 'judge_url', 'env', 'max_turns', + for k in ('judge', 'judge_url', 'max_turns', 'limit', 'limit_per_task', 'concurrency'): bench_cfg.pop(k, None) diff --git a/evalharness/config/default.yaml b/evalharness/config/default.yaml index 219a3f4..3ce0253 100644 --- a/evalharness/config/default.yaml +++ b/evalharness/config/default.yaml @@ -54,7 +54,9 @@ openai_mrcr: max_input_tokens: 128000 bfcl_v3: max_tokens: 4096 + env: bfcl_mock # agent 模式：工具调用轨迹 + 官方 AST 判分 general_fc: max_tokens: 4096 tau2_bench: max_tokens: 16384 + env: tau2_official # agent 模式：官方引擎