max_turns: config-driven (default 200), runner default 8 -> 200

YAML default section and per-bench (tau2_bench: 200) both set it; the
runner's generic 8-starved every agent dialog (tau2 reward-0 root
cause). CLI wiring pops the key into args.max_turns before the
generation keys are stripped.

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/cli.py

@@ -792,8 +792,12 @@ def _cmd_eval_run(args) -> int:
             _env_cfg = bench_cfg.pop('env', '') or ''
             if _env_cfg:
                 args.env = _env_cfg
+            # max_turns: per-bench from YAML (agent dialog step budget)
+            _mt_cfg = bench_cfg.pop('max_turns', None)
+            if _mt_cfg:
+                args.max_turns = int(_mt_cfg)
             # strip non-generation keys (they go to run_eval kwargs)
-            for k in ('judge', 'judge_url', 'max_turns',
+            for k in ('judge', 'judge_url',
                       'limit', 'limit_per_task', 'concurrency'):
                 bench_cfg.pop(k, None)
 

evalharness/config/default.yaml

@@ -3,6 +3,7 @@ default:
   top_p: 1.0
   stream: true
   max_tokens: 32768
+  max_turns: 200
 aime24:
   temperature: 1.0
   repeats: 12
@@ -60,3 +61,4 @@ general_fc:
 tau2_bench:
   max_tokens: 16384
   env: tau2_official      # agent 模式：官方引擎
+  max_turns: 200          # 一次完整电话流程（问候→验证→找单→政策→执行→确认）

evalharness/model/runner.py

@@ -38,7 +38,7 @@ async def generate_predictions(
     env_user_spec: str = '',
     no_shuffle: bool = False,
     system: str = '',
-    max_turns: int = 8,
+    max_turns: int = 200,
     max_input_chars: int = 0,
     max_input_tokens: int = 0,
     tokenizer_path: str = '',
@@ -657,7 +657,7 @@ async def run_eval(
     env_user_spec: str = '',
     no_shuffle: bool = False,  # fixed-order selection: raw first-N (same-questions parity)
     system: str = '',
-    max_turns: int = 8,
+    max_turns: int = 200,
     max_input_chars: int = 0,
     max_input_tokens: int = 0,
     limit_per_task: Optional[int] = None,