Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix: max_input_tokens extracted from YAML config and passed as the separate run_eval param (was going into gen_kwargs where truncation never saw it -> raw 128k text sent to gateway -> 400)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 09:28:13 +00:00
parent ff5d40aad5
commit 3f555878b8
1 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
evalharness/cli.py
View File

@ -652,10 +652,15 @@ def _cmd_eval_run(args) -> int:
else: else:
_print_phase(_console, _idx, total_runs, _name, msg) _print_phase(_console, _idx, total_runs, _name, msg)
_gen_kw = {**bench_cfg, **(getattr(args, '_gen_override', {}) or {})} _gen_kw = {**bench_cfg, **(getattr(args, '_gen_override', {}) or {})}
# max_input_tokens must be a SEPARATE run_eval param (it drives
# truncation in assemble(), not a gen_kwarg the adapter sees) --
# extract it from the YAML-derived dict
_mit = _gen_kw.pop('max_input_tokens', 0) or getattr(args, 'max_input_tokens', 0)
report = asyncio.run(run_eval( report = asyncio.run(run_eval(
ds, model_spec, concurrency=args.concurrency, limit=args.limit, ds, model_spec, concurrency=args.concurrency, limit=args.limit,
limit_per_task=args.limit_per_task, limit_per_task=args.limit_per_task,
gen_kwargs=_gen_kw or None, gen_kwargs=_gen_kw or None,
max_input_tokens=_mit,
checkpoint=args.resume, checkpoint=args.resume,
judge_spec=_compose_judge_spec(args), env=args.env, judge_spec=_compose_judge_spec(args), env=args.env,
api_key=getattr(args, 'api_key', ''), api_key=getattr(args, 'api_key', ''),