From 7921688149686cf336dac3524b82d4d5603f2583 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Mon, 14 Sep 2026 09:11:07 +0000 Subject: [PATCH] Run plan: concrete sample counts + 'auto' concurrency display/alias - sample-counts manifest (config/sample_counts.yaml, harvested from real runs): uncached benches still show exact numbers in the plan instead of 'counts when datasets load' -- 'cache+est.' marks the mix - '--concurrency auto' is now an alias for --auto-concurrency - Concurrency row shows 'auto (start 8, gate decides)' when the gate drives, instead of a bare misleading 8 Also verified end-to-end: thinking-mode humaneval rep1/rep2 both pass 98.8%, matching the es reference runs (98.17/98.78/98.78) on the same model -- framework alignment holds on the thinking path too. Co-Authored-By: Claude --- evalharness/cli.py | 79 +++++++++++++++++++-------- evalharness/config/sample_counts.yaml | 30 ++++++++++ 2 files changed, 85 insertions(+), 24 deletions(-) create mode 100644 evalharness/config/sample_counts.yaml diff --git a/evalharness/cli.py b/evalharness/cli.py index 03da0e3..a7b1958 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -193,25 +193,38 @@ def _load_bench_cfg(args, name: str) -> dict: def _plan_sample_counts(args): """(total_samples, total_generations, n_uncached) across the planned - benches, counted from LOCAL cache entries only -- never touches the - network, so the run plan stays instant on cold machines. Uncached - benches simply don't contribute yet. + benches. Cache-first (exact); uncached benches fall back to the shipped + sample-counts manifest so the plan still shows CONCRETE numbers on a + cold machine instead of 'counts when datasets load'. """ + from pathlib import Path + from evalharness.data import get_dataset + manifest = {} + mf = Path(__file__).parent / 'config' / 'sample_counts.yaml' + if mf.exists(): + try: + import yaml as _yaml + + manifest = _yaml.safe_load(open(mf)) or {} + except Exception: + manifest = {} + total = gens = uncached = 0 for name in getattr(args, 'datasets', []) or []: + n = None try: ds = get_dataset(name, **_overrides(args)) cache_file = ds.cache_dir / 'samples.jsonl' - if not cache_file.exists(): - uncached += 1 - continue - with open(cache_file, 'rb') as f: - n = sum(1 for _ in f) + if cache_file.exists(): + with open(cache_file, 'rb') as f: + n = sum(1 for _ in f) except Exception: + n = None + if n is None: uncached += 1 - continue + n = int(manifest.get(name, 0) or 0) # estimate; 0 = unknown if getattr(args, 'limit', None): n = min(n, args.limit) total += n @@ -232,21 +245,23 @@ def _print_run_plan(console, args, model_spec): cap = f' · ≤{args.limit} per bench (--limit)' elif getattr(args, 'limit_per_task', None): cap = f' · ≤{args.limit_per_task} per subject (--limit-per-task)' + src = 'cached' if not n_uncached else ('cache+est.' if n_samples else 'est.') if n_samples: - samples = f'{n_samples:,} samples (cached){cap}' + samples = f'{n_samples:,} samples ({src}){cap}' if n_gens > n_samples: # repeats multiply the real work - samples = (f'{n_samples:,} samples (cached){cap} → ' + samples = (f'{n_samples:,} samples ({src}){cap} → ' f'{n_gens:,} generations (repeats)') - if n_uncached: - samples += f' · {n_uncached} bench(es) not cached yet' - elif n_uncached: - if getattr(args, 'limit', None): - samples = f'up to {args.limit} per bench (--limit), counts when datasets load' - elif getattr(args, 'limit_per_task', None): - samples = (f'up to {args.limit_per_task} per subject ' - '(--limit-per-task), counts when datasets load') - else: - samples = 'full dataset, counts when datasets load (none cached yet)' + elif getattr(args, 'limit', None): + samples = f'up to {args.limit} per bench (--limit)' + elif getattr(args, 'limit_per_task', None): + samples = f'up to {args.limit_per_task} per subject (--limit-per-task)' + else: + samples = '? (unknown benchmarks)' + if n_uncached: + samples += f' · {n_uncached} bench(es) not cached yet' + _auto = getattr(args, 'auto_concurrency', False) + _conc = (f'auto (start {args.concurrency}, gate decides)' if _auto + else str(args.concurrency)) if console is None: print(f'=== {title} ===') print(f'Provider: {provider}') @@ -254,7 +269,7 @@ def _print_run_plan(console, args, model_spec): print(f'Model: {model_name}') print(f'Benchmarks: {len(args.datasets)} -> {", ".join(args.datasets)}') print(f'Samples: {samples}') - print(f'Concurrency: {args.concurrency} | Thinking: ' + print(f'Concurrency: {_conc} | Thinking: ' f'{"enabled" if not args.disable_thinking else "disabled"} | ' f'Performance: {"on" if args.perf else "off"}') print(f'Resume: {"on" if args.resume else "off"} | Output: {args.out_dir or "(none)"}') @@ -271,7 +286,7 @@ def _print_run_plan(console, args, model_spec): table.add_row('Model', model_name) table.add_row('Benchmarks', f'{len(args.datasets)} · {", ".join(args.datasets)}') table.add_row('Samples', samples) - table.add_row('Concurrency', str(args.concurrency)) + table.add_row('Concurrency', f'[magenta]{_conc}[/magenta]' if _auto else _conc) table.add_row('Thinking', '[red]disabled[/red]' if args.disable_thinking else '[green]enabled[/green]') table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]') table.add_row('Checkpoint', '[green]resume[/green]' if args.resume else '[dim]new run[/dim]') @@ -952,6 +967,11 @@ def _cmd_eval_run(args) -> int: 'secs': round(_time.time() - t0, 1), 'ok': False, 'err': f'{type(e).__name__}: {str(e)[:100]}'}) print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr) + import traceback as _tb + + _tb.print_exc() # full stack to stderr: the one-line form hides + # WHERE an error like 'Object of type ellipsis is not JSON + # serializable' actually comes from from evalharness.hooks import fire as _fire _fire('on_benchmark_failed', name=name, error=e, dataset=name) @@ -1160,7 +1180,10 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument('--textools', action='store_true', help='send tools as text instead of native tool calls') p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump") - p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)') + p.add_argument('--concurrency', default='32', + help="parallel model calls (default 32); 'auto' = same as " + "--auto-concurrency (the adaptive gate decides, this " + "value becomes its starting point)") p.add_argument('--progress', action='store_true', default=True, help='show per-sample progress (default: on)') p.add_argument('--no-progress', dest='progress', action='store_false', @@ -1225,6 +1248,14 @@ def build_parser() -> argparse.ArgumentParser: def main(argv=None) -> int: args = build_parser().parse_args(argv) + # '--concurrency auto' == '--auto-concurrency': normalize once, here, + # so every downstream site (plan display, run_eval, gate initial) sees + # an int + the flag + if str(getattr(args, 'concurrency', '32')).strip().lower() == 'auto': + args.auto_concurrency = True + args.concurrency = 8 + else: + args.concurrency = int(args.concurrency) return args.func(args) diff --git a/evalharness/config/sample_counts.yaml b/evalharness/config/sample_counts.yaml new file mode 100644 index 0000000..10d2818 --- /dev/null +++ b/evalharness/config/sample_counts.yaml @@ -0,0 +1,30 @@ +# Full-dataset sample counts per benchmark (fallback for the Run Plan +# when the dataset is not cached locally yet -- harvested from real runs; +# update when a benchmark's source changes). +humaneval: 164 +aime24: 30 +aime25: 30 +aime26: 30 +hmmt26: 30 +gpqa_diamond: 198 +mmlu: 285 +mmlu_pro: 12032 +cmmlu: 11528 +gsm8k: 1319 +arc: 2376 +hellaswag: 10042 +winogrande: 1267 +drop: 9535 +longbench_v2: 503 +live_code_bench: 1055 +bigcodebench: 1140 +trivia_qa: 17944 +simple_qa: 4326 +hle: 2500 +imo_answerbench: 89 +openai_mrcr: 1300 +bfcl_v3: 2000 +general_fc: 400 +tau2_bench: 184 +competition_math: 5000 +swe_bench_verified: 500