Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Run plan: concrete sample counts + 'auto' concurrency display/alias

Browse Source 
- sample-counts manifest (config/sample_counts.yaml, harvested from
  real runs): uncached benches still show exact numbers in the plan
  instead of 'counts when datasets load' -- 'cache+est.' marks the mix
- '--concurrency auto' is now an alias for --auto-concurrency
- Concurrency row shows 'auto (start 8, gate decides)' when the gate
  drives, instead of a bare misleading 8

Also verified end-to-end: thinking-mode humaneval rep1/rep2 both
pass 98.8%, matching the es reference runs (98.17/98.78/98.78) on the
same model -- framework alignment holds on the thinking path too.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-14 09:11:07 +00:00
parent acb94e3e20
commit 7921688149
2 changed files with 85 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

71
evalharness/cli.py
View File

@ -193,25 +193,38 @@ def _load_bench_cfg(args, name: str) -> dict:
def _plan_sample_counts(args):
"""(total_samples, total_generations, n_uncached) across the planned
benches, counted from LOCAL cache entries only -- never touches the
network, so the run plan stays instant on cold machines. Uncached
benches simply don't contribute yet.
benches. Cache-first (exact); uncached benches fall back to the shipped
sample-counts manifest so the plan still shows CONCRETE numbers on a
cold machine instead of 'counts when datasets load'.
"""
from pathlib import Path
from evalharness.data import get_dataset
manifest = {}
mf = Path(__file__).parent / 'config' / 'sample_counts.yaml'
if mf.exists():
try:
import yaml as _yaml
manifest = _yaml.safe_load(open(mf)) or {}
except Exception:
manifest = {}
total = gens = uncached = 0
for name in getattr(args, 'datasets', []) or []:
n = None
try:
ds = get_dataset(name, **_overrides(args))
cache_file = ds.cache_dir / 'samples.jsonl'
if not cache_file.exists():
uncached += 1
continue
if cache_file.exists():
with open(cache_file, 'rb') as f:
n = sum(1 for _ in f)
except Exception:
n = None
if n is None:
uncached += 1
continue
n = int(manifest.get(name, 0) or 0) # estimate; 0 = unknown
if getattr(args, 'limit', None):
n = min(n, args.limit)
total += n
@ -232,21 +245,23 @@ def _print_run_plan(console, args, model_spec):
cap = f' · ≤{args.limit} per bench (--limit)'
elif getattr(args, 'limit_per_task', None):
cap = f' · ≤{args.limit_per_task} per subject (--limit-per-task)'
src = 'cached' if not n_uncached else ('cache+est.' if n_samples else 'est.')
if n_samples:
samples = f'{n_samples:,} samples (cached){cap}'
samples = f'{n_samples:,} samples ({src}){cap}'
if n_gens > n_samples: # repeats multiply the real work
samples = (f'{n_samples:,} samples (cached){cap} → '
samples = (f'{n_samples:,} samples ({src}){cap} → '
f'{n_gens:,} generations (repeats)')
elif getattr(args, 'limit', None):
samples = f'up to {args.limit} per bench (--limit)'
elif getattr(args, 'limit_per_task', None):
samples = f'up to {args.limit_per_task} per subject (--limit-per-task)'
else:
samples = '? (unknown benchmarks)'
if n_uncached:
samples += f' · {n_uncached} bench(es) not cached yet'
elif n_uncached:
if getattr(args, 'limit', None):
samples = f'up to {args.limit} per bench (--limit), counts when datasets load'
elif getattr(args, 'limit_per_task', None):
samples = (f'up to {args.limit_per_task} per subject '
'(--limit-per-task), counts when datasets load')
else:
samples = 'full dataset, counts when datasets load (none cached yet)'
_auto = getattr(args, 'auto_concurrency', False)
_conc = (f'auto (start {args.concurrency}, gate decides)' if _auto
else str(args.concurrency))
if console is None:
print(f'=== {title} ===')
print(f'Provider: {provider}')
@ -254,7 +269,7 @@ def _print_run_plan(console, args, model_spec):
print(f'Model: {model_name}')
print(f'Benchmarks: {len(args.datasets)} -> {", ".join(args.datasets)}')
print(f'Samples: {samples}')
print(f'Concurrency: {args.concurrency} | Thinking: '
print(f'Concurrency: {_conc} | Thinking: '
f'{"enabled" if not args.disable_thinking else "disabled"} | '
f'Performance: {"on" if args.perf else "off"}')
print(f'Resume: {"on" if args.resume else "off"} | Output: {args.out_dir or "(none)"}')
@ -271,7 +286,7 @@ def _print_run_plan(console, args, model_spec):
table.add_row('Model', model_name)
table.add_row('Benchmarks', f'{len(args.datasets)} · {", ".join(args.datasets)}')
table.add_row('Samples', samples)
table.add_row('Concurrency', str(args.concurrency))
table.add_row('Concurrency', f'[magenta]{_conc}[/magenta]' if _auto else _conc)
table.add_row('Thinking', '[red]disabled[/red]' if args.disable_thinking else '[green]enabled[/green]')
table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]')
table.add_row('Checkpoint', '[green]resume[/green]' if args.resume else '[dim]new run[/dim]')
@ -952,6 +967,11 @@ def _cmd_eval_run(args) -> int:
'secs': round(_time.time() - t0, 1), 'ok': False,
'err': f'{type(e).__name__}: {str(e)[:100]}'})
print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr)
import traceback as _tb
_tb.print_exc() # full stack to stderr: the one-line form hides
# WHERE an error like 'Object of type ellipsis is not JSON
# serializable' actually comes from
from evalharness.hooks import fire as _fire
_fire('on_benchmark_failed', name=name, error=e, dataset=name)
@ -1160,7 +1180,10 @@ def build_parser() -> argparse.ArgumentParser:
p.add_argument('--textools', action='store_true',
help='send tools as text instead of native tool calls')
p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump")
p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)')
p.add_argument('--concurrency', default='32',
help="parallel model calls (default 32); 'auto' = same as "
"--auto-concurrency (the adaptive gate decides, this "
"value becomes its starting point)")
p.add_argument('--progress', action='store_true', default=True,
help='show per-sample progress (default: on)')
p.add_argument('--no-progress', dest='progress', action='store_false',
@ -1225,6 +1248,14 @@ def build_parser() -> argparse.ArgumentParser:
def main(argv=None) -> int:
args = build_parser().parse_args(argv)
# '--concurrency auto' == '--auto-concurrency': normalize once, here,
# so every downstream site (plan display, run_eval, gate initial) sees
# an int + the flag
if str(getattr(args, 'concurrency', '32')).strip().lower() == 'auto':
args.auto_concurrency = True
args.concurrency = 8
else:
args.concurrency = int(args.concurrency)
return args.func(args)

30
evalharness/config/sample_counts.yaml Normal file
View File

@ -0,0 +1,30 @@
# Full-dataset sample counts per benchmark (fallback for the Run Plan
# when the dataset is not cached locally yet -- harvested from real runs;
# update when a benchmark's source changes).
humaneval: 164
aime24: 30
aime25: 30
aime26: 30
hmmt26: 30
gpqa_diamond: 198
mmlu: 285
mmlu_pro: 12032
cmmlu: 11528
gsm8k: 1319
arc: 2376
hellaswag: 10042
winogrande: 1267
drop: 9535
longbench_v2: 503
live_code_bench: 1055
bigcodebench: 1140
trivia_qa: 17944
simple_qa: 4326
hle: 2500
imo_answerbench: 89
openai_mrcr: 1300
bfcl_v3: 2000
general_fc: 400
tau2_bench: 184
competition_math: 5000
swe_bench_verified: 500