c78b0d6f0f
Explicit key flows only into request headers (adapter attribute / pool member), so it cannot leak into the spec string, EvalReport, or logs -- verified by scanning a report produced with a sentinel key. Two-key setups run twice with different --api-key, or use per-host env vars. Co-Authored-By: Claude <noreply@anthropic.com>
601 lines
27 KiB
Python
601 lines
27 KiB
Python
"""EvalHarness CLI. Zero third-party deps beyond the data layer (pydantic)."""
|
|
|
|
import argparse
|
|
import time
|
|
import json
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
|
|
def _overrides(args):
|
|
"""Optional DatasetSpec field overrides shared by fetch/stats/show."""
|
|
if getattr(args, 'cache_dir', None):
|
|
from evalharness.data.dataset import set_cache_root
|
|
|
|
set_cache_root(args.cache_dir)
|
|
ov = {}
|
|
for k in ('source', 'split', 'subset'):
|
|
v = getattr(args, k, None)
|
|
if v is not None:
|
|
ov[k] = v
|
|
return ov
|
|
|
|
|
|
def _cmd_data_list(_args) -> int:
|
|
from evalharness.data import list_datasets
|
|
|
|
specs = list_datasets()
|
|
if not specs:
|
|
print('no datasets registered')
|
|
return 0
|
|
name_w = max(len(s.name) for s in specs)
|
|
type_w = max(len(s.task_type) for s in specs)
|
|
for s in specs:
|
|
print(f'{s.name:<{name_w}} {s.task_type:<{type_w}} {s.source} {s.description}')
|
|
print(f'\n{len(specs)} dataset(s) registered')
|
|
return 0
|
|
|
|
|
|
def _fetch_one(name: str, force: bool, overrides) -> str:
|
|
from evalharness.data import get_dataset
|
|
|
|
ds = get_dataset(name, **overrides)
|
|
ds.materialize(force=force)
|
|
origin = 'cache' if ds.lineage.get('from') == 'cache' else 'source'
|
|
return f'{name}: {len(ds)} sample(s) [{origin}] -> {ds.cache_dir}'
|
|
|
|
|
|
def _cmd_data_fetch(args) -> int:
|
|
names = args.names
|
|
if len(names) == 1:
|
|
print(_fetch_one(names[0], args.force, _overrides(args)))
|
|
return 0
|
|
# Concurrent prefetch: downloads are I/O-bound, threads suffice.
|
|
# Per-dataset file locks inside materialize() guard shared cache entries.
|
|
ok = True
|
|
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
|
futures = {pool.submit(_fetch_one, n, args.force, _overrides(args)): n for n in names}
|
|
for fut in as_completed(futures):
|
|
try:
|
|
print(fut.result())
|
|
except Exception as e: # one failure must not block the rest
|
|
ok = False
|
|
print(f'{futures[fut]}: FAILED ({e})', file=sys.stderr)
|
|
return 0 if ok else 1
|
|
|
|
|
|
def _cmd_data_stats(args) -> int:
|
|
from evalharness.data import get_dataset
|
|
|
|
stats = get_dataset(args.name, **_overrides(args)).stats()
|
|
print(json.dumps(stats, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
def _cmd_data_show(args) -> int:
|
|
from evalharness.data import get_dataset
|
|
|
|
ds = get_dataset(args.name, **_overrides(args))
|
|
for s in ds[: args.n]:
|
|
print(json.dumps(s.model_dump(), ensure_ascii=False, indent=2))
|
|
print('---')
|
|
return 0
|
|
|
|
|
|
def _cmd_data_unload(args) -> int:
|
|
from evalharness.data import get_dataset
|
|
|
|
for name in args.names:
|
|
ds = get_dataset(name, **_overrides(args))
|
|
removed = ds.unload()
|
|
print(f'{name}: cache {"removed" if removed else "not present (nothing to do)"} -> {ds.cache_dir}')
|
|
return 0
|
|
|
|
|
|
def _cmd_sandbox_prefetch(args) -> int:
|
|
from evalharness.data import get_dataset
|
|
from evalharness.sandbox import docker_available, images_for_dataset, prefetch_images
|
|
|
|
if not docker_available():
|
|
print('docker is not available on this host', file=sys.stderr)
|
|
return 1
|
|
ds = get_dataset(args.dataset, **_overrides(args))
|
|
images = images_for_dataset(ds, limit=args.limit)
|
|
if not images:
|
|
print(f'{args.dataset}: no sandbox images declared by its samples')
|
|
return 0
|
|
prefetch_images(images, workers=args.workers)
|
|
return 0
|
|
|
|
|
|
def _add_override_flags(p: argparse.ArgumentParser) -> None:
|
|
p.add_argument('--source', help='override DatasetSpec.source (e.g. a local dir)')
|
|
p.add_argument('--split', help='override DatasetSpec.split')
|
|
p.add_argument('--subset', help='override DatasetSpec.subset')
|
|
p.add_argument('--cache-dir', help='cache root (default: $EVALHARNESS_CACHE or ~/.cache/evalharness)')
|
|
|
|
|
|
def _cmd_eval_list(_args) -> int:
|
|
from evalharness.eval import list_evals
|
|
|
|
names = list_evals()
|
|
print('\n'.join(names) if names else 'no eval recipes registered')
|
|
print(f'\n{len(names)} eval recipe(s) registered')
|
|
return 0
|
|
|
|
|
|
def _print_run_progress(done, total, name='', status='running', started=None):
|
|
"""Print one live progress line for a multi-benchmark run."""
|
|
import time
|
|
|
|
width = 28
|
|
filled = int(width * done / max(total, 1))
|
|
bar = '#' * filled + '-' * (width - filled)
|
|
elapsed = time.time() - started if started else 0
|
|
label = f'{done}/{total} [{bar}] {status}: {name}'
|
|
print(f'\r{label} ({elapsed:.0f}s)', end='\n' if done >= total else '', flush=True)
|
|
|
|
|
|
def _rich_console():
|
|
"""Return a Rich console when installed; keep the CLI dependency-free."""
|
|
try:
|
|
from rich.console import Console
|
|
|
|
return Console()
|
|
except ImportError:
|
|
return None
|
|
|
|
|
|
def _print_run_plan(console, args, model_spec):
|
|
"""Print the important run facts before any dataset work starts."""
|
|
title = 'EvalHarness · Run Plan'
|
|
provider = getattr(args, 'provider', 'openai-chat') if model_spec else '—'
|
|
api_url = getattr(args, 'api_url', '') or '—'
|
|
model_name = getattr(args, 'model', '') or 'predictions file'
|
|
if console is None:
|
|
print(f'=== {title} ===')
|
|
print(f'Provider: {provider}')
|
|
print(f'API URL: {api_url}')
|
|
print(f'Model: {model_name}')
|
|
print(f'Benchmarks: {len(args.datasets)} -> {", ".join(args.datasets)}')
|
|
print(f'Concurrency: {args.concurrency} | Thinking: '
|
|
f'{"enabled" if not args.disable_thinking else "disabled"} | '
|
|
f'Performance: {"on" if args.perf else "off"}')
|
|
print(f'Resume: {"on" if args.resume else "off"} | Output: {args.out_dir or "(none)"}')
|
|
print('Samples: counted after each dataset is loaded')
|
|
return
|
|
|
|
from rich.panel import Panel
|
|
from rich.table import Table
|
|
|
|
table = Table(show_header=False, box=None, padding=(0, 1))
|
|
table.add_column('Item', style='cyan', no_wrap=True)
|
|
table.add_column('Value', style='white')
|
|
table.add_row('Provider', provider)
|
|
table.add_row('API URL', api_url)
|
|
table.add_row('Model', model_name)
|
|
table.add_row('Benchmarks', f'{len(args.datasets)} · {", ".join(args.datasets)}')
|
|
table.add_row('Samples', 'counted while loading each benchmark')
|
|
table.add_row('Concurrency', str(args.concurrency))
|
|
table.add_row('Thinking', '[red]disabled[/red]' if args.disable_thinking else '[green]enabled[/green]')
|
|
table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]')
|
|
table.add_row('Checkpoint', '[green]resume[/green]' if args.resume else '[dim]new run[/dim]')
|
|
table.add_row('Output', args.out_dir or '[dim](not specified)[/dim]')
|
|
console.print(Panel(table, title=title, border_style='blue', expand=False))
|
|
|
|
|
|
def _print_phase(console, index, total, name, message):
|
|
text = f'[{index}/{total}] {name}: {message}'
|
|
if console is not None:
|
|
console.print(f'[cyan]{text}[/cyan]')
|
|
else:
|
|
print(text, flush=True)
|
|
|
|
|
|
def _print_benchmark_result(console, index, total, name, status, elapsed):
|
|
if console is not None:
|
|
color = 'green' if status == 'done' else 'red'
|
|
icon = '✓' if status == 'done' else '✗'
|
|
console.print(f'[{color}]{icon}[/{color}] benchmark {index}/{total} '
|
|
f'{name} · {status} · elapsed={elapsed:.0f}s')
|
|
else:
|
|
_print_run_progress(index, total, name, status, time.time() - elapsed)
|
|
|
|
|
|
def _model_with_flags(model, args):
|
|
"""Translate explicit CLI flags to the adapter's internal options."""
|
|
for enabled, flag in ((getattr(args, 'disable_thinking', False), '!nothink'),
|
|
(getattr(args, 'perf', False), '!perf'),
|
|
(getattr(args, 'textools', False), '!textools')):
|
|
if enabled and not model.endswith(flag):
|
|
model += flag
|
|
return model
|
|
|
|
|
|
def _compose_model_spec(args):
|
|
"""Build the internal model spec from separate provider fields."""
|
|
model = args.model or ''
|
|
api_url = getattr(args, 'api_url', '') or ''
|
|
provider = getattr(args, 'provider', 'openai-chat') or 'openai-chat'
|
|
# openai-chat is the public name; the current client implementation
|
|
# remains registered as openai internally.
|
|
internal_provider = {'openai-chat': 'openai',
|
|
'openai-pool': 'openai-pool'}.get(provider, provider)
|
|
if api_url:
|
|
if not model:
|
|
raise SystemExit('error: --api-url requires --model (model name)')
|
|
model = f'{internal_provider}/{api_url.rstrip("/")}?{model}'
|
|
return _model_with_flags(model, args)
|
|
|
|
|
|
def _compose_judge_spec(args):
|
|
"""--judge accepts a bare model name (with --judge-api-url) or a full
|
|
legacy spec; keep both working like the main model flags."""
|
|
judge = getattr(args, 'judge', '') or ''
|
|
url = getattr(args, 'judge_api_url', '') or ''
|
|
provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
|
|
internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
|
|
if url and judge and '/' not in judge:
|
|
judge = f'{internal}/{url.rstrip("/")}?{judge}'
|
|
return judge or None
|
|
|
|
|
|
def _compose_judge_spec(args):
|
|
"""--judge accepts a bare model name (with --judge-api-url) or a full
|
|
legacy spec; both keep working, mirroring the main model flags."""
|
|
judge = getattr(args, 'judge', '') or ''
|
|
url = getattr(args, 'judge_api_url', '') or ''
|
|
provider = getattr(args, 'judge_provider', 'openai-chat') or 'openai-chat'
|
|
internal = {'openai-chat': 'openai', 'openai-pool': 'openai-pool'}.get(provider, provider)
|
|
if url and judge and '/' not in judge:
|
|
judge = f'{internal}/{url.rstrip("/")}?{judge}'
|
|
return judge or None
|
|
|
|
|
|
def _cmd_eval_run(args) -> int:
|
|
import asyncio
|
|
import time as _time
|
|
|
|
from evalharness.data import get_dataset
|
|
from evalharness.viz import render
|
|
|
|
overrides = _overrides(args)
|
|
out_dir = args.out_dir
|
|
if out_dir:
|
|
from pathlib import Path
|
|
|
|
Path(out_dir).mkdir(parents=True, exist_ok=True)
|
|
(Path(out_dir) / 'viz').mkdir(exist_ok=True)
|
|
(Path(out_dir) / 'reports').mkdir(exist_ok=True)
|
|
|
|
rows = []
|
|
run_started = _time.time()
|
|
total_runs = len(args.datasets)
|
|
model_spec = _compose_model_spec(args)
|
|
console = _rich_console()
|
|
_print_run_plan(console, args, model_spec)
|
|
for i, name in enumerate(args.datasets):
|
|
t0 = _time.time()
|
|
try:
|
|
_print_phase(console, i + 1, total_runs, name,
|
|
'loading/downloading dataset')
|
|
ds = get_dataset(name, **overrides)
|
|
sample_count = len(ds)
|
|
origin = ds.lineage.get('from', 'unknown')
|
|
_print_phase(console, i + 1, total_runs, name,
|
|
f'dataset ready · samples={sample_count} · source={origin}')
|
|
if model_spec: # generate + score in one go
|
|
from evalharness.model import run_eval
|
|
|
|
progress_reporter = None
|
|
if args.progress:
|
|
from evalharness.progress import RichTerminalProgress
|
|
|
|
if RichTerminalProgress is not None:
|
|
# share ONE console: phase lines printed by another
|
|
# writer during the live bar interleave incorrectly
|
|
progress_reporter = RichTerminalProgress(console=console)
|
|
|
|
def status_callback(msg, _idx=i, _name=name,
|
|
_reporter=progress_reporter,
|
|
_console=console):
|
|
if _reporter is not None:
|
|
_reporter.log(f'[{_idx}/{total_runs}] {_name}: {msg}')
|
|
else:
|
|
_print_phase(_console, _idx, total_runs, _name, msg)
|
|
report = asyncio.run(run_eval(
|
|
ds, model_spec, concurrency=args.concurrency, limit=args.limit,
|
|
limit_per_task=args.limit_per_task,
|
|
checkpoint=args.resume,
|
|
judge_spec=_compose_judge_spec(args), env=args.env,
|
|
api_key=getattr(args, 'api_key', ''),
|
|
judge_api_key=getattr(args, 'judge_api_key', ''),
|
|
gen_profile=getattr(args, 'profile', ''),
|
|
progress_reporter=progress_reporter,
|
|
status_callback=status_callback))
|
|
else:
|
|
from evalharness.eval import evaluate
|
|
|
|
if not args.predictions:
|
|
raise SystemExit('error: provide --model or a predictions file')
|
|
_print_phase(console, i + 1, total_runs, name, 'loading predictions')
|
|
preds_path = args.predictions[i] if len(args.predictions) > i else args.predictions[0]
|
|
preds = [json.loads(line) for line in open(preds_path, encoding='utf-8') if line.strip()]
|
|
preds = [p.get('raw', p.get('prediction', '')) if isinstance(p, dict) else p
|
|
for p in preds]
|
|
report = evaluate(ds, preds, model=model_spec or 'preds')
|
|
_print_phase(console, i + 1, total_runs, name, 'scoring complete')
|
|
if args.out:
|
|
report.save(args.out)
|
|
if out_dir:
|
|
_print_phase(console, i + 1, total_runs, name,
|
|
'writing report and visualization files')
|
|
report.save(f'{out_dir}/reports/{name}.report.json')
|
|
with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f:
|
|
f.write(render(report, style='text'))
|
|
if len(args.datasets) == 1 or args.verbose:
|
|
print(render(report, style=args.style))
|
|
primary = next(iter(report.metrics), '')
|
|
secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0)
|
|
for s in report.samples)
|
|
groups = {k: v for k, v in report.metric_groups.items()
|
|
if isinstance(v, dict) and k not in ('run_info',)
|
|
and not k.startswith('agg_error')}
|
|
rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary),
|
|
'n': report.num_samples,
|
|
'extract_fail': report.num_failed_extractions,
|
|
'secs': round(secs_total, 1),
|
|
'hours': round(secs_total / 3600, 2),
|
|
'groups': groups, 'ok': True})
|
|
_print_benchmark_result(console, i + 1, total_runs, name,
|
|
'done', _time.time() - t0)
|
|
except Exception as e:
|
|
rows.append({'name': name, 'metric': '-', 'value': None,
|
|
'secs': round(_time.time() - t0, 1), 'ok': False,
|
|
'err': f'{type(e).__name__}: {str(e)[:100]}'})
|
|
print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr)
|
|
_print_benchmark_result(console, i + 1, total_runs, name,
|
|
'failed', _time.time() - t0)
|
|
|
|
if len(rows) > 1:
|
|
_print_summary_table(console, rows)
|
|
ok = sum(1 for r in rows if r['ok'])
|
|
print(f'\n{ok}/{len(rows)} ok' + (f' -> artifacts in {out_dir}/' if out_dir else ''))
|
|
if out_dir:
|
|
import csv as _csv
|
|
|
|
with open(f'{out_dir}/viz/summary.csv', 'w', newline='', encoding='utf-8') as f:
|
|
w = _csv.writer(f)
|
|
w.writerow(['benchmark', 'score', 'metric', 'num_samples',
|
|
'time_h', 'time_s', 'extract_fail',
|
|
'success_rate', 'latency_mean_s', 'output_tps', 'request_qps',
|
|
'input_tokens_mean', 'output_tokens_mean', 'total_tokens',
|
|
'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s',
|
|
'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s',
|
|
'categories'])
|
|
for r in rows:
|
|
perf = (r.get('groups') or {}).get('perf') or {}
|
|
cats = '; '.join(f'{g}={_f3(v)}'
|
|
for gname, gv in (r.get('groups') or {}).items()
|
|
if gname != 'perf'
|
|
for g, v in (gv or {}).items()
|
|
if isinstance(v, (int, float)))[:2000]
|
|
w.writerow([r['name'], _f3(r.get('value')), r['metric'], r.get('n', ''),
|
|
r.get('hours', ''), r.get('secs', ''),
|
|
r.get('extract_fail', 0)] +
|
|
[perf.get(k, '') for k in (
|
|
'success_rate', 'latency_mean_s', 'output_tps', 'request_qps',
|
|
'input_tokens_mean', 'output_tokens_mean', 'total_tokens',
|
|
'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s',
|
|
'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] +
|
|
[cats])
|
|
with open(f'{out_dir}/viz/summary.md', 'w', encoding='utf-8') as f:
|
|
import time as _tt
|
|
model_names = {r.get('model', '') for r in rows if r.get('model')}
|
|
head = f"# eval run summary\n\n- model: {', '.join(model_names) or '?'}\n"
|
|
head += f"- created: {_tt.strftime('%Y-%m-%d %H:%M:%S')}\n"
|
|
head += f"- benchmarks: {sum(1 for r in rows if r['ok'])}/{len(rows)} ok\n\n"
|
|
f.write(head)
|
|
f.write('| benchmark | metric | score | n | time | status |\n'
|
|
'|---|---|---:|---:|---:|---|\n')
|
|
for r in rows:
|
|
v = _fmt_score(r.get('value'))
|
|
h = r.get('hours') or 0
|
|
t = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s"
|
|
st = 'ok' if r['ok'] else f"failed: {r.get('err', '')[:60]}"
|
|
f.write(f"| {r['name']} | {r['metric']} | {v} | "
|
|
f"{r.get('n', '')} | {t} | {st} |\n")
|
|
if out_dir:
|
|
print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)')
|
|
return 0 if all(r['ok'] for r in rows) else 1
|
|
|
|
|
|
def _fmt_score(v):
|
|
"""Unified score format: fractions render as percentages everywhere."""
|
|
try:
|
|
v = float(v)
|
|
except (TypeError, ValueError):
|
|
return 'ERR'
|
|
return f'{v * 100:.1f}%' if 0.0 <= v <= 1.0 else f'{v:g}'
|
|
|
|
|
|
def _print_summary_table(console, rows):
|
|
"""Rich multi-benchmark summary (falls back to aligned plain text)."""
|
|
if console is not None:
|
|
from rich.table import Table
|
|
|
|
t = Table(title='Run Summary', header_style='bold cyan',
|
|
title_style='bold', expand=False)
|
|
for col, just in (('benchmark', 'left'), ('metric', 'left'),
|
|
('score', 'right'), ('n', 'right'), ('time', 'right')):
|
|
t.add_column(col, justify=just)
|
|
for r in rows:
|
|
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
|
|
h = r.get('hours') or 0
|
|
tm = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s"
|
|
style = 'green' if r['ok'] else 'red'
|
|
t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm,
|
|
style=style)
|
|
console.print(t)
|
|
return
|
|
print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}')
|
|
print('-' * 64)
|
|
for r in rows:
|
|
v = _fmt_score(r.get('value')) if r['ok'] else 'ERR'
|
|
h = r.get('hours') or 0
|
|
tm = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s"
|
|
err = f" {r.get('err', '')}" if not r['ok'] else ''
|
|
print(f"{r['name']:<20} {r['metric']:<16} {v:>8} "
|
|
f"{str(r.get('n', '')):>6} {tm:>8}{err}")
|
|
|
|
|
|
def _f3(v):
|
|
try:
|
|
return round(float(v), 4)
|
|
except (TypeError, ValueError):
|
|
return v
|
|
|
|
|
|
def _cmd_viz_show(args) -> int:
|
|
from evalharness.viz import render
|
|
|
|
if args.style == 'excel':
|
|
from evalharness.eval.record import EvalReport
|
|
|
|
reps = [EvalReport.load(p) for p in args.reports]
|
|
out = render(reps, style='excel',
|
|
**({'n': args.n} if args.n else {}),
|
|
**({'out': args.out} if args.out else {}))
|
|
print(f'excel -> {out}')
|
|
return 0
|
|
print(render([*args.reports], style=args.style, **({'n': args.n} if args.n else {})))
|
|
return 0
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(prog='evalharness', description='EvalHarness CLI')
|
|
sub = parser.add_subparsers(dest='command', required=True)
|
|
|
|
data = sub.add_parser('data', help='dataset plugin commands')
|
|
dsub = data.add_subparsers(dest='data_command', required=True)
|
|
|
|
p = dsub.add_parser('list', help='list registered datasets (no download)')
|
|
p.set_defaults(func=_cmd_data_list)
|
|
|
|
p = dsub.add_parser('fetch', help='materialize dataset(s) into the cache')
|
|
p.add_argument('names', nargs='+')
|
|
p.add_argument('--force', action='store_true', help='re-download and rebuild the cache')
|
|
p.add_argument('--workers', type=int, default=8, help='concurrent downloads (default 8)')
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_data_fetch)
|
|
|
|
p = dsub.add_parser('unload', help='drop cache entries (raw + samples); images belong to the sandbox layer')
|
|
p.add_argument('names', nargs='+')
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_data_unload)
|
|
|
|
p = dsub.add_parser('stats', help='materialize and show dataset statistics')
|
|
p.add_argument('name')
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_data_stats)
|
|
|
|
p = dsub.add_parser('show', help='print the first N samples')
|
|
p.add_argument('name')
|
|
p.add_argument('-n', type=int, default=2)
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_data_show)
|
|
|
|
# ---- eval ----
|
|
ev = sub.add_parser('eval', help='evaluation recipes & runs')
|
|
esub = ev.add_subparsers(dest='eval_command', required=True)
|
|
|
|
p = esub.add_parser('list', help='list registered eval recipes')
|
|
p.set_defaults(func=_cmd_eval_list)
|
|
|
|
p = esub.add_parser('run', help='score predictions (file) or generate+score (--model); multiple datasets OK')
|
|
p.add_argument('datasets', nargs='+', help='dataset name(s) (recipe auto-resolved)')
|
|
p.add_argument('predictions', nargs='?', help='jsonl: one raw string or {"raw": ...} per sample')
|
|
p.add_argument('--model', default='',
|
|
help='served model name when --api-url is used; or full legacy model spec')
|
|
p.add_argument('--api-url', default='',
|
|
help='API base URL when --model is only the served model name')
|
|
p.add_argument('--provider', default='openai-chat',
|
|
choices=('openai-chat', 'openai-pool'),
|
|
help='API protocol/provider (default: openai-chat)')
|
|
p.add_argument('--judge', default='',
|
|
help='judge model name with --judge-api-url, or full spec')
|
|
p.add_argument('--judge-api-url', default='',
|
|
help='judge API base URL when --judge is only the model name')
|
|
p.add_argument('--api-key', default='',
|
|
help='explicit API key for the model endpoint (overrides '
|
|
'env-based resolution; never written into reports)')
|
|
p.add_argument('--judge-api-key', default='',
|
|
help='explicit API key for the judge endpoint')
|
|
p.add_argument('--judge-provider', default='openai-chat',
|
|
help='judge protocol/provider (default openai-chat; '
|
|
'openai-pool for multi-endpoint judges)')
|
|
p.add_argument('--profile', default='',
|
|
help='named gen-params profile (dp4-nothink | qwen3-es-parity | t1-short '
|
|
'or any @register_gen_profile name); layers: plugin default < '
|
|
"profile.default < profile['<bench>'] < explicit kwargs")
|
|
p.add_argument('--disable-thinking', action='store_true',
|
|
help='send enable_thinking=false to the OpenAI-compatible model')
|
|
p.add_argument('--perf', action='store_true',
|
|
help='collect streaming TTFT and ITL metrics')
|
|
p.add_argument('--textools', action='store_true',
|
|
help='send tools as text instead of native tool calls')
|
|
p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump")
|
|
p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)')
|
|
p.add_argument('--progress', action='store_true', default=True,
|
|
help='show per-sample Rich terminal progress (default: on)')
|
|
p.add_argument('--no-progress', dest='progress', action='store_false',
|
|
help='disable per-sample Rich terminal progress')
|
|
p.add_argument('--limit', type=int, help='evaluate only the first N samples total')
|
|
p.add_argument('--resume', nargs='?', const=True, default=False,
|
|
help='resume from per-sample checkpoint (default path auto-derived; '
|
|
'pass a path to override)')
|
|
p.add_argument('--limit-per-task', type=int,
|
|
help='first N samples PER subset/category (evalscope --limit semantics); '
|
|
'composable with --limit (intersection)')
|
|
p.add_argument('--out', help='save the EvalReport json here (single dataset)')
|
|
p.add_argument('--out-dir', help='save reports/<name>.json + viz/<name>.txt + summary.md '
|
|
'here (multi-dataset runs)')
|
|
p.add_argument('--style', default='text', help='result render style (text/md/radar/errors)')
|
|
p.add_argument('--verbose', action='store_true', help='print full render for every dataset')
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_eval_run)
|
|
|
|
# ---- sandbox ----
|
|
sb = sub.add_parser('sandbox', help='execution environment management')
|
|
bsub = sb.add_subparsers(dest='sandbox_command', required=True)
|
|
|
|
p = bsub.add_parser('prefetch', help='parallel docker pull of a dataset\'s sandbox images')
|
|
p.add_argument('dataset', help='dataset whose samples declare images (e.g. swe_bench_verified)')
|
|
p.add_argument('--workers', type=int, default=8, help='concurrent pulls (default 8)')
|
|
p.add_argument('--limit', type=int, default=0, help='only first N samples (0=all)')
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_sandbox_prefetch)
|
|
|
|
# ---- viz ----
|
|
vz = sub.add_parser('viz', help='render saved EvalReport artifacts')
|
|
zsub = vz.add_subparsers(dest='viz_command', required=True)
|
|
|
|
p = zsub.add_parser('show', help='render report file(s)')
|
|
p.add_argument('reports', nargs='+')
|
|
p.add_argument('--style', default='text',
|
|
help='text | md | md_compare | radar | errors | excel (writes .xlsx)')
|
|
p.add_argument('--out', help='excel output path (default ./evalharness_report.xlsx)')
|
|
p.add_argument('-n', type=int, help='for errors style: how many samples')
|
|
p.set_defaults(func=_cmd_viz_show)
|
|
|
|
return parser
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
return args.func(args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|