218 lines
7.8 KiB
Python
218 lines
7.8 KiB
Python
"""EvalHarness CLI. Zero third-party deps beyond the data layer (pydantic)."""
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
|
|
def _overrides(args):
|
|
"""Optional DatasetSpec field overrides shared by fetch/stats/show."""
|
|
if getattr(args, 'cache_dir', None):
|
|
from evalharness.data.dataset import set_cache_root
|
|
|
|
set_cache_root(args.cache_dir)
|
|
ov = {}
|
|
for k in ('source', 'split', 'subset'):
|
|
v = getattr(args, k, None)
|
|
if v is not None:
|
|
ov[k] = v
|
|
return ov
|
|
|
|
|
|
def _cmd_data_list(_args) -> int:
|
|
from evalharness.data import list_datasets
|
|
|
|
specs = list_datasets()
|
|
if not specs:
|
|
print('no datasets registered')
|
|
return 0
|
|
name_w = max(len(s.name) for s in specs)
|
|
type_w = max(len(s.task_type) for s in specs)
|
|
for s in specs:
|
|
print(f'{s.name:<{name_w}} {s.task_type:<{type_w}} {s.source} {s.description}')
|
|
print(f'\n{len(specs)} dataset(s) registered')
|
|
return 0
|
|
|
|
|
|
def _fetch_one(name: str, force: bool, overrides) -> str:
|
|
from evalharness.data import get_dataset
|
|
|
|
ds = get_dataset(name, **overrides)
|
|
ds.materialize(force=force)
|
|
origin = 'cache' if ds.lineage.get('from') == 'cache' else 'source'
|
|
return f'{name}: {len(ds)} sample(s) [{origin}] -> {ds.cache_dir}'
|
|
|
|
|
|
def _cmd_data_fetch(args) -> int:
|
|
names = args.names
|
|
if len(names) == 1:
|
|
print(_fetch_one(names[0], args.force, _overrides(args)))
|
|
return 0
|
|
# Concurrent prefetch: downloads are I/O-bound, threads suffice.
|
|
# Per-dataset file locks inside materialize() guard shared cache entries.
|
|
ok = True
|
|
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
|
futures = {pool.submit(_fetch_one, n, args.force, _overrides(args)): n for n in names}
|
|
for fut in as_completed(futures):
|
|
try:
|
|
print(fut.result())
|
|
except Exception as e: # one failure must not block the rest
|
|
ok = False
|
|
print(f'{futures[fut]}: FAILED ({e})', file=sys.stderr)
|
|
return 0 if ok else 1
|
|
|
|
|
|
def _cmd_data_stats(args) -> int:
|
|
from evalharness.data import get_dataset
|
|
|
|
stats = get_dataset(args.name, **_overrides(args)).stats()
|
|
print(json.dumps(stats, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
def _cmd_data_show(args) -> int:
|
|
from evalharness.data import get_dataset
|
|
|
|
ds = get_dataset(args.name, **_overrides(args))
|
|
for s in ds[: args.n]:
|
|
print(json.dumps(s.model_dump(), ensure_ascii=False, indent=2))
|
|
print('---')
|
|
return 0
|
|
|
|
|
|
def _cmd_data_unload(args) -> int:
|
|
from evalharness.data import get_dataset
|
|
|
|
for name in args.names:
|
|
ds = get_dataset(name, **_overrides(args))
|
|
removed = ds.unload()
|
|
print(f'{name}: cache {"removed" if removed else "not present (nothing to do)"} -> {ds.cache_dir}')
|
|
return 0
|
|
|
|
|
|
def _add_override_flags(p: argparse.ArgumentParser) -> None:
|
|
p.add_argument('--source', help='override DatasetSpec.source (e.g. a local dir)')
|
|
p.add_argument('--split', help='override DatasetSpec.split')
|
|
p.add_argument('--subset', help='override DatasetSpec.subset')
|
|
p.add_argument('--cache-dir', help='cache root (default: $EVALHARNESS_CACHE or ~/.cache/evalharness)')
|
|
|
|
|
|
def _cmd_eval_list(_args) -> int:
|
|
from evalharness.eval import list_evals
|
|
|
|
names = list_evals()
|
|
print('\n'.join(names) if names else 'no eval recipes registered')
|
|
print(f'\n{len(names)} eval recipe(s) registered')
|
|
return 0
|
|
|
|
|
|
def _cmd_eval_run(args) -> int:
|
|
import asyncio
|
|
|
|
from evalharness.data import get_dataset
|
|
from evalharness.viz import render
|
|
|
|
ds = get_dataset(args.dataset, **_overrides(args))
|
|
if args.model: # generate + score in one go
|
|
from evalharness.model import run_eval
|
|
|
|
report = asyncio.run(run_eval(
|
|
ds, args.model, concurrency=args.concurrency, limit=args.limit,
|
|
judge_spec=args.judge))
|
|
else:
|
|
from evalharness.eval import evaluate
|
|
|
|
preds = [json.loads(line) for line in open(args.predictions, encoding='utf-8') if line.strip()]
|
|
preds = [p.get('raw', p.get('prediction', '')) if isinstance(p, dict) else p for p in preds]
|
|
report = evaluate(ds, preds, model=args.model or 'preds')
|
|
if args.out:
|
|
report.save(args.out)
|
|
print(f'saved -> {args.out}')
|
|
print(render(report, style=args.style))
|
|
return 0
|
|
|
|
|
|
def _cmd_viz_show(args) -> int:
|
|
from evalharness.viz import render
|
|
|
|
print(render([*args.reports], style=args.style, **({'n': args.n} if args.n else {})))
|
|
return 0
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(prog='evalharness', description='EvalHarness CLI')
|
|
sub = parser.add_subparsers(dest='command', required=True)
|
|
|
|
data = sub.add_parser('data', help='dataset plugin commands')
|
|
dsub = data.add_subparsers(dest='data_command', required=True)
|
|
|
|
p = dsub.add_parser('list', help='list registered datasets (no download)')
|
|
p.set_defaults(func=_cmd_data_list)
|
|
|
|
p = dsub.add_parser('fetch', help='materialize dataset(s) into the cache')
|
|
p.add_argument('names', nargs='+')
|
|
p.add_argument('--force', action='store_true', help='re-download and rebuild the cache')
|
|
p.add_argument('--workers', type=int, default=8, help='concurrent downloads (default 8)')
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_data_fetch)
|
|
|
|
p = dsub.add_parser('unload', help='drop cache entries (raw + samples); images belong to the sandbox layer')
|
|
p.add_argument('names', nargs='+')
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_data_unload)
|
|
|
|
p = dsub.add_parser('stats', help='materialize and show dataset statistics')
|
|
p.add_argument('name')
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_data_stats)
|
|
|
|
p = dsub.add_parser('show', help='print the first N samples')
|
|
p.add_argument('name')
|
|
p.add_argument('-n', type=int, default=2)
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_data_show)
|
|
|
|
# ---- eval ----
|
|
ev = sub.add_parser('eval', help='evaluation recipes & runs')
|
|
esub = ev.add_subparsers(dest='eval_command', required=True)
|
|
|
|
p = esub.add_parser('list', help='list registered eval recipes')
|
|
p.set_defaults(func=_cmd_eval_list)
|
|
|
|
p = esub.add_parser('run', help='score predictions (file) or generate+score (--model)')
|
|
p.add_argument('dataset', help='dataset name (recipe auto-resolved)')
|
|
p.add_argument('predictions', nargs='?', help='jsonl: one raw string or {"raw": ...} per sample')
|
|
p.add_argument('--model', default='',
|
|
help="generate with model spec: mock | mock:boxed | "
|
|
"openai/http://host:8000/v1?model | deploy:vllm/model")
|
|
p.add_argument('--judge', default='', help='judge model spec for llm_judge recipes')
|
|
p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)')
|
|
p.add_argument('--limit', type=int, help='evaluate only the first N samples')
|
|
p.add_argument('--out', help='save the EvalReport json here')
|
|
p.add_argument('--style', default='text', help='result render style (text/md/radar/errors)')
|
|
_add_override_flags(p)
|
|
p.set_defaults(func=_cmd_eval_run)
|
|
|
|
# ---- viz ----
|
|
vz = sub.add_parser('viz', help='render saved EvalReport artifacts')
|
|
zsub = vz.add_subparsers(dest='viz_command', required=True)
|
|
|
|
p = zsub.add_parser('show', help='render report file(s)')
|
|
p.add_argument('reports', nargs='+')
|
|
p.add_argument('--style', default='text',
|
|
help='text | md | md_compare (multi) | radar | errors')
|
|
p.add_argument('-n', type=int, help='for errors style: how many samples')
|
|
p.set_defaults(func=_cmd_viz_show)
|
|
|
|
return parser
|
|
|
|
|
|
def main(argv=None) -> int:
|
|
args = build_parser().parse_args(argv)
|
|
return args.func(args)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|