"""EvalHarness CLI. Zero third-party deps beyond the data layer (pydantic).""" import argparse import json import sys from concurrent.futures import ThreadPoolExecutor, as_completed def _overrides(args): """Optional DatasetSpec field overrides shared by fetch/stats/show.""" if getattr(args, 'cache_dir', None): from evalharness.data.dataset import set_cache_root set_cache_root(args.cache_dir) ov = {} for k in ('source', 'split', 'subset'): v = getattr(args, k, None) if v is not None: ov[k] = v return ov def _cmd_data_list(_args) -> int: from evalharness.data import list_datasets specs = list_datasets() if not specs: print('no datasets registered') return 0 name_w = max(len(s.name) for s in specs) type_w = max(len(s.task_type) for s in specs) for s in specs: print(f'{s.name:<{name_w}} {s.task_type:<{type_w}} {s.source} {s.description}') print(f'\n{len(specs)} dataset(s) registered') return 0 def _fetch_one(name: str, force: bool, overrides) -> str: from evalharness.data import get_dataset ds = get_dataset(name, **overrides) ds.materialize(force=force) origin = 'cache' if ds.lineage.get('from') == 'cache' else 'source' return f'{name}: {len(ds)} sample(s) [{origin}] -> {ds.cache_dir}' def _cmd_data_fetch(args) -> int: names = args.names if len(names) == 1: print(_fetch_one(names[0], args.force, _overrides(args))) return 0 # Concurrent prefetch: downloads are I/O-bound, threads suffice. # Per-dataset file locks inside materialize() guard shared cache entries. ok = True with ThreadPoolExecutor(max_workers=args.workers) as pool: futures = {pool.submit(_fetch_one, n, args.force, _overrides(args)): n for n in names} for fut in as_completed(futures): try: print(fut.result()) except Exception as e: # one failure must not block the rest ok = False print(f'{futures[fut]}: FAILED ({e})', file=sys.stderr) return 0 if ok else 1 def _cmd_data_stats(args) -> int: from evalharness.data import get_dataset stats = get_dataset(args.name, **_overrides(args)).stats() print(json.dumps(stats, ensure_ascii=False, indent=2)) return 0 def _cmd_data_show(args) -> int: from evalharness.data import get_dataset ds = get_dataset(args.name, **_overrides(args)) for s in ds[: args.n]: print(json.dumps(s.model_dump(), ensure_ascii=False, indent=2)) print('---') return 0 def _cmd_data_unload(args) -> int: from evalharness.data import get_dataset for name in args.names: ds = get_dataset(name, **_overrides(args)) removed = ds.unload() print(f'{name}: cache {"removed" if removed else "not present (nothing to do)"} -> {ds.cache_dir}') return 0 def _add_override_flags(p: argparse.ArgumentParser) -> None: p.add_argument('--source', help='override DatasetSpec.source (e.g. a local dir)') p.add_argument('--split', help='override DatasetSpec.split') p.add_argument('--subset', help='override DatasetSpec.subset') p.add_argument('--cache-dir', help='cache root (default: $EVALHARNESS_CACHE or ~/.cache/evalharness)') def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog='evalharness', description='EvalHarness CLI') sub = parser.add_subparsers(dest='command', required=True) data = sub.add_parser('data', help='dataset plugin commands') dsub = data.add_subparsers(dest='data_command', required=True) p = dsub.add_parser('list', help='list registered datasets (no download)') p.set_defaults(func=_cmd_data_list) p = dsub.add_parser('fetch', help='materialize dataset(s) into the cache') p.add_argument('names', nargs='+') p.add_argument('--force', action='store_true', help='re-download and rebuild the cache') p.add_argument('--workers', type=int, default=8, help='concurrent downloads (default 8)') _add_override_flags(p) p.set_defaults(func=_cmd_data_fetch) p = dsub.add_parser('unload', help='drop cache entries (raw + samples); images belong to the sandbox layer') p.add_argument('names', nargs='+') _add_override_flags(p) p.set_defaults(func=_cmd_data_unload) p = dsub.add_parser('stats', help='materialize and show dataset statistics') p.add_argument('name') _add_override_flags(p) p.set_defaults(func=_cmd_data_stats) p = dsub.add_parser('show', help='print the first N samples') p.add_argument('name') p.add_argument('-n', type=int, default=2) _add_override_flags(p) p.set_defaults(func=_cmd_data_show) return parser def main(argv=None) -> int: args = build_parser().parse_args(argv) return args.func(args) if __name__ == '__main__': sys.exit(main())