"""EvalHarness CLI. Zero third-party deps beyond the data layer (pydantic)."""

import argparse
import json
import sys
from concurrent.futures import ThreadPoolExecutor, as_completed


def _overrides(args):
    """Optional DatasetSpec field overrides shared by fetch/stats/show."""
    if getattr(args, 'cache_dir', None):
        from evalharness.data.dataset import set_cache_root

        set_cache_root(args.cache_dir)
    ov = {}
    for k in ('source', 'split', 'subset'):
        v = getattr(args, k, None)
        if v is not None:
            ov[k] = v
    return ov


def _cmd_data_list(_args) -> int:
    from evalharness.data import list_datasets

    specs = list_datasets()
    if not specs:
        print('no datasets registered')
        return 0
    name_w = max(len(s.name) for s in specs)
    type_w = max(len(s.task_type) for s in specs)
    for s in specs:
        print(f'{s.name:<{name_w}}  {s.task_type:<{type_w}}  {s.source}  {s.description}')
    print(f'\n{len(specs)} dataset(s) registered')
    return 0


def _fetch_one(name: str, force: bool, overrides) -> str:
    from evalharness.data import get_dataset

    ds = get_dataset(name, **overrides)
    ds.materialize(force=force)
    origin = 'cache' if ds.lineage.get('from') == 'cache' else 'source'
    return f'{name}: {len(ds)} sample(s) [{origin}]  ->  {ds.cache_dir}'


def _cmd_data_fetch(args) -> int:
    names = args.names
    if len(names) == 1:
        print(_fetch_one(names[0], args.force, _overrides(args)))
        return 0
    # Concurrent prefetch: downloads are I/O-bound, threads suffice.
    # Per-dataset file locks inside materialize() guard shared cache entries.
    ok = True
    with ThreadPoolExecutor(max_workers=args.workers) as pool:
        futures = {pool.submit(_fetch_one, n, args.force, _overrides(args)): n for n in names}
        for fut in as_completed(futures):
            try:
                print(fut.result())
            except Exception as e:  # one failure must not block the rest
                ok = False
                print(f'{futures[fut]}: FAILED ({e})', file=sys.stderr)
    return 0 if ok else 1


def _cmd_data_stats(args) -> int:
    from evalharness.data import get_dataset

    stats = get_dataset(args.name, **_overrides(args)).stats()
    print(json.dumps(stats, ensure_ascii=False, indent=2))
    return 0


def _cmd_data_show(args) -> int:
    from evalharness.data import get_dataset

    ds = get_dataset(args.name, **_overrides(args))
    for s in ds[: args.n]:
        print(json.dumps(s.model_dump(), ensure_ascii=False, indent=2))
        print('---')
    return 0


def _cmd_data_unload(args) -> int:
    from evalharness.data import get_dataset

    for name in args.names:
        ds = get_dataset(name, **_overrides(args))
        removed = ds.unload()
        print(f'{name}: cache {"removed" if removed else "not present (nothing to do)"}  ->  {ds.cache_dir}')
    return 0


def _cmd_sandbox_prefetch(args) -> int:
    from evalharness.data import get_dataset
    from evalharness.sandbox import docker_available, images_for_dataset, prefetch_images

    if not docker_available():
        print('docker is not available on this host', file=sys.stderr)
        return 1
    ds = get_dataset(args.dataset, **_overrides(args))
    images = images_for_dataset(ds, limit=args.limit)
    if not images:
        print(f'{args.dataset}: no sandbox images declared by its samples')
        return 0
    prefetch_images(images, workers=args.workers)
    return 0


def _add_override_flags(p: argparse.ArgumentParser) -> None:
    p.add_argument('--source', help='override DatasetSpec.source (e.g. a local dir)')
    p.add_argument('--split', help='override DatasetSpec.split')
    p.add_argument('--subset', help='override DatasetSpec.subset')
    p.add_argument('--cache-dir', help='cache root (default: $EVALHARNESS_CACHE or ~/.cache/evalharness)')


def _cmd_eval_list(_args) -> int:
    from evalharness.eval import list_evals

    names = list_evals()
    print('\n'.join(names) if names else 'no eval recipes registered')
    print(f'\n{len(names)} eval recipe(s) registered')
    return 0


def _cmd_eval_run(args) -> int:
    import asyncio
    import time as _time

    from evalharness.data import get_dataset
    from evalharness.viz import render

    overrides = _overrides(args)
    out_dir = args.out_dir
    if out_dir:
        from pathlib import Path

        Path(out_dir).mkdir(parents=True, exist_ok=True)
        (Path(out_dir) / 'viz').mkdir(exist_ok=True)

    rows = []
    for i, name in enumerate(args.datasets):
        t0 = _time.time()
        try:
            ds = get_dataset(name, **overrides)
            if args.model:  # generate + score in one go
                from evalharness.model import run_eval

                report = asyncio.run(run_eval(
                    ds, args.model, concurrency=args.concurrency, limit=args.limit,
                    limit_per_task=args.limit_per_task,
                    checkpoint=args.resume,
                    judge_spec=args.judge, env=args.env))
            else:
                from evalharness.eval import evaluate

                if not args.predictions:
                    raise SystemExit('error: provide --model or a predictions file')
                preds_path = args.predictions[i] if len(args.predictions) > i else args.predictions[0]
                preds = [json.loads(line) for line in open(preds_path, encoding='utf-8') if line.strip()]
                preds = [p.get('raw', p.get('prediction', '')) if isinstance(p, dict) else p
                         for p in preds]
                report = evaluate(ds, preds, model=args.model or 'preds')
            if args.out:
                report.save(args.out)
            if out_dir:
                report.save(f'{out_dir}/reports/{name}.report.json')
                with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f:
                    f.write(render(report, style='text'))
            if len(args.datasets) == 1 or args.verbose:
                print(render(report, style=args.style))
                print(render(report, style=args.style))
            primary = next(iter(report.metrics), '')
            secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0)
                             for s in report.samples)
            groups = {k: v for k, v in report.metric_groups.items()
                      if isinstance(v, dict) and k not in ('run_info',)
                      and not k.startswith('agg_error')}
            rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary),
                         'n': report.num_samples,
                         'extract_fail': report.num_failed_extractions,
                         'secs': round(secs_total, 1),
                         'hours': round(secs_total / 3600, 2),
                         'groups': groups, 'ok': True})
        except Exception as e:
            rows.append({'name': name, 'metric': '-', 'value': None,
                         'secs': round(_time.time() - t0, 1), 'ok': False,
                         'err': f'{type(e).__name__}: {str(e)[:100]}'})
            print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr)

    if len(rows) > 1:
        print(f'\n{"benchmark":<20} {"metric":<14} {"score":>8} {"n":>5} {"time":>9}')
        print('-' * 62)
        for r in rows:
            val = 'ERR' if not r['ok'] else _f3(r['value'])
            h = r.get('hours') or 0
            tdisp = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s"
            print(f"{r['name']:<20} {r['metric']:<14} {val!s:>8} {r.get('n', '')!s:>5} {tdisp:>9}"
                  + (f"  {r.get('err', '')}" if not r['ok'] else ''))
        ok = sum(1 for r in rows if r['ok'])
        print(f'\n{ok}/{len(rows)} ok' + (f' -> artifacts in {out_dir}/' if out_dir else ''))
        if out_dir:
            import csv as _csv

            with open(f'{out_dir}/viz/summary.csv', 'w', newline='', encoding='utf-8') as f:
                w = _csv.writer(f)
                w.writerow(['benchmark', 'score', 'metric', 'num_samples',
                            'time_h', 'time_s', 'extract_fail', 'categories'])
                for r in rows:
                    cats = '; '.join(f'{g}={_f3(v)}'
                                     for gname, gv in (r.get('groups') or {}).items()
                                     for g, v in (gv or {}).items()
                                     if isinstance(v, (int, float)))[:2000]
                    w.writerow([r['name'], _f3(r.get('value')), r['metric'], r.get('n', ''),
                                r.get('hours', ''), r.get('secs', ''),
                                r.get('extract_fail', 0), cats])
            with open(f'{out_dir}/viz/summary.md', 'w', encoding='utf-8') as f:
                f.write(f'# eval run summary\n\n| dataset | metric | value | secs |\n|---|---|---|---|\n')
                for r in rows:
                    f.write(f"| {r['name']} | {r['metric']} | {r['value']} | {r['secs']} |\n")
        print(f'summary csv -> {out_dir}/viz/summary.csv')
    return 0 if all(r['ok'] for r in rows) else 1


def _f3(v):
    try:
        return round(float(v), 4)
    except (TypeError, ValueError):
        return v


def _cmd_viz_show(args) -> int:
    from evalharness.viz import render

    print(render([*args.reports], style=args.style, **({'n': args.n} if args.n else {})))
    return 0


def build_parser() -> argparse.ArgumentParser:
    parser = argparse.ArgumentParser(prog='evalharness', description='EvalHarness CLI')
    sub = parser.add_subparsers(dest='command', required=True)

    data = sub.add_parser('data', help='dataset plugin commands')
    dsub = data.add_subparsers(dest='data_command', required=True)

    p = dsub.add_parser('list', help='list registered datasets (no download)')
    p.set_defaults(func=_cmd_data_list)

    p = dsub.add_parser('fetch', help='materialize dataset(s) into the cache')
    p.add_argument('names', nargs='+')
    p.add_argument('--force', action='store_true', help='re-download and rebuild the cache')
    p.add_argument('--workers', type=int, default=8, help='concurrent downloads (default 8)')
    _add_override_flags(p)
    p.set_defaults(func=_cmd_data_fetch)

    p = dsub.add_parser('unload', help='drop cache entries (raw + samples); images belong to the sandbox layer')
    p.add_argument('names', nargs='+')
    _add_override_flags(p)
    p.set_defaults(func=_cmd_data_unload)

    p = dsub.add_parser('stats', help='materialize and show dataset statistics')
    p.add_argument('name')
    _add_override_flags(p)
    p.set_defaults(func=_cmd_data_stats)

    p = dsub.add_parser('show', help='print the first N samples')
    p.add_argument('name')
    p.add_argument('-n', type=int, default=2)
    _add_override_flags(p)
    p.set_defaults(func=_cmd_data_show)

    # ---- eval ----
    ev = sub.add_parser('eval', help='evaluation recipes & runs')
    esub = ev.add_subparsers(dest='eval_command', required=True)

    p = esub.add_parser('list', help='list registered eval recipes')
    p.set_defaults(func=_cmd_eval_list)

    p = esub.add_parser('run', help='score predictions (file) or generate+score (--model); multiple datasets OK')
    p.add_argument('datasets', nargs='+', help='dataset name(s) (recipe auto-resolved)')
    p.add_argument('predictions', nargs='?', help='jsonl: one raw string or {"raw": ...} per sample')
    p.add_argument('--model', default='',
                   help="generate with model spec: mock | mock:boxed | "
                        "openai/http://host:8000/v1?model | deploy:vllm/model")
    p.add_argument('--judge', default='', help='judge model spec for llm_judge recipes')
    p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump")
    p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)')
    p.add_argument('--limit', type=int, help='evaluate only the first N samples total')
    p.add_argument('--resume', nargs='?', const=True, default=False,
                   help='resume from per-sample checkpoint (default path auto-derived; '
                        'pass a path to override)')
    p.add_argument('--limit-per-task', type=int,
                   help='first N samples PER subset/category (evalscope --limit semantics); '
                        'composable with --limit (intersection)')
    p.add_argument('--out', help='save the EvalReport json here (single dataset)')
    p.add_argument('--out-dir', help='save reports/<name>.json + viz/<name>.txt + summary.md '
                                     'here (multi-dataset runs)')
    p.add_argument('--style', default='text', help='result render style (text/md/radar/errors)')
    p.add_argument('--verbose', action='store_true', help='print full render for every dataset')
    _add_override_flags(p)
    p.set_defaults(func=_cmd_eval_run)

    # ---- sandbox ----
    sb = sub.add_parser('sandbox', help='execution environment management')
    bsub = sb.add_subparsers(dest='sandbox_command', required=True)

    p = bsub.add_parser('prefetch', help='parallel docker pull of a dataset\'s sandbox images')
    p.add_argument('dataset', help='dataset whose samples declare images (e.g. swe_bench_verified)')
    p.add_argument('--workers', type=int, default=8, help='concurrent pulls (default 8)')
    p.add_argument('--limit', type=int, default=0, help='only first N samples (0=all)')
    _add_override_flags(p)
    p.set_defaults(func=_cmd_sandbox_prefetch)

    # ---- viz ----
    vz = sub.add_parser('viz', help='render saved EvalReport artifacts')
    zsub = vz.add_subparsers(dest='viz_command', required=True)

    p = zsub.add_parser('show', help='render report file(s)')
    p.add_argument('reports', nargs='+')
    p.add_argument('--style', default='text',
                   help='text | md | md_compare (multi) | radar | errors')
    p.add_argument('-n', type=int, help='for errors style: how many samples')
    p.set_defaults(func=_cmd_viz_show)

    return parser


def main(argv=None) -> int:
    args = build_parser().parse_args(argv)
    return args.func(args)


if __name__ == '__main__':
    sys.exit(main())