From 3d16ab9103d55e33e87e0811f9cb0caf8a388ee9 Mon Sep 17 00:00:00 2001 From: sora Date: Mon, 24 Aug 2026 08:43:49 +0000 Subject: [PATCH] Add SWE-bench single-turn harness (patch apply + FAIL_TO_PASS in official sweb image, sh entries in docker sandbox), bigcodebench official all-libs image, docker default for humaneval/LCB, input truncation (max_input_chars) + context assembly (passage/context), API retry with backoff, judge thread-safe bridge, parquet blob integrity check, summary.csv in out-dir, sandbox prefetch CLI --- evalharness/cli.py | 125 ++++++++++++++++++++---- evalharness/data/loader.py | 20 +++- evalharness/eval/recipes/agent.py | 57 ++++++++--- evalharness/eval/runner.py | 6 +- evalharness/eval/scorer.py | 3 +- evalharness/model/adapter.py | 19 +++- evalharness/model/runner.py | 62 +++++++++--- evalharness/sandbox/__init__.py | 2 + evalharness/sandbox/docker.py | 4 +- evalharness/sandbox/prefetch.py | 78 +++++++++++++++ evalharness/viz/renderers/html.py | 157 ++++++++++++++++++++++++++++++ 11 files changed, 480 insertions(+), 53 deletions(-) create mode 100644 evalharness/sandbox/prefetch.py create mode 100644 evalharness/viz/renderers/html.py diff --git a/evalharness/cli.py b/evalharness/cli.py index 82a1bc4..26ac634 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -91,6 +91,22 @@ def _cmd_data_unload(args) -> int: return 0 +def _cmd_sandbox_prefetch(args) -> int: + from evalharness.data import get_dataset + from evalharness.sandbox import docker_available, images_for_dataset, prefetch_images + + if not docker_available(): + print('docker is not available on this host', file=sys.stderr) + return 1 + ds = get_dataset(args.dataset, **_overrides(args)) + images = images_for_dataset(ds, limit=args.limit) + if not images: + print(f'{args.dataset}: no sandbox images declared by its samples') + return 0 + prefetch_images(images, workers=args.workers) + return 0 + + def _add_override_flags(p: argparse.ArgumentParser) -> None: p.add_argument('--source', help='override DatasetSpec.source (e.g. a local dir)') p.add_argument('--split', help='override DatasetSpec.split') @@ -109,28 +125,85 @@ def _cmd_eval_list(_args) -> int: def _cmd_eval_run(args) -> int: import asyncio + import time as _time from evalharness.data import get_dataset from evalharness.viz import render - ds = get_dataset(args.dataset, **_overrides(args)) - if args.model: # generate + score in one go - from evalharness.model import run_eval + overrides = _overrides(args) + out_dir = args.out_dir + if out_dir: + from pathlib import Path - report = asyncio.run(run_eval( - ds, args.model, concurrency=args.concurrency, limit=args.limit, - judge_spec=args.judge, env=args.env)) - else: - from evalharness.eval import evaluate + Path(out_dir).mkdir(parents=True, exist_ok=True) + (Path(out_dir) / 'viz').mkdir(exist_ok=True) - preds = [json.loads(line) for line in open(args.predictions, encoding='utf-8') if line.strip()] - preds = [p.get('raw', p.get('prediction', '')) if isinstance(p, dict) else p for p in preds] - report = evaluate(ds, preds, model=args.model or 'preds') - if args.out: - report.save(args.out) - print(f'saved -> {args.out}') - print(render(report, style=args.style)) - return 0 + rows = [] + for i, name in enumerate(args.datasets): + t0 = _time.time() + try: + ds = get_dataset(name, **overrides) + if args.model: # generate + score in one go + from evalharness.model import run_eval + + report = asyncio.run(run_eval( + ds, args.model, concurrency=args.concurrency, limit=args.limit, + judge_spec=args.judge, env=args.env)) + else: + from evalharness.eval import evaluate + + if not args.predictions: + raise SystemExit('error: provide --model or a predictions file') + preds_path = args.predictions[i] if len(args.predictions) > i else args.predictions[0] + preds = [json.loads(line) for line in open(preds_path, encoding='utf-8') if line.strip()] + preds = [p.get('raw', p.get('prediction', '')) if isinstance(p, dict) else p + for p in preds] + report = evaluate(ds, preds, model=args.model or 'preds') + if args.out: + report.save(args.out) + if out_dir: + report.save(f'{out_dir}/reports/{name}.report.json') + with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f: + f.write(render(report, style='text')) + if len(args.datasets) == 1 or args.verbose: + print(render(report, style=args.style)) + print(render(report, style=args.style)) + primary = next(iter(report.metrics), '') + rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary), + 'n': report.num_samples, + 'extract_fail': report.num_failed_extractions, + 'secs': round(_time.time() - t0, 1), 'ok': True}) + except Exception as e: + rows.append({'name': name, 'metric': '-', 'value': None, + 'secs': round(_time.time() - t0, 1), 'ok': False, + 'err': f'{type(e).__name__}: {str(e)[:100]}'}) + print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr) + + if len(rows) > 1: + print(f'\n{"dataset":<20} {"metric":<14} {"value":<8} {"secs":>5}') + print('-' * 52) + for r in rows: + val = 'ERR' if not r['ok'] else round(r['value'], 4) + print(f"{r['name']:<20} {r['metric']:<14} {val!s:<8} {r['secs']:>5}" + + (f" {r.get('err', '')}" if not r['ok'] else '')) + ok = sum(1 for r in rows if r['ok']) + print(f'\n{ok}/{len(rows)} ok' + (f' -> artifacts in {out_dir}/' if out_dir else '')) + if out_dir: + import csv as _csv + + with open(f'{out_dir}/viz/summary.csv', 'w', newline='', encoding='utf-8') as f: + w = _csv.writer(f) + w.writerow(['dataset', 'model', 'n', 'metric', 'value', 'extract_fail', 'secs']) + for r in rows: + w.writerow([r['name'], args.model, r.get('n', ''), + r['metric'], r.get('value', ''), r.get('extract_fail', ''), + r['secs']]) + with open(f'{out_dir}/viz/summary.md', 'w', encoding='utf-8') as f: + f.write(f'# eval run summary\n\n| dataset | metric | value | secs |\n|---|---|---|---|\n') + for r in rows: + f.write(f"| {r['name']} | {r['metric']} | {r['value']} | {r['secs']} |\n") + print(f'summary csv -> {out_dir}/viz/summary.csv') + return 0 if all(r['ok'] for r in rows) else 1 def _cmd_viz_show(args) -> int: @@ -180,8 +253,8 @@ def build_parser() -> argparse.ArgumentParser: p = esub.add_parser('list', help='list registered eval recipes') p.set_defaults(func=_cmd_eval_list) - p = esub.add_parser('run', help='score predictions (file) or generate+score (--model)') - p.add_argument('dataset', help='dataset name (recipe auto-resolved)') + p = esub.add_parser('run', help='score predictions (file) or generate+score (--model); multiple datasets OK') + p.add_argument('datasets', nargs='+', help='dataset name(s) (recipe auto-resolved)') p.add_argument('predictions', nargs='?', help='jsonl: one raw string or {"raw": ...} per sample') p.add_argument('--model', default='', help="generate with model spec: mock | mock:boxed | " @@ -190,11 +263,25 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump") p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)') p.add_argument('--limit', type=int, help='evaluate only the first N samples') - p.add_argument('--out', help='save the EvalReport json here') + p.add_argument('--out', help='save the EvalReport json here (single dataset)') + p.add_argument('--out-dir', help='save reports/.json + viz/.txt + summary.md ' + 'here (multi-dataset runs)') p.add_argument('--style', default='text', help='result render style (text/md/radar/errors)') + p.add_argument('--verbose', action='store_true', help='print full render for every dataset') _add_override_flags(p) p.set_defaults(func=_cmd_eval_run) + # ---- sandbox ---- + sb = sub.add_parser('sandbox', help='execution environment management') + bsub = sb.add_subparsers(dest='sandbox_command', required=True) + + p = bsub.add_parser('prefetch', help='parallel docker pull of a dataset\'s sandbox images') + p.add_argument('dataset', help='dataset whose samples declare images (e.g. swe_bench_verified)') + p.add_argument('--workers', type=int, default=8, help='concurrent pulls (default 8)') + p.add_argument('--limit', type=int, default=0, help='only first N samples (0=all)') + _add_override_flags(p) + p.set_defaults(func=_cmd_sandbox_prefetch) + # ---- viz ---- vz = sub.add_parser('viz', help='render saved EvalReport artifacts') zsub = vz.add_subparsers(dest='viz_command', required=True) diff --git a/evalharness/data/loader.py b/evalharness/data/loader.py index 9b03a0d..14fd930 100644 --- a/evalharness/data/loader.py +++ b/evalharness/data/loader.py @@ -200,11 +200,24 @@ def _ms_match_files(spec: DatasetSpec, files: List[str]) -> List[str]: return sorted(in_subset_dir or candidates) +def _parquet_ok(path: Path) -> bool: + """Cheap integrity check: parquet files end with magic 'PAR1'.""" + try: + with open(path, 'rb') as f: + f.seek(-4, 2) + return f.read(4) == b'PAR1' + except OSError: + return False + + def _ms_download(repo: str, path: str, dest_dir: Path) -> Path: """Download one repo file into the raw cache (content-addressed, reused).""" dest = dest_dir / os.path.basename(path) - if dest.exists() and dest.stat().st_size > 0: + if dest.exists() and dest.stat().st_size > 0 \ + and (os.path.splitext(dest)[1] != '.parquet' or _parquet_ok(dest)): return dest + if dest.exists(): # truncated/corrupt (e.g. an interrupted download) + dest.unlink() dest_dir.mkdir(parents=True, exist_ok=True) url = f'{_MS_API}/{repo}/repo?Revision=master&FilePath={path}' tmp = dest.with_name(dest.name + f'.part-{os.getpid()}') @@ -325,8 +338,11 @@ def _hf_match_files(spec: DatasetSpec, files: List[str]) -> List[str]: def _hf_download(repo: str, path: str, dest_dir: Path) -> Path: """Download one repo file (follows the CDN redirect) into the blob store.""" dest = dest_dir / os.path.basename(path) - if dest.exists() and dest.stat().st_size > 0: + if dest.exists() and dest.stat().st_size > 0 \ + and (os.path.splitext(dest)[1] != '.parquet' or _parquet_ok(dest)): return dest + if dest.exists(): # truncated/corrupt (e.g. an interrupted download) + dest.unlink() dest_dir.mkdir(parents=True, exist_ok=True) url = f'{_hf_base()}/datasets/{repo}/resolve/main/{path}' tmp = dest.with_name(dest.name + f'.part-{os.getpid()}') diff --git a/evalharness/eval/recipes/agent.py b/evalharness/eval/recipes/agent.py index f54d595..c287753 100644 --- a/evalharness/eval/recipes/agent.py +++ b/evalharness/eval/recipes/agent.py @@ -18,20 +18,17 @@ def humaneval(): name='humaneval', extract='code_any', scorers={'pass': {'name': 'execution', 'harness': _humaneval_harness, - 'sandbox': 'local', 'timeout_s': 30}}, + 'sandbox': 'docker', 'timeout_s': 30}}, aggregators={'pass': 'pass_at_k'}, description='HumanEval; completion + official tests in a sandbox, pass@k.', ) -def _bcb_harness_factory(requirements: str): - def harness(sample, pred: str): - test = sample.metadata.get('test', '') - entry = sample.metadata.get('entry_point', 'f') - prog = f'{sample.input}{pred}\n\n{test}\n\ncheck({entry})\nprint("PASSED")\n' - return {'main.py': prog} - - return harness +def _bcb_harness(sample, pred: str): + test = sample.metadata.get('test', '') + entry = sample.metadata.get('entry_point', 'f') + prog = f'{sample.input}{pred}\n\n{test}\n\ncheck({entry})\nprint("PASSED")\n' + return {'main.py': prog} @register_eval('bigcodebench') @@ -39,10 +36,12 @@ def bigcodebench(): return EvalRecipe( name='bigcodebench', extract='code_any', - scorers={'pass': {'name': 'execution', 'harness': _bcb_harness_factory('libs'), + scorers={'pass': {'name': 'execution', 'harness': _bcb_harness, + # official image bundles every task's libs (sympy/pandas/...) + 'image': 'bigcodebench/bigcodebench-eval:latest', 'sandbox': 'docker', 'timeout_s': 120}}, aggregators={'pass': 'pass_at_k'}, - description='BigCodeBench; library-level tasks need the docker sandbox (pip deps).', + description='BigCodeBench; official all-libs docker image, pass@k.', ) @@ -91,13 +90,45 @@ def live_code_bench(): ) +import json as _json + + +def _swe_harness(sample, pred: str): + """Apply the predicted patch in the official per-instance sweb image and + run FAIL_TO_PASS (+PASS_TO_PASS) tests. Single-turn protocol: the model + reads problem_statement and emits a unified diff.""" + f2p = _json.loads(sample.metadata.get('FAIL_TO_PASS') or '[]') + p2p = _json.loads(sample.metadata.get('PASS_TO_PASS') or '[]') + tests = f2p + p2p[:20] # guard: cap regression tests for runtime + script = f'''set -e +cd /testbed +git apply --whitespace=fix /work/patch.diff || {{ echo PATCH_FAILED; exit 2; }} +FAIL=0 +while IFS= read -r t; do + [ -z "$t" ] && continue + if ! (conda run -n testbed python -m pytest -x -q "$t" > /dev/null 2>&1); then + echo "TEST_FAILED $t"; FAIL=1 + fi +done <<'EOF' +{chr(10).join(tests)} +EOF +[ "$FAIL" = 0 ] && echo RESOLVED +exit $FAIL +''' + return {'patch.diff': pred or '', 'run.sh': script} + + @register_eval('swe_bench_verified') def swe_bench_verified(): return EvalRecipe( name='swe_bench_verified', extract='identity', # a patch, not an answer - scorers={'resolved': 'env_reward'}, - description='SWE-bench Verified; docker env, FAIL_TO_PASS/PASS_TO_PASS.', + scorers={'resolved': {'name': 'execution', 'harness': _swe_harness, + 'entry': 'run.sh', 'sandbox': 'docker', + 'timeout_s': 900}}, + description='SWE-bench Verified single-turn: model emits a unified diff; ' + 'applied in the official sweb.eval.* image, FAIL_TO_PASS(+P2P) ' + 'must pass. Prefetch: evalharness sandbox prefetch swe_bench_verified', ) diff --git a/evalharness/eval/runner.py b/evalharness/eval/runner.py index 9588c66..147bbba 100644 --- a/evalharness/eval/runner.py +++ b/evalharness/eval/runner.py @@ -61,9 +61,9 @@ def evaluate( task_type=sample.task_type, raw_prediction=raw, target=sample.target, - group_key=override.get('group_key') - or sample.metadata.get('group_key') - or (sample.metadata.get('task_id') or sample.metadata.get('id') or ''), + group_key=str(override.get('group_key') + or sample.metadata.get('group_key') + or (sample.metadata.get('task_id') or sample.metadata.get('id') or '')), metadata={k: v for k, v in (sample.metadata or {}).items() if k in ('category', 'subject', 'test_category', 'bin', 'difficulty')}, ) diff --git a/evalharness/eval/scorer.py b/evalharness/eval/scorer.py index b2ffaf1..c9a9eac 100644 --- a/evalharness/eval/scorer.py +++ b/evalharness/eval/scorer.py @@ -304,7 +304,8 @@ def execution(pred: str, target, sample: Sample, ctx: ScoreContext): sbx = get_sandbox(ctx.params.get('sandbox', 'local')) files = harness(sample, pred or '') result = sbx.exec(files, entry=ctx.params.get('entry', 'main.py'), - timeout_s=ctx.params.get('timeout_s', 30)) + timeout_s=ctx.params.get('timeout_s', 30), + image=ctx.params.get('image', '')) ok = result.ok return ({'pass': 1.0} if ok else {'pass': 0.0}), {'pass': { 'exit_code': result.exit_code, diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index 85cc4c6..e853a9c 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -121,8 +121,23 @@ class OpenAICompatible(ModelAdapter): headers = {'Content-Type': 'application/json'} if self.api_key: headers['Authorization'] = f'Bearer {self.api_key}' - data = await self._post(f'{self.api_base}/chat/completions', payload, headers) - return self._parse(data) + retries = self.extra.get('retries', 3) + last_exc: Exception = None + for attempt in range(retries + 1): + try: + data = await self._post(f'{self.api_base}/chat/completions', payload, headers) + return self._parse(data) + except Exception as e: # 5xx/429/timeouts: worth retrying + last_exc = e + retryable = 'Server error' in str(e) or '504' in str(e) or '502' in str(e) \ + or '429' in str(e) or 'timeout' in str(e).lower() \ + or 'TimeoutException' in type(e).__name__ + if attempt >= retries or not retryable: + raise + import asyncio + + await asyncio.sleep(min(2 ** attempt * 2, 30)) + raise last_exc # unreachable def _payload(self, messages, tools, kw) -> Dict[str, Any]: msgs = [{'role': m.role, 'content': m.content} for m in messages] diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 4749a28..332dbc1 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -33,14 +33,38 @@ async def generate_predictions( env_factory=None, system: str = '', max_turns: int = 8, + max_input_chars: int = 0, + attach_context_keys: tuple = ('passage', 'context'), ) -> tuple: """Fan out model calls; returns (pred-dicts, total_usage). Without env_factory: single-turn generation (text or tool-call JSON). With env_factory(sample)->Environment: the agent message pump runs per sample and predictions carry trajectory/env_state/usage. + + max_input_chars: hard cap on the assembled input (anti-OOM for 128k + contexts); 0 = no cap. Truncation keeps the head AND the question tail. + attach_context_keys: metadata fields (passage/context) prepended to the + question at generation time -- the data layer keeps them separate, the + runner assembles the actual prompt. """ gen_kwargs = gen_kwargs or {} + + def assemble(sample: Sample) -> str: + parts = [] + for key in attach_context_keys: + ctx = (sample.metadata or {}).get(key) + if ctx: + parts.append(str(ctx)) + parts.append(sample.input_text) + text = '\n\n'.join(parts) + if max_input_chars and len(text) > max_input_chars: + keep = max_input_chars // 2 + head = text[:keep] + tail = text[-keep:] + text = f'{head}\n\n...[truncated {len(text) - 2 * keep} chars]...\n\n{tail}' + return text + sem = asyncio.Semaphore(concurrency) total_usage = Usage() done_count = 0 @@ -64,8 +88,8 @@ async def generate_predictions( _progress(progress, done_count, len(samples), t0, total_usage) return pred - messages = ([ChatMessage(role='user', content=sample.input)] if isinstance(sample.input, str) - else list(sample.input)) + messages = ([ChatMessage(role='user', content=assemble(sample))] + if isinstance(sample.input, str) else list(sample.input)) tools = None if sample.tools: tools = [{'name': t.name, 'description': t.description or '', @@ -115,6 +139,7 @@ async def run_eval( env: str = '', system: str = '', max_turns: int = 8, + max_input_chars: int = 0, ) -> EvalReport: """Generate + score in one call. Model spec examples: 'mock', 'mock:boxed', 'openai/http://gpu03:8000/v1?qwen3-8b', 'deploy:vllm/qwen3-8b'. @@ -154,7 +179,7 @@ async def run_eval( preds, _usages, usage = await generate_predictions( adapter, samples, concurrency, progress=progress, gen_kwargs=gen_kwargs, env_factory=env_factory, - system=system, max_turns=max_turns) + system=system, max_turns=max_turns, max_input_chars=max_input_chars) finally: await adapter.close() if judge is None and judge_spec: @@ -190,15 +215,28 @@ def _make_adapter(spec: str) -> ModelAdapter: def _judge_callable(judge_adapter: ModelAdapter): - async def ask(messages) -> str: - out = await judge_adapter.generate([ChatMessage(role='user', content=str(m)) for m in messages] - if isinstance(messages, list) and messages and isinstance(messages[0], dict) - else messages) - return out.text + """Sync judge bridge. Works inside a running event loop (evaluate() may be + called from async run_eval): the coroutine runs on a private loop in a + worker thread.""" - import asyncio + def ask(messages) -> str: + import asyncio - def sync_ask(messages): - return asyncio.run(ask(messages)) + if isinstance(messages, list) and messages and isinstance(messages[0], dict): + messages = [ChatMessage(role=m.get('role', 'user'), content=m.get('content', '')) + for m in messages] - return sync_ask + async def go(): + out = await judge_adapter.generate(messages) + return out.text + + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(go()) + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, go()).result() + + return ask diff --git a/evalharness/sandbox/__init__.py b/evalharness/sandbox/__init__.py index ca23bb6..2325025 100644 --- a/evalharness/sandbox/__init__.py +++ b/evalharness/sandbox/__init__.py @@ -23,9 +23,11 @@ from .base import ( ) from .docker import DockerSandbox, docker_available, docker_serve, serve_env from .local import LocalSandbox +from .prefetch import images_for_dataset, prefetch_images __all__ = [ 'Sandbox', 'DockerSandbox', 'LocalSandbox', 'ExecResult', 'EnvHandle', 'SANDBOX_REGISTRY', 'register_sandbox', 'get_sandbox', 'acquire', 'stop_all', 'docker_serve', 'serve_env', 'docker_available', + 'prefetch_images', 'images_for_dataset', ] diff --git a/evalharness/sandbox/docker.py b/evalharness/sandbox/docker.py index 6de7f4c..34398bb 100644 --- a/evalharness/sandbox/docker.py +++ b/evalharness/sandbox/docker.py @@ -61,7 +61,9 @@ class DockerSandbox(Sandbox): out_host = Path(hpath).expanduser() out_host.mkdir(parents=True, exist_ok=True) cmd += ['-v', f'{out_host}:{cpath}:rw'] - cmd += [img, 'python', f'/work/{entry}'] + runner = ['python', f'/work/{entry}'] if entry.endswith('.py') \ + else ['sh', f'/work/{entry}'] + cmd += [img, *runner] t0 = time.time() try: proc = _run(cmd, timeout=timeout_s + 30) diff --git a/evalharness/sandbox/prefetch.py b/evalharness/sandbox/prefetch.py new file mode 100644 index 0000000..b4aca65 --- /dev/null +++ b/evalharness/sandbox/prefetch.py @@ -0,0 +1,78 @@ +"""Image prefetch: parallel docker pull for execution environments. + +SWE-bench Verified declares ~500 per-instance images (sweb.eval.x86_64.*). +Pulling them lazily during an eval run would stall it serially; prefetch +pulls them ahead of time with bounded concurrency and progress reporting. + + from evalharness.sandbox.prefetch import prefetch_images + done = prefetch_images(['sweb.eval.x86_64.django__django-12345', ...], workers=8) + # CLI: evalharness sandbox prefetch swe_bench_verified --workers 8 --limit 50 +""" + +import subprocess +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from typing import Iterable, List + +from ..data.dataset import Dataset + + +def images_for_dataset(ds: Dataset, limit: int = 0) -> List[str]: + """Collect distinct sandbox images declared by a dataset's samples.""" + seen: List[str] = [] + add = seen.append + for i, s in enumerate(ds): + if limit and i >= limit: + break + if s.sandbox and s.sandbox.image and s.sandbox.image not in seen: + add(s.sandbox.image) + return seen + + +def local_images() -> set: + out = subprocess.run(['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'], + capture_output=True, text=True) + return {line for line in out.stdout.splitlines() if line} + + +def prefetch_images(images: Iterable[str], workers: int = 8) -> List[str]: + """Pull images with bounded concurrency; skip ones already local. + + Returns the list newly pulled. Failures are reported and skipped (one + missing instance must not block the rest). + """ + have = local_images() + todo = [img for img in images if img not in have] + if not todo: + print(f'prefetch: all {len(images)} images already local') + return [] + print(f'prefetch: {len(todo)} to pull ({len(images) - len(todo)} already local), ' + f'workers={workers}') + pulled: List[str] = [] + failed: List[str] = [] + t0 = time.time() + done = 0 + with ThreadPoolExecutor(max_workers=workers) as pool: + futs = {pool.submit(_pull_one, img): img for img in todo} + for fut in as_completed(futs): + img = futs[fut] + done += 1 + try: + fut.result() + pulled.append(img) + except Exception as e: + failed.append(img) + print(f' FAIL {img}: {str(e)[:120]}', flush=True) + if done % 10 == 0 or done == len(todo): + rate = done / max(time.time() - t0, 1e-6) + print(f' [{done}/{len(todo)}] {rate:.2f} imgs/s, ' + f'{len(failed)} failed', flush=True) + print(f'prefetch done: {len(pulled)} pulled, {len(failed)} failed ' + f'in {time.time() - t0:.0f}s') + return pulled + + +def _pull_one(image: str) -> None: + r = subprocess.run(['docker', 'pull', image], capture_output=True, text=True, timeout=3600) + if r.returncode != 0: + raise RuntimeError(r.stderr.strip()[:200]) diff --git a/evalharness/viz/renderers/html.py b/evalharness/viz/renderers/html.py new file mode 100644 index 0000000..7fb7d08 --- /dev/null +++ b/evalharness/viz/renderers/html.py @@ -0,0 +1,157 @@ +"""HTML renderer: a self-contained dashboard page (no server, no deps). + +Generates one index.html with sortable metric tables, per-benchmark bars, +per-sample drill-down and error browser. Consumed by CLI --out-dir. +""" + +import html +import json +from typing import Dict, List, Union + +from ...eval.record import EvalReport +from .. import register_renderer + + +@register_renderer('html') +def html_dashboard(target: Union[EvalReport, List[EvalReport]], opts: Dict) -> str: + reports = target if isinstance(target, list) else [target] + reports = [r for r in reports if isinstance(r, EvalReport)] + cards = [] + for rep in reports: + cards.append(_card(rep)) + rows_json = json.dumps([_row(r) for r in reports], ensure_ascii=False) + return _PAGE.replace('__ROWS__', rows_json).replace('__CARDS__', '\n'.join(cards)) + + +def _row(rep: EvalReport) -> Dict: + primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '') + return { + 'dataset': rep.dataset, 'recipe': rep.recipe, 'model': rep.model, + 'n': rep.num_samples, 'primary': primary, + 'value': rep.metrics.get(primary, 0.0), + 'metrics': {k: v for k, v in rep.metrics.items()}, + 'groups': {k: v for k, v in rep.metric_groups.items() if isinstance(v, dict)}, + 'extract_fail': rep.num_failed_extractions, + 'samples': [ + {'id': s.sample_id, 'ok': s.extraction_ok, 'extracted': s.extracted_prediction[:200], + 'raw': s.raw_prediction[:300], 'target': str(s.target)[:120], + 'scores': s.scores, 'error': s.error[:200]} + for s in rep.samples[:200] + ], + } + + +def _card(rep: EvalReport) -> str: + primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '') + value = rep.metrics.get(primary, 0.0) + pct = f'{value * 100:.1f}%' + warn = (f'extraction failures: {rep.num_failed_extractions}' + f'/{rep.num_samples}') if rep.num_failed_extractions else '' + return (f'
{html.escape(rep.dataset)}
' + f'
{pct}
{html.escape(primary)} · n={rep.num_samples}' + f' · {html.escape(rep.model or "?")}
{warn}
') + + +_PAGE = """ + + + +EvalHarness Dashboard + + + +

EvalHarness Dashboard generated by evalharness viz

+
+
__CARDS__
+ + + + +
datasetmodelnmetricvalueshareextract✗
+
click a row for per-sample details +
+
+ + +"""