Add SWE-bench single-turn harness (patch apply + FAIL_TO_PASS in official sweb image, sh entries in docker sandbox), bigcodebench official all-libs image, docker default for humaneval/LCB, input truncation (max_input_chars) + context assembly (passage/context), API retry with backoff, judge thread-safe bridge, parquet blob integrity check, summary.csv in out-dir, sandbox prefetch CLI
This commit is contained in:
parent
b2e7133b20
commit
3d16ab9103
@ -91,6 +91,22 @@ def _cmd_data_unload(args) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def _cmd_sandbox_prefetch(args) -> int:
|
||||
from evalharness.data import get_dataset
|
||||
from evalharness.sandbox import docker_available, images_for_dataset, prefetch_images
|
||||
|
||||
if not docker_available():
|
||||
print('docker is not available on this host', file=sys.stderr)
|
||||
return 1
|
||||
ds = get_dataset(args.dataset, **_overrides(args))
|
||||
images = images_for_dataset(ds, limit=args.limit)
|
||||
if not images:
|
||||
print(f'{args.dataset}: no sandbox images declared by its samples')
|
||||
return 0
|
||||
prefetch_images(images, workers=args.workers)
|
||||
return 0
|
||||
|
||||
|
||||
def _add_override_flags(p: argparse.ArgumentParser) -> None:
|
||||
p.add_argument('--source', help='override DatasetSpec.source (e.g. a local dir)')
|
||||
p.add_argument('--split', help='override DatasetSpec.split')
|
||||
@ -109,28 +125,85 @@ def _cmd_eval_list(_args) -> int:
|
||||
|
||||
def _cmd_eval_run(args) -> int:
|
||||
import asyncio
|
||||
import time as _time
|
||||
|
||||
from evalharness.data import get_dataset
|
||||
from evalharness.viz import render
|
||||
|
||||
ds = get_dataset(args.dataset, **_overrides(args))
|
||||
if args.model: # generate + score in one go
|
||||
from evalharness.model import run_eval
|
||||
overrides = _overrides(args)
|
||||
out_dir = args.out_dir
|
||||
if out_dir:
|
||||
from pathlib import Path
|
||||
|
||||
report = asyncio.run(run_eval(
|
||||
ds, args.model, concurrency=args.concurrency, limit=args.limit,
|
||||
judge_spec=args.judge, env=args.env))
|
||||
else:
|
||||
from evalharness.eval import evaluate
|
||||
Path(out_dir).mkdir(parents=True, exist_ok=True)
|
||||
(Path(out_dir) / 'viz').mkdir(exist_ok=True)
|
||||
|
||||
preds = [json.loads(line) for line in open(args.predictions, encoding='utf-8') if line.strip()]
|
||||
preds = [p.get('raw', p.get('prediction', '')) if isinstance(p, dict) else p for p in preds]
|
||||
report = evaluate(ds, preds, model=args.model or 'preds')
|
||||
if args.out:
|
||||
report.save(args.out)
|
||||
print(f'saved -> {args.out}')
|
||||
print(render(report, style=args.style))
|
||||
return 0
|
||||
rows = []
|
||||
for i, name in enumerate(args.datasets):
|
||||
t0 = _time.time()
|
||||
try:
|
||||
ds = get_dataset(name, **overrides)
|
||||
if args.model: # generate + score in one go
|
||||
from evalharness.model import run_eval
|
||||
|
||||
report = asyncio.run(run_eval(
|
||||
ds, args.model, concurrency=args.concurrency, limit=args.limit,
|
||||
judge_spec=args.judge, env=args.env))
|
||||
else:
|
||||
from evalharness.eval import evaluate
|
||||
|
||||
if not args.predictions:
|
||||
raise SystemExit('error: provide --model or a predictions file')
|
||||
preds_path = args.predictions[i] if len(args.predictions) > i else args.predictions[0]
|
||||
preds = [json.loads(line) for line in open(preds_path, encoding='utf-8') if line.strip()]
|
||||
preds = [p.get('raw', p.get('prediction', '')) if isinstance(p, dict) else p
|
||||
for p in preds]
|
||||
report = evaluate(ds, preds, model=args.model or 'preds')
|
||||
if args.out:
|
||||
report.save(args.out)
|
||||
if out_dir:
|
||||
report.save(f'{out_dir}/reports/{name}.report.json')
|
||||
with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f:
|
||||
f.write(render(report, style='text'))
|
||||
if len(args.datasets) == 1 or args.verbose:
|
||||
print(render(report, style=args.style))
|
||||
print(render(report, style=args.style))
|
||||
primary = next(iter(report.metrics), '')
|
||||
rows.append({'name': name, 'metric': primary, 'value': report.metrics.get(primary),
|
||||
'n': report.num_samples,
|
||||
'extract_fail': report.num_failed_extractions,
|
||||
'secs': round(_time.time() - t0, 1), 'ok': True})
|
||||
except Exception as e:
|
||||
rows.append({'name': name, 'metric': '-', 'value': None,
|
||||
'secs': round(_time.time() - t0, 1), 'ok': False,
|
||||
'err': f'{type(e).__name__}: {str(e)[:100]}'})
|
||||
print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr)
|
||||
|
||||
if len(rows) > 1:
|
||||
print(f'\n{"dataset":<20} {"metric":<14} {"value":<8} {"secs":>5}')
|
||||
print('-' * 52)
|
||||
for r in rows:
|
||||
val = 'ERR' if not r['ok'] else round(r['value'], 4)
|
||||
print(f"{r['name']:<20} {r['metric']:<14} {val!s:<8} {r['secs']:>5}"
|
||||
+ (f" {r.get('err', '')}" if not r['ok'] else ''))
|
||||
ok = sum(1 for r in rows if r['ok'])
|
||||
print(f'\n{ok}/{len(rows)} ok' + (f' -> artifacts in {out_dir}/' if out_dir else ''))
|
||||
if out_dir:
|
||||
import csv as _csv
|
||||
|
||||
with open(f'{out_dir}/viz/summary.csv', 'w', newline='', encoding='utf-8') as f:
|
||||
w = _csv.writer(f)
|
||||
w.writerow(['dataset', 'model', 'n', 'metric', 'value', 'extract_fail', 'secs'])
|
||||
for r in rows:
|
||||
w.writerow([r['name'], args.model, r.get('n', ''),
|
||||
r['metric'], r.get('value', ''), r.get('extract_fail', ''),
|
||||
r['secs']])
|
||||
with open(f'{out_dir}/viz/summary.md', 'w', encoding='utf-8') as f:
|
||||
f.write(f'# eval run summary\n\n| dataset | metric | value | secs |\n|---|---|---|---|\n')
|
||||
for r in rows:
|
||||
f.write(f"| {r['name']} | {r['metric']} | {r['value']} | {r['secs']} |\n")
|
||||
print(f'summary csv -> {out_dir}/viz/summary.csv')
|
||||
return 0 if all(r['ok'] for r in rows) else 1
|
||||
|
||||
|
||||
def _cmd_viz_show(args) -> int:
|
||||
@ -180,8 +253,8 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
p = esub.add_parser('list', help='list registered eval recipes')
|
||||
p.set_defaults(func=_cmd_eval_list)
|
||||
|
||||
p = esub.add_parser('run', help='score predictions (file) or generate+score (--model)')
|
||||
p.add_argument('dataset', help='dataset name (recipe auto-resolved)')
|
||||
p = esub.add_parser('run', help='score predictions (file) or generate+score (--model); multiple datasets OK')
|
||||
p.add_argument('datasets', nargs='+', help='dataset name(s) (recipe auto-resolved)')
|
||||
p.add_argument('predictions', nargs='?', help='jsonl: one raw string or {"raw": ...} per sample')
|
||||
p.add_argument('--model', default='',
|
||||
help="generate with model spec: mock | mock:boxed | "
|
||||
@ -190,11 +263,25 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump")
|
||||
p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)')
|
||||
p.add_argument('--limit', type=int, help='evaluate only the first N samples')
|
||||
p.add_argument('--out', help='save the EvalReport json here')
|
||||
p.add_argument('--out', help='save the EvalReport json here (single dataset)')
|
||||
p.add_argument('--out-dir', help='save reports/<name>.json + viz/<name>.txt + summary.md '
|
||||
'here (multi-dataset runs)')
|
||||
p.add_argument('--style', default='text', help='result render style (text/md/radar/errors)')
|
||||
p.add_argument('--verbose', action='store_true', help='print full render for every dataset')
|
||||
_add_override_flags(p)
|
||||
p.set_defaults(func=_cmd_eval_run)
|
||||
|
||||
# ---- sandbox ----
|
||||
sb = sub.add_parser('sandbox', help='execution environment management')
|
||||
bsub = sb.add_subparsers(dest='sandbox_command', required=True)
|
||||
|
||||
p = bsub.add_parser('prefetch', help='parallel docker pull of a dataset\'s sandbox images')
|
||||
p.add_argument('dataset', help='dataset whose samples declare images (e.g. swe_bench_verified)')
|
||||
p.add_argument('--workers', type=int, default=8, help='concurrent pulls (default 8)')
|
||||
p.add_argument('--limit', type=int, default=0, help='only first N samples (0=all)')
|
||||
_add_override_flags(p)
|
||||
p.set_defaults(func=_cmd_sandbox_prefetch)
|
||||
|
||||
# ---- viz ----
|
||||
vz = sub.add_parser('viz', help='render saved EvalReport artifacts')
|
||||
zsub = vz.add_subparsers(dest='viz_command', required=True)
|
||||
|
||||
@ -200,11 +200,24 @@ def _ms_match_files(spec: DatasetSpec, files: List[str]) -> List[str]:
|
||||
return sorted(in_subset_dir or candidates)
|
||||
|
||||
|
||||
def _parquet_ok(path: Path) -> bool:
|
||||
"""Cheap integrity check: parquet files end with magic 'PAR1'."""
|
||||
try:
|
||||
with open(path, 'rb') as f:
|
||||
f.seek(-4, 2)
|
||||
return f.read(4) == b'PAR1'
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _ms_download(repo: str, path: str, dest_dir: Path) -> Path:
|
||||
"""Download one repo file into the raw cache (content-addressed, reused)."""
|
||||
dest = dest_dir / os.path.basename(path)
|
||||
if dest.exists() and dest.stat().st_size > 0:
|
||||
if dest.exists() and dest.stat().st_size > 0 \
|
||||
and (os.path.splitext(dest)[1] != '.parquet' or _parquet_ok(dest)):
|
||||
return dest
|
||||
if dest.exists(): # truncated/corrupt (e.g. an interrupted download)
|
||||
dest.unlink()
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
url = f'{_MS_API}/{repo}/repo?Revision=master&FilePath={path}'
|
||||
tmp = dest.with_name(dest.name + f'.part-{os.getpid()}')
|
||||
@ -325,8 +338,11 @@ def _hf_match_files(spec: DatasetSpec, files: List[str]) -> List[str]:
|
||||
def _hf_download(repo: str, path: str, dest_dir: Path) -> Path:
|
||||
"""Download one repo file (follows the CDN redirect) into the blob store."""
|
||||
dest = dest_dir / os.path.basename(path)
|
||||
if dest.exists() and dest.stat().st_size > 0:
|
||||
if dest.exists() and dest.stat().st_size > 0 \
|
||||
and (os.path.splitext(dest)[1] != '.parquet' or _parquet_ok(dest)):
|
||||
return dest
|
||||
if dest.exists(): # truncated/corrupt (e.g. an interrupted download)
|
||||
dest.unlink()
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
url = f'{_hf_base()}/datasets/{repo}/resolve/main/{path}'
|
||||
tmp = dest.with_name(dest.name + f'.part-{os.getpid()}')
|
||||
|
||||
@ -18,20 +18,17 @@ def humaneval():
|
||||
name='humaneval',
|
||||
extract='code_any',
|
||||
scorers={'pass': {'name': 'execution', 'harness': _humaneval_harness,
|
||||
'sandbox': 'local', 'timeout_s': 30}},
|
||||
'sandbox': 'docker', 'timeout_s': 30}},
|
||||
aggregators={'pass': 'pass_at_k'},
|
||||
description='HumanEval; completion + official tests in a sandbox, pass@k.',
|
||||
)
|
||||
|
||||
|
||||
def _bcb_harness_factory(requirements: str):
|
||||
def harness(sample, pred: str):
|
||||
test = sample.metadata.get('test', '')
|
||||
entry = sample.metadata.get('entry_point', 'f')
|
||||
prog = f'{sample.input}{pred}\n\n{test}\n\ncheck({entry})\nprint("PASSED")\n'
|
||||
return {'main.py': prog}
|
||||
|
||||
return harness
|
||||
def _bcb_harness(sample, pred: str):
|
||||
test = sample.metadata.get('test', '')
|
||||
entry = sample.metadata.get('entry_point', 'f')
|
||||
prog = f'{sample.input}{pred}\n\n{test}\n\ncheck({entry})\nprint("PASSED")\n'
|
||||
return {'main.py': prog}
|
||||
|
||||
|
||||
@register_eval('bigcodebench')
|
||||
@ -39,10 +36,12 @@ def bigcodebench():
|
||||
return EvalRecipe(
|
||||
name='bigcodebench',
|
||||
extract='code_any',
|
||||
scorers={'pass': {'name': 'execution', 'harness': _bcb_harness_factory('libs'),
|
||||
scorers={'pass': {'name': 'execution', 'harness': _bcb_harness,
|
||||
# official image bundles every task's libs (sympy/pandas/...)
|
||||
'image': 'bigcodebench/bigcodebench-eval:latest',
|
||||
'sandbox': 'docker', 'timeout_s': 120}},
|
||||
aggregators={'pass': 'pass_at_k'},
|
||||
description='BigCodeBench; library-level tasks need the docker sandbox (pip deps).',
|
||||
description='BigCodeBench; official all-libs docker image, pass@k.',
|
||||
)
|
||||
|
||||
|
||||
@ -91,13 +90,45 @@ def live_code_bench():
|
||||
)
|
||||
|
||||
|
||||
import json as _json
|
||||
|
||||
|
||||
def _swe_harness(sample, pred: str):
|
||||
"""Apply the predicted patch in the official per-instance sweb image and
|
||||
run FAIL_TO_PASS (+PASS_TO_PASS) tests. Single-turn protocol: the model
|
||||
reads problem_statement and emits a unified diff."""
|
||||
f2p = _json.loads(sample.metadata.get('FAIL_TO_PASS') or '[]')
|
||||
p2p = _json.loads(sample.metadata.get('PASS_TO_PASS') or '[]')
|
||||
tests = f2p + p2p[:20] # guard: cap regression tests for runtime
|
||||
script = f'''set -e
|
||||
cd /testbed
|
||||
git apply --whitespace=fix /work/patch.diff || {{ echo PATCH_FAILED; exit 2; }}
|
||||
FAIL=0
|
||||
while IFS= read -r t; do
|
||||
[ -z "$t" ] && continue
|
||||
if ! (conda run -n testbed python -m pytest -x -q "$t" > /dev/null 2>&1); then
|
||||
echo "TEST_FAILED $t"; FAIL=1
|
||||
fi
|
||||
done <<'EOF'
|
||||
{chr(10).join(tests)}
|
||||
EOF
|
||||
[ "$FAIL" = 0 ] && echo RESOLVED
|
||||
exit $FAIL
|
||||
'''
|
||||
return {'patch.diff': pred or '', 'run.sh': script}
|
||||
|
||||
|
||||
@register_eval('swe_bench_verified')
|
||||
def swe_bench_verified():
|
||||
return EvalRecipe(
|
||||
name='swe_bench_verified',
|
||||
extract='identity', # a patch, not an answer
|
||||
scorers={'resolved': 'env_reward'},
|
||||
description='SWE-bench Verified; docker env, FAIL_TO_PASS/PASS_TO_PASS.',
|
||||
scorers={'resolved': {'name': 'execution', 'harness': _swe_harness,
|
||||
'entry': 'run.sh', 'sandbox': 'docker',
|
||||
'timeout_s': 900}},
|
||||
description='SWE-bench Verified single-turn: model emits a unified diff; '
|
||||
'applied in the official sweb.eval.* image, FAIL_TO_PASS(+P2P) '
|
||||
'must pass. Prefetch: evalharness sandbox prefetch swe_bench_verified',
|
||||
)
|
||||
|
||||
|
||||
|
||||
@ -61,9 +61,9 @@ def evaluate(
|
||||
task_type=sample.task_type,
|
||||
raw_prediction=raw,
|
||||
target=sample.target,
|
||||
group_key=override.get('group_key')
|
||||
or sample.metadata.get('group_key')
|
||||
or (sample.metadata.get('task_id') or sample.metadata.get('id') or ''),
|
||||
group_key=str(override.get('group_key')
|
||||
or sample.metadata.get('group_key')
|
||||
or (sample.metadata.get('task_id') or sample.metadata.get('id') or '')),
|
||||
metadata={k: v for k, v in (sample.metadata or {}).items()
|
||||
if k in ('category', 'subject', 'test_category', 'bin', 'difficulty')},
|
||||
)
|
||||
|
||||
@ -304,7 +304,8 @@ def execution(pred: str, target, sample: Sample, ctx: ScoreContext):
|
||||
sbx = get_sandbox(ctx.params.get('sandbox', 'local'))
|
||||
files = harness(sample, pred or '')
|
||||
result = sbx.exec(files, entry=ctx.params.get('entry', 'main.py'),
|
||||
timeout_s=ctx.params.get('timeout_s', 30))
|
||||
timeout_s=ctx.params.get('timeout_s', 30),
|
||||
image=ctx.params.get('image', ''))
|
||||
ok = result.ok
|
||||
return ({'pass': 1.0} if ok else {'pass': 0.0}), {'pass': {
|
||||
'exit_code': result.exit_code,
|
||||
|
||||
@ -121,8 +121,23 @@ class OpenAICompatible(ModelAdapter):
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
if self.api_key:
|
||||
headers['Authorization'] = f'Bearer {self.api_key}'
|
||||
data = await self._post(f'{self.api_base}/chat/completions', payload, headers)
|
||||
return self._parse(data)
|
||||
retries = self.extra.get('retries', 3)
|
||||
last_exc: Exception = None
|
||||
for attempt in range(retries + 1):
|
||||
try:
|
||||
data = await self._post(f'{self.api_base}/chat/completions', payload, headers)
|
||||
return self._parse(data)
|
||||
except Exception as e: # 5xx/429/timeouts: worth retrying
|
||||
last_exc = e
|
||||
retryable = 'Server error' in str(e) or '504' in str(e) or '502' in str(e) \
|
||||
or '429' in str(e) or 'timeout' in str(e).lower() \
|
||||
or 'TimeoutException' in type(e).__name__
|
||||
if attempt >= retries or not retryable:
|
||||
raise
|
||||
import asyncio
|
||||
|
||||
await asyncio.sleep(min(2 ** attempt * 2, 30))
|
||||
raise last_exc # unreachable
|
||||
|
||||
def _payload(self, messages, tools, kw) -> Dict[str, Any]:
|
||||
msgs = [{'role': m.role, 'content': m.content} for m in messages]
|
||||
|
||||
@ -33,14 +33,38 @@ async def generate_predictions(
|
||||
env_factory=None,
|
||||
system: str = '',
|
||||
max_turns: int = 8,
|
||||
max_input_chars: int = 0,
|
||||
attach_context_keys: tuple = ('passage', 'context'),
|
||||
) -> tuple:
|
||||
"""Fan out model calls; returns (pred-dicts, total_usage).
|
||||
|
||||
Without env_factory: single-turn generation (text or tool-call JSON).
|
||||
With env_factory(sample)->Environment: the agent message pump runs per
|
||||
sample and predictions carry trajectory/env_state/usage.
|
||||
|
||||
max_input_chars: hard cap on the assembled input (anti-OOM for 128k
|
||||
contexts); 0 = no cap. Truncation keeps the head AND the question tail.
|
||||
attach_context_keys: metadata fields (passage/context) prepended to the
|
||||
question at generation time -- the data layer keeps them separate, the
|
||||
runner assembles the actual prompt.
|
||||
"""
|
||||
gen_kwargs = gen_kwargs or {}
|
||||
|
||||
def assemble(sample: Sample) -> str:
|
||||
parts = []
|
||||
for key in attach_context_keys:
|
||||
ctx = (sample.metadata or {}).get(key)
|
||||
if ctx:
|
||||
parts.append(str(ctx))
|
||||
parts.append(sample.input_text)
|
||||
text = '\n\n'.join(parts)
|
||||
if max_input_chars and len(text) > max_input_chars:
|
||||
keep = max_input_chars // 2
|
||||
head = text[:keep]
|
||||
tail = text[-keep:]
|
||||
text = f'{head}\n\n...[truncated {len(text) - 2 * keep} chars]...\n\n{tail}'
|
||||
return text
|
||||
|
||||
sem = asyncio.Semaphore(concurrency)
|
||||
total_usage = Usage()
|
||||
done_count = 0
|
||||
@ -64,8 +88,8 @@ async def generate_predictions(
|
||||
_progress(progress, done_count, len(samples), t0, total_usage)
|
||||
return pred
|
||||
|
||||
messages = ([ChatMessage(role='user', content=sample.input)] if isinstance(sample.input, str)
|
||||
else list(sample.input))
|
||||
messages = ([ChatMessage(role='user', content=assemble(sample))]
|
||||
if isinstance(sample.input, str) else list(sample.input))
|
||||
tools = None
|
||||
if sample.tools:
|
||||
tools = [{'name': t.name, 'description': t.description or '',
|
||||
@ -115,6 +139,7 @@ async def run_eval(
|
||||
env: str = '',
|
||||
system: str = '',
|
||||
max_turns: int = 8,
|
||||
max_input_chars: int = 0,
|
||||
) -> EvalReport:
|
||||
"""Generate + score in one call. Model spec examples:
|
||||
'mock', 'mock:boxed', 'openai/http://gpu03:8000/v1?qwen3-8b', 'deploy:vllm/qwen3-8b'.
|
||||
@ -154,7 +179,7 @@ async def run_eval(
|
||||
preds, _usages, usage = await generate_predictions(
|
||||
adapter, samples, concurrency, progress=progress,
|
||||
gen_kwargs=gen_kwargs, env_factory=env_factory,
|
||||
system=system, max_turns=max_turns)
|
||||
system=system, max_turns=max_turns, max_input_chars=max_input_chars)
|
||||
finally:
|
||||
await adapter.close()
|
||||
if judge is None and judge_spec:
|
||||
@ -190,15 +215,28 @@ def _make_adapter(spec: str) -> ModelAdapter:
|
||||
|
||||
|
||||
def _judge_callable(judge_adapter: ModelAdapter):
|
||||
async def ask(messages) -> str:
|
||||
out = await judge_adapter.generate([ChatMessage(role='user', content=str(m)) for m in messages]
|
||||
if isinstance(messages, list) and messages and isinstance(messages[0], dict)
|
||||
else messages)
|
||||
return out.text
|
||||
"""Sync judge bridge. Works inside a running event loop (evaluate() may be
|
||||
called from async run_eval): the coroutine runs on a private loop in a
|
||||
worker thread."""
|
||||
|
||||
import asyncio
|
||||
def ask(messages) -> str:
|
||||
import asyncio
|
||||
|
||||
def sync_ask(messages):
|
||||
return asyncio.run(ask(messages))
|
||||
if isinstance(messages, list) and messages and isinstance(messages[0], dict):
|
||||
messages = [ChatMessage(role=m.get('role', 'user'), content=m.get('content', ''))
|
||||
for m in messages]
|
||||
|
||||
return sync_ask
|
||||
async def go():
|
||||
out = await judge_adapter.generate(messages)
|
||||
return out.text
|
||||
|
||||
try:
|
||||
asyncio.get_running_loop()
|
||||
except RuntimeError:
|
||||
return asyncio.run(go())
|
||||
import concurrent.futures
|
||||
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
||||
return pool.submit(asyncio.run, go()).result()
|
||||
|
||||
return ask
|
||||
|
||||
@ -23,9 +23,11 @@ from .base import (
|
||||
)
|
||||
from .docker import DockerSandbox, docker_available, docker_serve, serve_env
|
||||
from .local import LocalSandbox
|
||||
from .prefetch import images_for_dataset, prefetch_images
|
||||
|
||||
__all__ = [
|
||||
'Sandbox', 'DockerSandbox', 'LocalSandbox', 'ExecResult', 'EnvHandle',
|
||||
'SANDBOX_REGISTRY', 'register_sandbox', 'get_sandbox', 'acquire', 'stop_all',
|
||||
'docker_serve', 'serve_env', 'docker_available',
|
||||
'prefetch_images', 'images_for_dataset',
|
||||
]
|
||||
|
||||
@ -61,7 +61,9 @@ class DockerSandbox(Sandbox):
|
||||
out_host = Path(hpath).expanduser()
|
||||
out_host.mkdir(parents=True, exist_ok=True)
|
||||
cmd += ['-v', f'{out_host}:{cpath}:rw']
|
||||
cmd += [img, 'python', f'/work/{entry}']
|
||||
runner = ['python', f'/work/{entry}'] if entry.endswith('.py') \
|
||||
else ['sh', f'/work/{entry}']
|
||||
cmd += [img, *runner]
|
||||
t0 = time.time()
|
||||
try:
|
||||
proc = _run(cmd, timeout=timeout_s + 30)
|
||||
|
||||
78
evalharness/sandbox/prefetch.py
Normal file
78
evalharness/sandbox/prefetch.py
Normal file
@ -0,0 +1,78 @@
|
||||
"""Image prefetch: parallel docker pull for execution environments.
|
||||
|
||||
SWE-bench Verified declares ~500 per-instance images (sweb.eval.x86_64.*).
|
||||
Pulling them lazily during an eval run would stall it serially; prefetch
|
||||
pulls them ahead of time with bounded concurrency and progress reporting.
|
||||
|
||||
from evalharness.sandbox.prefetch import prefetch_images
|
||||
done = prefetch_images(['sweb.eval.x86_64.django__django-12345', ...], workers=8)
|
||||
# CLI: evalharness sandbox prefetch swe_bench_verified --workers 8 --limit 50
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Iterable, List
|
||||
|
||||
from ..data.dataset import Dataset
|
||||
|
||||
|
||||
def images_for_dataset(ds: Dataset, limit: int = 0) -> List[str]:
|
||||
"""Collect distinct sandbox images declared by a dataset's samples."""
|
||||
seen: List[str] = []
|
||||
add = seen.append
|
||||
for i, s in enumerate(ds):
|
||||
if limit and i >= limit:
|
||||
break
|
||||
if s.sandbox and s.sandbox.image and s.sandbox.image not in seen:
|
||||
add(s.sandbox.image)
|
||||
return seen
|
||||
|
||||
|
||||
def local_images() -> set:
|
||||
out = subprocess.run(['docker', 'images', '--format', '{{.Repository}}:{{.Tag}}'],
|
||||
capture_output=True, text=True)
|
||||
return {line for line in out.stdout.splitlines() if line}
|
||||
|
||||
|
||||
def prefetch_images(images: Iterable[str], workers: int = 8) -> List[str]:
|
||||
"""Pull images with bounded concurrency; skip ones already local.
|
||||
|
||||
Returns the list newly pulled. Failures are reported and skipped (one
|
||||
missing instance must not block the rest).
|
||||
"""
|
||||
have = local_images()
|
||||
todo = [img for img in images if img not in have]
|
||||
if not todo:
|
||||
print(f'prefetch: all {len(images)} images already local')
|
||||
return []
|
||||
print(f'prefetch: {len(todo)} to pull ({len(images) - len(todo)} already local), '
|
||||
f'workers={workers}')
|
||||
pulled: List[str] = []
|
||||
failed: List[str] = []
|
||||
t0 = time.time()
|
||||
done = 0
|
||||
with ThreadPoolExecutor(max_workers=workers) as pool:
|
||||
futs = {pool.submit(_pull_one, img): img for img in todo}
|
||||
for fut in as_completed(futs):
|
||||
img = futs[fut]
|
||||
done += 1
|
||||
try:
|
||||
fut.result()
|
||||
pulled.append(img)
|
||||
except Exception as e:
|
||||
failed.append(img)
|
||||
print(f' FAIL {img}: {str(e)[:120]}', flush=True)
|
||||
if done % 10 == 0 or done == len(todo):
|
||||
rate = done / max(time.time() - t0, 1e-6)
|
||||
print(f' [{done}/{len(todo)}] {rate:.2f} imgs/s, '
|
||||
f'{len(failed)} failed', flush=True)
|
||||
print(f'prefetch done: {len(pulled)} pulled, {len(failed)} failed '
|
||||
f'in {time.time() - t0:.0f}s')
|
||||
return pulled
|
||||
|
||||
|
||||
def _pull_one(image: str) -> None:
|
||||
r = subprocess.run(['docker', 'pull', image], capture_output=True, text=True, timeout=3600)
|
||||
if r.returncode != 0:
|
||||
raise RuntimeError(r.stderr.strip()[:200])
|
||||
157
evalharness/viz/renderers/html.py
Normal file
157
evalharness/viz/renderers/html.py
Normal file
@ -0,0 +1,157 @@
|
||||
"""HTML renderer: a self-contained dashboard page (no server, no deps).
|
||||
|
||||
Generates one index.html with sortable metric tables, per-benchmark bars,
|
||||
per-sample drill-down and error browser. Consumed by CLI --out-dir.
|
||||
"""
|
||||
|
||||
import html
|
||||
import json
|
||||
from typing import Dict, List, Union
|
||||
|
||||
from ...eval.record import EvalReport
|
||||
from .. import register_renderer
|
||||
|
||||
|
||||
@register_renderer('html')
|
||||
def html_dashboard(target: Union[EvalReport, List[EvalReport]], opts: Dict) -> str:
|
||||
reports = target if isinstance(target, list) else [target]
|
||||
reports = [r for r in reports if isinstance(r, EvalReport)]
|
||||
cards = []
|
||||
for rep in reports:
|
||||
cards.append(_card(rep))
|
||||
rows_json = json.dumps([_row(r) for r in reports], ensure_ascii=False)
|
||||
return _PAGE.replace('__ROWS__', rows_json).replace('__CARDS__', '\n'.join(cards))
|
||||
|
||||
|
||||
def _row(rep: EvalReport) -> Dict:
|
||||
primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '')
|
||||
return {
|
||||
'dataset': rep.dataset, 'recipe': rep.recipe, 'model': rep.model,
|
||||
'n': rep.num_samples, 'primary': primary,
|
||||
'value': rep.metrics.get(primary, 0.0),
|
||||
'metrics': {k: v for k, v in rep.metrics.items()},
|
||||
'groups': {k: v for k, v in rep.metric_groups.items() if isinstance(v, dict)},
|
||||
'extract_fail': rep.num_failed_extractions,
|
||||
'samples': [
|
||||
{'id': s.sample_id, 'ok': s.extraction_ok, 'extracted': s.extracted_prediction[:200],
|
||||
'raw': s.raw_prediction[:300], 'target': str(s.target)[:120],
|
||||
'scores': s.scores, 'error': s.error[:200]}
|
||||
for s in rep.samples[:200]
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _card(rep: EvalReport) -> str:
|
||||
primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '')
|
||||
value = rep.metrics.get(primary, 0.0)
|
||||
pct = f'{value * 100:.1f}%'
|
||||
warn = (f'<span class="warn">extraction failures: {rep.num_failed_extractions}'
|
||||
f'/{rep.num_samples}</span>') if rep.num_failed_extractions else ''
|
||||
return (f'<div class="card"><div class="card-title">{html.escape(rep.dataset)}</div>'
|
||||
f'<div class="big">{pct}</div><div class="sub">{html.escape(primary)} · n={rep.num_samples}'
|
||||
f' · {html.escape(rep.model or "?")}</div>{warn}</div>')
|
||||
|
||||
|
||||
_PAGE = """<!DOCTYPE html>
|
||||
<html lang="zh">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<title>EvalHarness Dashboard</title>
|
||||
<style>
|
||||
:root { --bg:#0f1115; --panel:#171a21; --line:#2a2f3a; --fg:#e6e9ef; --dim:#8b93a3;
|
||||
--acc:#4f8cff; --ok:#3fb950; --bad:#f85149; --warn:#d29922; }
|
||||
* { box-sizing:border-box; }
|
||||
body { margin:0; background:var(--bg); color:var(--fg);
|
||||
font:14px/1.5 -apple-system,'Segoe UI',Roboto,'PingFang SC',sans-serif; }
|
||||
header { padding:18px 24px; border-bottom:1px solid var(--line); }
|
||||
h1 { margin:0; font-size:18px; } h1 span { color:var(--dim); font-weight:400; font-size:13px; }
|
||||
.wrap { padding:20px 24px; }
|
||||
.cards { display:flex; flex-wrap:wrap; gap:12px; margin-bottom:22px; }
|
||||
.card { background:var(--panel); border:1px solid var(--line); border-radius:10px;
|
||||
padding:14px 16px; min-width:190px; }
|
||||
.card-title { color:var(--dim); font-size:12px; text-transform:uppercase; letter-spacing:.05em; }
|
||||
.big { font-size:28px; font-weight:700; margin:4px 0; color:var(--acc); }
|
||||
.sub { color:var(--dim); font-size:12px; }
|
||||
.warn { color:var(--warn); font-size:12px; }
|
||||
table { border-collapse:collapse; width:100%; background:var(--panel);
|
||||
border:1px solid var(--line); border-radius:10px; overflow:hidden; }
|
||||
th, td { text-align:left; padding:8px 12px; border-bottom:1px solid var(--line); }
|
||||
th { color:var(--dim); font-size:12px; cursor:pointer; user-select:none;
|
||||
text-transform:uppercase; letter-spacing:.04em; background:#1b1f28; }
|
||||
th:hover { color:var(--fg); }
|
||||
tr:hover td { background:#1c2029; }
|
||||
.bar { display:inline-block; height:8px; border-radius:4px; background:var(--acc);
|
||||
vertical-align:middle; margin-right:8px; }
|
||||
.num { font-variant-numeric:tabular-nums; }
|
||||
.tag { display:inline-block; padding:1px 8px; border-radius:10px; font-size:11px;
|
||||
background:#222836; color:var(--dim); margin:1px; }
|
||||
details { margin-top:14px; } summary { cursor:pointer; color:var(--dim); }
|
||||
.sample { border:1px solid var(--line); border-radius:8px; padding:10px 12px;
|
||||
margin:8px 0; background:var(--panel); }
|
||||
pre { white-space:pre-wrap; word-break:break-all; color:#a8b3c5; font-size:12px;
|
||||
background:#10131a; padding:8px; border-radius:6px; margin:6px 0; }
|
||||
.ok { color:var(--ok); } .bad { color:var(--bad); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header><h1>EvalHarness Dashboard <span>generated by evalharness viz</span></h1></header>
|
||||
<div class="wrap">
|
||||
<div class="cards">__CARDS__</div>
|
||||
<table id="tbl"><thead><tr>
|
||||
<th data-k="dataset">dataset</th><th data-k="model">model</th>
|
||||
<th data-k="n">n</th><th data-k="primary">metric</th>
|
||||
<th data-k="value">value</th><th>share</th><th data-k="extract_fail">extract✗</th>
|
||||
</tr></thead><tbody id="tb"></tbody></table>
|
||||
<details id="drill"><summary>click a row for per-sample details</summary>
|
||||
<div id="drill-body"></div></details>
|
||||
</div>
|
||||
<script>
|
||||
const ROWS = __ROWS__;
|
||||
const tb = document.getElementById('tb');
|
||||
let sortKey = 'value', sortAsc = false;
|
||||
function fmt(v) { return (v === null || v === undefined) ? '—' : (v*100).toFixed(1) + '%'; }
|
||||
function render() {
|
||||
const rows = [...ROWS].sort((a, b) => {
|
||||
const av = a[sortKey], bv = b[sortKey];
|
||||
const c = (typeof av === 'number' && typeof bv === 'number') ? av - bv
|
||||
: String(av ?? '').localeCompare(String(bv ?? ''));
|
||||
return sortAsc ? c : -c;
|
||||
});
|
||||
tb.innerHTML = rows.map((r, i) => `
|
||||
<tr data-i="${i}">
|
||||
<td><b>${r.dataset}</b> <span class="tag">${r.recipe}</span></td>
|
||||
<td>${r.model || '?'}</td><td class="num">${r.n}</td>
|
||||
<td>${r.primary}</td>
|
||||
<td class="num"><b>${fmt(r.value)}</b></td>
|
||||
<td><span class="bar" style="width:${Math.round(r.value*120)}px"></span></td>
|
||||
<td class="num">${r.extract_fail || 0}</td>
|
||||
</tr>`).join('');
|
||||
tb.querySelectorAll('tr').forEach(tr => tr.onclick = () => drill(rows[+tr.dataset.i]));
|
||||
}
|
||||
function drill(r) {
|
||||
const el = document.getElementById('drill-body');
|
||||
el.innerHTML = `<h3>${r.dataset} · ${r.model}</h3>`
|
||||
+ Object.entries(r.metrics).map(([k, v]) =>
|
||||
`<span class="tag">${k}: ${typeof v === 'number' ? fmt(v) : v}</span>`).join(' ')
|
||||
+ Object.entries(r.groups || {}).map(([g, sub]) =>
|
||||
`<details><summary>group: ${g}</summary><div>`
|
||||
+ Object.entries(sub).map(([k, v]) =>
|
||||
`<span class="tag">${k}: ${typeof v === 'number' ? fmt(v) : v}</span>`).join(' ')
|
||||
+ '</div></details>').join('')
|
||||
+ r.samples.map(s => `
|
||||
<div class="sample"><b>#${s.id}</b> ${s.ok ? '<span class="ok">extracted</span>' : '<span class="bad">extract✗</span>'}
|
||||
<span class="tag">${JSON.stringify(s.scores)}</span>
|
||||
${s.error ? `<div class="bad">${s.error}</div>` : ''}
|
||||
<div>target: <b>${s.target}</b></div>
|
||||
<div>extracted: <b>${s.extracted || '—'}</b></div>
|
||||
<pre>${s.raw}</pre></div>`).join('');
|
||||
document.getElementById('drill').open = true;
|
||||
}
|
||||
document.querySelectorAll('th[data-k]').forEach(th => th.onclick = () => {
|
||||
const k = th.dataset.k;
|
||||
sortAsc = (k === sortKey) ? !sortAsc : false; sortKey = k; render();
|
||||
});
|
||||
render();
|
||||
</script>
|
||||
</body></html>
|
||||
"""
|
||||
Loading…
x
Reference in New Issue
Block a user