Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

All narration lines as full sentences (Checkpoint: 4/4 predictions already generated, 0 samples left to run / Generation skipped: ... / Scoring complete: acc 100.0% / Writing results to ...)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 02:59:22 +00:00
parent 20e9b87248
commit a1fa75bf89
2 changed files with 16 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
evalharness/cli.py
View File

@ -206,14 +206,14 @@ def _narration(msg: str) -> str:
# generation counts, few-shot counts (curated, not rich's blanket highlight) # generation counts, few-shot counts (curated, not rich's blanket highlight)
import re as _re0 import re as _re0
msg = _re0.sub(r'(?<![\w/])(\d+(?:/\d+)?)(?=[\s]|$)', msg = _re0.sub(r'(?<![\w/%.])(\d+(?:/\d+)?)(?=[\s]|$)',
r'[bold]\1[/bold]', msg) r'[bold]\1[/bold]', msg)
icon = '' icon = ''
m = msg.lower() m = msg.lower()
if m.startswith('loading/'): if m.startswith('loading/'):
icon = '⬇ ' icon = '⬇ '
elif 'dataset ready' in m: elif 'dataset ready' in m or m.startswith('dataset ready'):
icon = '📦 ' icon = '📦 '
elif 'few-shot' in m: elif 'few-shot' in m:
icon = '✳ ' icon = '✳ '
@ -235,7 +235,7 @@ def _narration(msg: str) -> str:
# gets bold green -- it is what the eye should find first # gets bold green -- it is what the eye should find first
import re as _re1 import re as _re1
m = _re1.search(r'(· [a-zA-Z_@]+ [0-9.]+%)$', msg) m = _re1.search(r'[:·] ([a-zA-Z_@]+ [0-9.]+%)$', msg)
if m: if m:
return (f'{icon}{msg[:m.start()]}' return (f'{icon}{msg[:m.start()]}'
f'[bold green]{m.group(1)}[/bold green]') f'[bold green]{m.group(1)}[/bold green]')
@ -551,7 +551,7 @@ def _cmd_eval_run(args) -> int:
else: else:
_print_phase(console, _i + 1, total_runs, _n, msg) _print_phase(console, _i + 1, total_runs, _n, msg)
_emit('loading/downloading dataset') _emit('Loading dataset (downloads on first use, cached afterwards)')
ds = get_dataset(name, **overrides) ds = get_dataset(name, **overrides)
if _shared_reporter is not None: if _shared_reporter is not None:
_shared_reporter.pause() # let hub tqdm print cleanly _shared_reporter.pause() # let hub tqdm print cleanly
@ -559,7 +559,7 @@ def _cmd_eval_run(args) -> int:
if _shared_reporter is not None: if _shared_reporter is not None:
_shared_reporter.resume() _shared_reporter.resume()
origin = ds.lineage.get('from', 'unknown') origin = ds.lineage.get('from', 'unknown')
_emit(f'dataset ready · samples={sample_count} · source={origin}') _emit(f'Dataset ready: {sample_count} samples from {origin}')
if model_spec: # generate + score in one go if model_spec: # generate + score in one go
from evalharness.model import run_eval from evalharness.model import run_eval
@ -621,8 +621,8 @@ def _cmd_eval_run(args) -> int:
if args.out: if args.out:
report.save(args.out) report.save(args.out)
if out_dir: if out_dir:
_emit(f'writing results -> {out_dir}/{name}/' if out_dir _emit(f'Writing results to {out_dir}/{name}/' if out_dir
else 'writing results') else 'Writing results')
from pathlib import Path as _P from pathlib import Path as _P
bench_dir = _P(out_dir) / name bench_dir = _P(out_dir) / name

18
evalharness/model/runner.py
View File

@ -398,10 +398,10 @@ async def generate_predictions(
pending.append((i, s)) pending.append((i, s))
if status_callback: if status_callback:
if restored: if restored:
status_callback(f'{len(work)} samples · checkpoint {len(restored)}/{len(work)} ' status_callback(f'Checkpoint: {len(restored)}/{len(work)} predictions already generated, '
f'restored, {len(pending)} to generate') f'{len(pending)} samples left to run')
else: else:
status_callback(f'{len(work)} samples') status_callback(f'{len(work)} samples to evaluate')
elif restored: elif restored:
print(f'checkpoint: restored {len(restored)} predictions ' print(f'checkpoint: restored {len(restored)} predictions '
f'({len(pending)} to generate) -> {ckpt_store.path}', flush=True) f'({len(pending)} to generate) -> {ckpt_store.path}', flush=True)
@ -434,16 +434,16 @@ async def generate_predictions(
try: try:
if status_callback: if status_callback:
if pending: if pending:
status_callback(f'generating {len(pending)} responses') status_callback(f'Generating {len(pending)} model responses')
else: else:
status_callback('generation skipped · checkpoint complete') status_callback('Generation skipped: the checkpoint already covers every sample')
fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending)) fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending))
for i, pred in fresh: for i, pred in fresh:
preds_by_key[keys[i]] = pred preds_by_key[keys[i]] = pred
preds = [preds_by_key[k] for k in keys] preds = [preds_by_key[k] for k in keys]
usages = [p.get('usage', {}) for p in preds] usages = [p.get('usage', {}) for p in preds]
if status_callback and pending: if status_callback and pending:
status_callback(f'generation complete · {len(preds)} responses') status_callback(f'Generation complete: {len(preds)} responses collected')
return preds, usages, total_usage return preds, usages, total_usage
finally: finally:
# reporter lifecycle belongs to the CALLER (CLI reuses one reporter # reporter lifecycle belongs to the CALLER (CLI reuses one reporter
@ -659,7 +659,7 @@ async def run_eval(
few_shot_text = None few_shot_text = None
if few_shot_num: if few_shot_num:
if status_callback: if status_callback:
status_callback(f'loading few-shot examples: {few_shot_num}') status_callback(f'Loading {few_shot_num} few-shot exemplars')
from ..data.registry import get_dataset_provider from ..data.registry import get_dataset_provider
prov = get_dataset_provider(name) prov = get_dataset_provider(name)
@ -732,7 +732,7 @@ async def run_eval(
judge = _judge_callable(judge_adapter) judge = _judge_callable(judge_adapter)
if status_callback: if status_callback:
status_callback('scoring predictions') status_callback('Scoring predictions against the benchmark recipe')
report = evaluate( report = evaluate(
samples, preds, recipe, samples, preds, recipe,
model=model_spec, model=model_spec,
@ -746,7 +746,7 @@ async def run_eval(
if status_callback: if status_callback:
_m = next(((k, v) for k, v in report.metrics.items() _m = next(((k, v) for k, v in report.metrics.items()
if k != 'extraction_failure_rate'), None) if k != 'extraction_failure_rate'), None)
status_callback(f'scoring complete · {_m[0]} {_m[1] * 100:.1f}%' status_callback(f'Scoring complete: {_m[0]} {_m[1] * 100:.1f}%'
if _m else 'scoring complete') if _m else 'scoring complete')
# performance profile: pool success rate + latency/ttft percentiles # performance profile: pool success rate + latency/ttft percentiles
try: try: