Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

All narration lines as full sentences (Checkpoint: 4/4 predictions already generated, 0 samples left to run / Generation skipped: ... / Scoring complete: acc 100.0% / Writing results to ...)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-11 02:59:22 +00:00
parent 20e9b87248
commit a1fa75bf89
2 changed files with 16 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
evalharness/cli.py
View File

@ -206,14 +206,14 @@ def _narration(msg: str) -> str:
# generation counts, few-shot counts (curated, not rich's blanket highlight)
import re as _re0
msg = _re0.sub(r'(?<![\w/])(\d+(?:/\d+)?)(?=[\s]|$)',
msg = _re0.sub(r'(?<![\w/%.])(\d+(?:/\d+)?)(?=[\s]|$)',
r'[bold]\1[/bold]', msg)
icon = ''
m = msg.lower()
if m.startswith('loading/'):
icon = '⬇ '
elif 'dataset ready' in m:
elif 'dataset ready' in m or m.startswith('dataset ready'):
icon = '📦 '
elif 'few-shot' in m:
icon = '✳ '
@ -235,7 +235,7 @@ def _narration(msg: str) -> str:
# gets bold green -- it is what the eye should find first
import re as _re1
m = _re1.search(r'(· [a-zA-Z_@]+ [0-9.]+%)$', msg)
m = _re1.search(r'[:·] ([a-zA-Z_@]+ [0-9.]+%)$', msg)
if m:
return (f'{icon}{msg[:m.start()]}'
f'[bold green]{m.group(1)}[/bold green]')
@ -551,7 +551,7 @@ def _cmd_eval_run(args) -> int:
else:
_print_phase(console, _i + 1, total_runs, _n, msg)
_emit('loading/downloading dataset')
_emit('Loading dataset (downloads on first use, cached afterwards)')
ds = get_dataset(name, **overrides)
if _shared_reporter is not None:
_shared_reporter.pause() # let hub tqdm print cleanly
@ -559,7 +559,7 @@ def _cmd_eval_run(args) -> int:
if _shared_reporter is not None:
_shared_reporter.resume()
origin = ds.lineage.get('from', 'unknown')
_emit(f'dataset ready · samples={sample_count} · source={origin}')
_emit(f'Dataset ready: {sample_count} samples from {origin}')
if model_spec: # generate + score in one go
from evalharness.model import run_eval
@ -621,8 +621,8 @@ def _cmd_eval_run(args) -> int:
if args.out:
report.save(args.out)
if out_dir:
_emit(f'writing results -> {out_dir}/{name}/' if out_dir
else 'writing results')
_emit(f'Writing results to {out_dir}/{name}/' if out_dir
else 'Writing results')
from pathlib import Path as _P
bench_dir = _P(out_dir) / name

18
evalharness/model/runner.py
View File

@ -398,10 +398,10 @@ async def generate_predictions(
pending.append((i, s))
if status_callback:
if restored:
status_callback(f'{len(work)} samples · checkpoint {len(restored)}/{len(work)} '
f'restored, {len(pending)} to generate')
status_callback(f'Checkpoint: {len(restored)}/{len(work)} predictions already generated, '
f'{len(pending)} samples left to run')
else:
status_callback(f'{len(work)} samples')
status_callback(f'{len(work)} samples to evaluate')
elif restored:
print(f'checkpoint: restored {len(restored)} predictions '
f'({len(pending)} to generate) -> {ckpt_store.path}', flush=True)
@ -434,16 +434,16 @@ async def generate_predictions(
try:
if status_callback:
if pending:
status_callback(f'generating {len(pending)} responses')
status_callback(f'Generating {len(pending)} model responses')
else:
status_callback('generation skipped · checkpoint complete')
status_callback('Generation skipped: the checkpoint already covers every sample')
fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending))
for i, pred in fresh:
preds_by_key[keys[i]] = pred
preds = [preds_by_key[k] for k in keys]
usages = [p.get('usage', {}) for p in preds]
if status_callback and pending:
status_callback(f'generation complete · {len(preds)} responses')
status_callback(f'Generation complete: {len(preds)} responses collected')
return preds, usages, total_usage
finally:
# reporter lifecycle belongs to the CALLER (CLI reuses one reporter
@ -659,7 +659,7 @@ async def run_eval(
few_shot_text = None
if few_shot_num:
if status_callback:
status_callback(f'loading few-shot examples: {few_shot_num}')
status_callback(f'Loading {few_shot_num} few-shot exemplars')
from ..data.registry import get_dataset_provider
prov = get_dataset_provider(name)
@ -732,7 +732,7 @@ async def run_eval(
judge = _judge_callable(judge_adapter)
if status_callback:
status_callback('scoring predictions')
status_callback('Scoring predictions against the benchmark recipe')
report = evaluate(
samples, preds, recipe,
model=model_spec,
@ -746,7 +746,7 @@ async def run_eval(
if status_callback:
_m = next(((k, v) for k, v in report.metrics.items()
if k != 'extraction_failure_rate'), None)
status_callback(f'scoring complete · {_m[0]} {_m[1] * 100:.1f}%'
status_callback(f'Scoring complete: {_m[0]} {_m[1] * 100:.1f}%'
if _m else 'scoring complete')
# performance profile: pool success rate + latency/ttft percentiles
try: