diff --git a/evalharness/cli.py b/evalharness/cli.py index 05ff19a..2089c8f 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -206,14 +206,14 @@ def _narration(msg: str) -> str: # generation counts, few-shot counts (curated, not rich's blanket highlight) import re as _re0 - msg = _re0.sub(r'(? str: # gets bold green -- it is what the eye should find first import re as _re1 - m = _re1.search(r'(· [a-zA-Z_@]+ [0-9.]+%)$', msg) + m = _re1.search(r'[:·] ([a-zA-Z_@]+ [0-9.]+%)$', msg) if m: return (f'{icon}{msg[:m.start()]}' f'[bold green]{m.group(1)}[/bold green]') @@ -551,7 +551,7 @@ def _cmd_eval_run(args) -> int: else: _print_phase(console, _i + 1, total_runs, _n, msg) - _emit('loading/downloading dataset') + _emit('Loading dataset (downloads on first use, cached afterwards)') ds = get_dataset(name, **overrides) if _shared_reporter is not None: _shared_reporter.pause() # let hub tqdm print cleanly @@ -559,7 +559,7 @@ def _cmd_eval_run(args) -> int: if _shared_reporter is not None: _shared_reporter.resume() origin = ds.lineage.get('from', 'unknown') - _emit(f'dataset ready · samples={sample_count} · source={origin}') + _emit(f'Dataset ready: {sample_count} samples from {origin}') if model_spec: # generate + score in one go from evalharness.model import run_eval @@ -621,8 +621,8 @@ def _cmd_eval_run(args) -> int: if args.out: report.save(args.out) if out_dir: - _emit(f'writing results -> {out_dir}/{name}/' if out_dir - else 'writing results') + _emit(f'Writing results to {out_dir}/{name}/' if out_dir + else 'Writing results') from pathlib import Path as _P bench_dir = _P(out_dir) / name diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index a769f1c..f24c25b 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -398,10 +398,10 @@ async def generate_predictions( pending.append((i, s)) if status_callback: if restored: - status_callback(f'{len(work)} samples · checkpoint {len(restored)}/{len(work)} ' - f'restored, {len(pending)} to generate') + status_callback(f'Checkpoint: {len(restored)}/{len(work)} predictions already generated, ' + f'{len(pending)} samples left to run') else: - status_callback(f'{len(work)} samples') + status_callback(f'{len(work)} samples to evaluate') elif restored: print(f'checkpoint: restored {len(restored)} predictions ' f'({len(pending)} to generate) -> {ckpt_store.path}', flush=True) @@ -434,16 +434,16 @@ async def generate_predictions( try: if status_callback: if pending: - status_callback(f'generating {len(pending)} responses') + status_callback(f'Generating {len(pending)} model responses') else: - status_callback('generation skipped · checkpoint complete') + status_callback('Generation skipped: the checkpoint already covers every sample') fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending)) for i, pred in fresh: preds_by_key[keys[i]] = pred preds = [preds_by_key[k] for k in keys] usages = [p.get('usage', {}) for p in preds] if status_callback and pending: - status_callback(f'generation complete · {len(preds)} responses') + status_callback(f'Generation complete: {len(preds)} responses collected') return preds, usages, total_usage finally: # reporter lifecycle belongs to the CALLER (CLI reuses one reporter @@ -659,7 +659,7 @@ async def run_eval( few_shot_text = None if few_shot_num: if status_callback: - status_callback(f'loading few-shot examples: {few_shot_num}') + status_callback(f'Loading {few_shot_num} few-shot exemplars') from ..data.registry import get_dataset_provider prov = get_dataset_provider(name) @@ -732,7 +732,7 @@ async def run_eval( judge = _judge_callable(judge_adapter) if status_callback: - status_callback('scoring predictions') + status_callback('Scoring predictions against the benchmark recipe') report = evaluate( samples, preds, recipe, model=model_spec, @@ -746,7 +746,7 @@ async def run_eval( if status_callback: _m = next(((k, v) for k, v in report.metrics.items() if k != 'extraction_failure_rate'), None) - status_callback(f'scoring complete · {_m[0]} {_m[1] * 100:.1f}%' + status_callback(f'Scoring complete: {_m[0]} {_m[1] * 100:.1f}%' if _m else 'scoring complete') # performance profile: pool success rate + latency/ttft percentiles try: