diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index a720fc1..7ce7752 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -360,8 +360,6 @@ async def generate_predictions( return {'raw': text, 'usage': out.usage.model_dump()} work = _apply_limits(samples, limit, limit_per_task, shuffle=not no_shuffle) - if status_callback: - status_callback(f'preparing {len(work)} evaluation samples') # checkpointing: restore completed samples, generate only the rest ckpt_store = None if checkpoint: @@ -398,15 +396,15 @@ async def generate_predictions( preds_by_key[k] = restored[k] else: pending.append((i, s)) - if ckpt_store is not None and restored: - line = (f'checkpoint restored {len(restored)} predictions ' - f'({len(pending)} to generate) -> {ckpt_store.path}') - if status_callback: - status_callback(line) + if status_callback: + if restored: + status_callback(f'{len(work)} samples · checkpoint {len(restored)}/{len(work)} ' + f'restored, {len(pending)} to generate') else: - print(f'checkpoint: {line}', flush=True) - if status_callback: - status_callback(f'checkpoint restored: {len(restored)} ready, {len(pending)} pending') + status_callback(f'{len(work)} samples') + elif restored: + print(f'checkpoint: restored {len(restored)} predictions ' + f'({len(pending)} to generate) -> {ckpt_store.path}', flush=True) if progress_reporter is not None: progress_reporter.reset_samples(len(work), dataset_name, completed=len(restored)) @@ -435,14 +433,17 @@ async def generate_predictions( try: if status_callback: - status_callback(f'generating model responses: {len(pending)} pending') + if pending: + status_callback(f'generating {len(pending)} responses') + else: + status_callback('generation skipped · checkpoint complete') fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending)) for i, pred in fresh: preds_by_key[keys[i]] = pred preds = [preds_by_key[k] for k in keys] usages = [p.get('usage', {}) for p in preds] - if status_callback: - status_callback(f'generation complete: {len(preds)} responses') + if status_callback and pending: + status_callback(f'generation complete · {len(preds)} responses') return preds, usages, total_usage finally: # reporter lifecycle belongs to the CALLER (CLI reuses one reporter @@ -730,7 +731,10 @@ async def run_eval( report.model = model_spec report.dataset = name if status_callback: - status_callback('scoring complete') + _m = next(((k, v) for k, v in report.metrics.items() + if k != 'extraction_failure_rate'), None) + status_callback(f'scoring complete · {_m[0]} {_m[1] * 100:.1f}%' + if _m else 'scoring complete') # performance profile: pool success rate + latency/ttft percentiles try: from .aggregator import get_aggregator