Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Narration density: preparing+checkpoint merged into one line (N samples · checkpoint k/N restored, m to generate); zero-pending runs say 'generation skipped' instead of the misleading generating-0/complete pair; scoring complete carries the primary metric

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 11:26:23 +00:00
parent 3911019315
commit 3a045e0569
1 changed files with 18 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
evalharness/model/runner.py
View File

@ -360,8 +360,6 @@ async def generate_predictions(
return {'raw': text, 'usage': out.usage.model_dump()} return {'raw': text, 'usage': out.usage.model_dump()}
work = _apply_limits(samples, limit, limit_per_task, shuffle=not no_shuffle) work = _apply_limits(samples, limit, limit_per_task, shuffle=not no_shuffle)
if status_callback:
status_callback(f'preparing {len(work)} evaluation samples')
# checkpointing: restore completed samples, generate only the rest # checkpointing: restore completed samples, generate only the rest
ckpt_store = None ckpt_store = None
if checkpoint: if checkpoint:
@ -398,15 +396,15 @@ async def generate_predictions(
preds_by_key[k] = restored[k] preds_by_key[k] = restored[k]
else: else:
pending.append((i, s)) pending.append((i, s))
if ckpt_store is not None and restored: if status_callback:
line = (f'checkpoint restored {len(restored)} predictions ' if restored:
f'({len(pending)} to generate) -> {ckpt_store.path}') status_callback(f'{len(work)} samples · checkpoint {len(restored)}/{len(work)} '
if status_callback: f'restored, {len(pending)} to generate')
status_callback(line)
else: else:
print(f'checkpoint: {line}', flush=True) status_callback(f'{len(work)} samples')
if status_callback: elif restored:
status_callback(f'checkpoint restored: {len(restored)} ready, {len(pending)} pending') print(f'checkpoint: restored {len(restored)} predictions '
f'({len(pending)} to generate) -> {ckpt_store.path}', flush=True)
if progress_reporter is not None: if progress_reporter is not None:
progress_reporter.reset_samples(len(work), dataset_name, completed=len(restored)) progress_reporter.reset_samples(len(work), dataset_name, completed=len(restored))
@ -435,14 +433,17 @@ async def generate_predictions(
try: try:
if status_callback: if status_callback:
status_callback(f'generating model responses: {len(pending)} pending') if pending:
status_callback(f'generating {len(pending)} responses')
else:
status_callback('generation skipped · checkpoint complete')
fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending)) fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending))
for i, pred in fresh: for i, pred in fresh:
preds_by_key[keys[i]] = pred preds_by_key[keys[i]] = pred
preds = [preds_by_key[k] for k in keys] preds = [preds_by_key[k] for k in keys]
usages = [p.get('usage', {}) for p in preds] usages = [p.get('usage', {}) for p in preds]
if status_callback: if status_callback and pending:
status_callback(f'generation complete: {len(preds)} responses') status_callback(f'generation complete · {len(preds)} responses')
return preds, usages, total_usage return preds, usages, total_usage
finally: finally:
# reporter lifecycle belongs to the CALLER (CLI reuses one reporter # reporter lifecycle belongs to the CALLER (CLI reuses one reporter
@ -730,7 +731,10 @@ async def run_eval(
report.model = model_spec report.model = model_spec
report.dataset = name report.dataset = name
if status_callback: if status_callback:
status_callback('scoring complete') _m = next(((k, v) for k, v in report.metrics.items()
if k != 'extraction_failure_rate'), None)
status_callback(f'scoring complete · {_m[0]} {_m[1] * 100:.1f}%'
if _m else 'scoring complete')
# performance profile: pool success rate + latency/ttft percentiles # performance profile: pool success rate + latency/ttft percentiles
try: try:
from .aggregator import get_aggregator from .aggregator import get_aggregator