Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Narration density: preparing+checkpoint merged into one line (N samples · checkpoint k/N restored, m to generate); zero-pending runs say 'generation skipped' instead of the misleading generating-0/complete pair; scoring complete carries the primary metric

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 11:26:23 +00:00
parent 3911019315
commit 3a045e0569
1 changed files with 18 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
evalharness/model/runner.py
View File

@ -360,8 +360,6 @@ async def generate_predictions(
return {'raw': text, 'usage': out.usage.model_dump()}
work = _apply_limits(samples, limit, limit_per_task, shuffle=not no_shuffle)
if status_callback:
status_callback(f'preparing {len(work)} evaluation samples')
# checkpointing: restore completed samples, generate only the rest
ckpt_store = None
if checkpoint:
@ -398,15 +396,15 @@ async def generate_predictions(
preds_by_key[k] = restored[k]
else:
pending.append((i, s))
if ckpt_store is not None and restored:
line = (f'checkpoint restored {len(restored)} predictions '
f'({len(pending)} to generate) -> {ckpt_store.path}')
if status_callback:
status_callback(line)
if status_callback:
if restored:
status_callback(f'{len(work)} samples · checkpoint {len(restored)}/{len(work)} '
f'restored, {len(pending)} to generate')
else:
print(f'checkpoint: {line}', flush=True)
if status_callback:
status_callback(f'checkpoint restored: {len(restored)} ready, {len(pending)} pending')
status_callback(f'{len(work)} samples')
elif restored:
print(f'checkpoint: restored {len(restored)} predictions '
f'({len(pending)} to generate) -> {ckpt_store.path}', flush=True)
if progress_reporter is not None:
progress_reporter.reset_samples(len(work), dataset_name, completed=len(restored))
@ -435,14 +433,17 @@ async def generate_predictions(
try:
if status_callback:
status_callback(f'generating model responses: {len(pending)} pending')
if pending:
status_callback(f'generating {len(pending)} responses')
else:
status_callback('generation skipped · checkpoint complete')
fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending))
for i, pred in fresh:
preds_by_key[keys[i]] = pred
preds = [preds_by_key[k] for k in keys]
usages = [p.get('usage', {}) for p in preds]
if status_callback:
status_callback(f'generation complete: {len(preds)} responses')
if status_callback and pending:
status_callback(f'generation complete · {len(preds)} responses')
return preds, usages, total_usage
finally:
# reporter lifecycle belongs to the CALLER (CLI reuses one reporter
@ -730,7 +731,10 @@ async def run_eval(
report.model = model_spec
report.dataset = name
if status_callback:
status_callback('scoring complete')
_m = next(((k, v) for k, v in report.metrics.items()
if k != 'extraction_failure_rate'), None)
status_callback(f'scoring complete · {_m[0]} {_m[1] * 100:.1f}%'
if _m else 'scoring complete')
# performance profile: pool success rate + latency/ttft percentiles
try:
from .aggregator import get_aggregator