Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Uniform [i/N] prefix on every narration line (status_callback restores the tag; checkpoint-restored detail routes through status_callback; probe line remains pre-callback)

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 11:15:04 +00:00
parent 392cc8daca
commit 5734f7fbe6
2 changed files with 10 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
evalharness/cli.py
View File

@ -528,9 +528,10 @@ def _cmd_eval_run(args) -> int:
def status_callback(msg, _idx=i + 1, _name=name, def status_callback(msg, _idx=i + 1, _name=name,
_reporter=progress_reporter, _reporter=progress_reporter,
_console=console): _console=console):
_tag = f'[{_idx}/{total_runs}] ' if total_runs > 1 else ''
if _reporter is not None: if _reporter is not None:
_col = _phase_color(msg) _col = _phase_color(msg)
_reporter.log(f'[{_col}]{_name}: {msg}[/{_col}]') _reporter.log(f'[{_col}]{_tag}{_name}: {msg}[/{_col}]')
if 'scoring' in msg: if 'scoring' in msg:
_reporter.set_phase('scoring') _reporter.set_phase('scoring')
elif 'generating model responses' in msg: elif 'generating model responses' in msg:

12
evalharness/model/runner.py
View File

@ -399,8 +399,12 @@ async def generate_predictions(
else: else:
pending.append((i, s)) pending.append((i, s))
if ckpt_store is not None and restored: if ckpt_store is not None and restored:
print(f'checkpoint: restored {len(restored)} predictions ' line = (f'checkpoint restored {len(restored)} predictions '
f'({len(pending)} to generate) -> {ckpt_store.path}', flush=True) f'({len(pending)} to generate) -> {ckpt_store.path}')
if status_callback:
status_callback(line)
else:
print(f'checkpoint: {line}', flush=True)
if status_callback: if status_callback:
status_callback(f'checkpoint restored: {len(restored)} ready, {len(pending)} pending') status_callback(f'checkpoint restored: {len(restored)} ready, {len(pending)} pending')
@ -544,8 +548,8 @@ async def _probe_model(adapter, model_spec: str) -> None:
' and --model matches the served model name. Verify manually:\n' ' and --model matches the served model name. Verify manually:\n'
f' {curl}') f' {curl}')
if members and not bad: if members and not bad:
print(f'· model endpoint ok ({len(members)} instance(s), ' msg = f'model endpoint ok ({len(members)} instance(s), {members[0].api_base})'
f'{members[0].api_base})', flush=True) print(f'· {msg}', flush=True) # probe runs before status_callback exists
_PROBED_SPECS.add(model_spec) _PROBED_SPECS.add(model_spec)
async def run_eval( async def run_eval(