Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Auto-save results by default (evalharness-results/<stamp>-<model>/); final notice: run finished + saved location with clickable links

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:05:29 +00:00
parent c1896e0a63
commit ed36fda365
1 changed files with 19 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
evalharness/cli.py
View File

@ -435,6 +435,18 @@ def _cmd_eval_run(args) -> int:
run_started = _time.time()
total_runs = len(args.datasets)
model_spec = _compose_model_spec(args)
if not out_dir and model_spec and not args.out:
# always persist results: default dir = evalharness-results/<stamp>-<model>/
import re as _re
stamp = _time.strftime('%Y%m%d-%H%M%S')
tag = _re.sub(r'[^A-Za-z0-9._-]+', '-', args.model or 'run')[:40].strip('-')
out_dir = f'evalharness-results/{stamp}-{tag}'
from pathlib import Path
Path(out_dir).mkdir(parents=True, exist_ok=True)
(Path(out_dir) / 'viz').mkdir(exist_ok=True)
(Path(out_dir) / 'reports').mkdir(exist_ok=True)
console = _rich_console()
_print_run_plan(console, args, model_spec)
for i, name in enumerate(args.datasets):
@ -599,18 +611,13 @@ def _cmd_eval_run(args) -> int:
else:
print(f'\n{label} -> ' + ' · '.join(os.path.abspath(p) for p in paths))
if len(rows) > 1:
if out_dir:
_notice('结果已生成', f'{out_dir}/reports/<bench>.report.json',
f'{out_dir}/viz/summary.md')
else:
print('\n提示: 加 --out-dir <目录> 可保存全部报告 (reports/*.json + summary.md)')
elif rows and rows[0]['ok']:
saved = args.out or (f'{out_dir}/reports/{rows[0]["name"]}.report.json' if out_dir else '')
if saved:
_notice('结果已生成', saved)
else:
print('\n提示: 加 --out <文件> 或 --out-dir <目录> 可保存报告')
ok_n = sum(1 for r in rows if r['ok'])
if out_dir:
_notice('运行结束 · 结果已保存', f'{out_dir}/reports/<bench>.report.json',
f'{out_dir}/viz/summary.md')
elif rows and rows[0]['ok'] and args.out:
_notice('运行结束 · 结果已保存', args.out)
print(f'\n{ok_n}/{len(rows)} benchmarks ok')
return 0 if all(r['ok'] for r in rows) else 1