Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Auto-export consolidated excel workbook (results.xlsx: Summary/Perf/Categories/Samples) on every --out-dir run; per-bench artifact .md instead of .txt; xlsxwriter joins core deps

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:29:13 +00:00
parent 8cd45e4d16
commit bd35b5b724
2 changed files with 18 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
evalharness/cli.py
View File

@ -449,6 +449,7 @@ def _cmd_eval_run(args) -> int:
(Path(out_dir) / 'reports').mkdir(exist_ok=True)
rows = []
all_reports = []
run_started = _time.time()
total_runs = len(args.datasets)
model_spec = _compose_model_spec(args)
@ -524,14 +525,15 @@ def _cmd_eval_run(args) -> int:
_print_phase(console, i + 1, total_runs, name,
'writing report and visualization files')
report.save(f'{out_dir}/reports/{name}.report.json')
with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f:
f.write(render(report, style='text'))
with open(f'{out_dir}/viz/{name}.md', 'w', encoding='utf-8') as f:
f.write(render(report, style='md'))
if args.verbose:
if not _print_result_panel(console, report, _time.time() - t0):
print(render(report, style=args.style))
elif len(args.datasets) == 1:
if console is None:
print(render(report, style=args.style))
all_reports.append(report)
primary = next(iter(report.metrics), '')
secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0)
for s in report.samples)
@ -576,6 +578,16 @@ def _cmd_eval_run(args) -> int:
_print_benchmark_result(console, i + 1, total_runs, name,
'failed', _time.time() - t0)
if all_reports and out_dir:
try:
from evalharness.viz import render as _render
xb = _render(all_reports, style='excel',
out=f'{out_dir}/viz/results.xlsx')
print(f'excel -> {xb}', flush=True)
except Exception as e:
print(f'excel export skipped: {type(e).__name__}: {str(e)[:80]}',
file=sys.stderr)
if rows:
_print_summary_table(console, rows)
if out_dir:
@ -637,10 +649,12 @@ def _cmd_eval_run(args) -> int:
f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 {ap}\n'
f' ├─ reports/<bench>.report.json\n'
f' ├─ viz/results.xlsx (excel 打开)\n'
f' └─ viz/summary.md (+ summary.csv)')
else:
print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 {ap}\n ├─ reports/<bench>.report.json\n └─ viz/summary.md')
f' 结果 {ap}\n ├─ reports/<bench>.report.json\n'
f' ├─ viz/results.xlsx\n └─ viz/summary.md')
elif rows and rows[0]['ok'] and args.out:
_notice('运行结束 · 结果已保存', args.out)
print(f'{ok_n}/{len(rows)} benchmarks ok')

1
pyproject.toml
View File

@ -16,6 +16,7 @@ dependencies = [
"numpy", # official DROP aligner
"scipy",
"rich",
"xlsxwriter", # excel result workbook
"tree_sitter>=0.21", # vendored BFCL official AST checker (python)
"tree-sitter-java>=0.21", # bfcl java categories
"tree-sitter-javascript>=0.21", # bfcl javascript categories