Auto-export consolidated excel workbook (results.xlsx: Summary/Perf/Categories/Samples) on every --out-dir run; per-bench artifact .md instead of .txt; xlsxwriter joins core deps
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
8cd45e4d16
commit
bd35b5b724
@ -449,6 +449,7 @@ def _cmd_eval_run(args) -> int:
|
|||||||
(Path(out_dir) / 'reports').mkdir(exist_ok=True)
|
(Path(out_dir) / 'reports').mkdir(exist_ok=True)
|
||||||
|
|
||||||
rows = []
|
rows = []
|
||||||
|
all_reports = []
|
||||||
run_started = _time.time()
|
run_started = _time.time()
|
||||||
total_runs = len(args.datasets)
|
total_runs = len(args.datasets)
|
||||||
model_spec = _compose_model_spec(args)
|
model_spec = _compose_model_spec(args)
|
||||||
@ -524,14 +525,15 @@ def _cmd_eval_run(args) -> int:
|
|||||||
_print_phase(console, i + 1, total_runs, name,
|
_print_phase(console, i + 1, total_runs, name,
|
||||||
'writing report and visualization files')
|
'writing report and visualization files')
|
||||||
report.save(f'{out_dir}/reports/{name}.report.json')
|
report.save(f'{out_dir}/reports/{name}.report.json')
|
||||||
with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f:
|
with open(f'{out_dir}/viz/{name}.md', 'w', encoding='utf-8') as f:
|
||||||
f.write(render(report, style='text'))
|
f.write(render(report, style='md'))
|
||||||
if args.verbose:
|
if args.verbose:
|
||||||
if not _print_result_panel(console, report, _time.time() - t0):
|
if not _print_result_panel(console, report, _time.time() - t0):
|
||||||
print(render(report, style=args.style))
|
print(render(report, style=args.style))
|
||||||
elif len(args.datasets) == 1:
|
elif len(args.datasets) == 1:
|
||||||
if console is None:
|
if console is None:
|
||||||
print(render(report, style=args.style))
|
print(render(report, style=args.style))
|
||||||
|
all_reports.append(report)
|
||||||
primary = next(iter(report.metrics), '')
|
primary = next(iter(report.metrics), '')
|
||||||
secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0)
|
secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0)
|
||||||
for s in report.samples)
|
for s in report.samples)
|
||||||
@ -576,6 +578,16 @@ def _cmd_eval_run(args) -> int:
|
|||||||
_print_benchmark_result(console, i + 1, total_runs, name,
|
_print_benchmark_result(console, i + 1, total_runs, name,
|
||||||
'failed', _time.time() - t0)
|
'failed', _time.time() - t0)
|
||||||
|
|
||||||
|
if all_reports and out_dir:
|
||||||
|
try:
|
||||||
|
from evalharness.viz import render as _render
|
||||||
|
|
||||||
|
xb = _render(all_reports, style='excel',
|
||||||
|
out=f'{out_dir}/viz/results.xlsx')
|
||||||
|
print(f'excel -> {xb}', flush=True)
|
||||||
|
except Exception as e:
|
||||||
|
print(f'excel export skipped: {type(e).__name__}: {str(e)[:80]}',
|
||||||
|
file=sys.stderr)
|
||||||
if rows:
|
if rows:
|
||||||
_print_summary_table(console, rows)
|
_print_summary_table(console, rows)
|
||||||
if out_dir:
|
if out_dir:
|
||||||
@ -637,10 +649,12 @@ def _cmd_eval_run(args) -> int:
|
|||||||
f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n'
|
f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n'
|
||||||
f' 结果 {ap}\n'
|
f' 结果 {ap}\n'
|
||||||
f' ├─ reports/<bench>.report.json\n'
|
f' ├─ reports/<bench>.report.json\n'
|
||||||
|
f' ├─ viz/results.xlsx (excel 打开)\n'
|
||||||
f' └─ viz/summary.md (+ summary.csv)')
|
f' └─ viz/summary.md (+ summary.csv)')
|
||||||
else:
|
else:
|
||||||
print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n'
|
print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n'
|
||||||
f' 结果 {ap}\n ├─ reports/<bench>.report.json\n └─ viz/summary.md')
|
f' 结果 {ap}\n ├─ reports/<bench>.report.json\n'
|
||||||
|
f' ├─ viz/results.xlsx\n └─ viz/summary.md')
|
||||||
elif rows and rows[0]['ok'] and args.out:
|
elif rows and rows[0]['ok'] and args.out:
|
||||||
_notice('运行结束 · 结果已保存', args.out)
|
_notice('运行结束 · 结果已保存', args.out)
|
||||||
print(f'{ok_n}/{len(rows)} benchmarks ok')
|
print(f'{ok_n}/{len(rows)} benchmarks ok')
|
||||||
|
|||||||
@ -16,6 +16,7 @@ dependencies = [
|
|||||||
"numpy", # official DROP aligner
|
"numpy", # official DROP aligner
|
||||||
"scipy",
|
"scipy",
|
||||||
"rich",
|
"rich",
|
||||||
|
"xlsxwriter", # excel result workbook
|
||||||
"tree_sitter>=0.21", # vendored BFCL official AST checker (python)
|
"tree_sitter>=0.21", # vendored BFCL official AST checker (python)
|
||||||
"tree-sitter-java>=0.21", # bfcl java categories
|
"tree-sitter-java>=0.21", # bfcl java categories
|
||||||
"tree-sitter-javascript>=0.21", # bfcl javascript categories
|
"tree-sitter-javascript>=0.21", # bfcl javascript categories
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user