Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Output dir restructure (evalscope/inspect-style): top-level summary.{xlsx,md,csv} + one directory per benchmark (report.json + detail.md); drops the opaque viz/ and reports/ layers

Browse Source 
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-10 10:31:38 +00:00
parent bd35b5b724
commit f9cb34fc17
1 changed files with 15 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
evalharness/cli.py
View File

@ -445,8 +445,6 @@ def _cmd_eval_run(args) -> int:
from pathlib import Path from pathlib import Path
Path(out_dir).mkdir(parents=True, exist_ok=True) Path(out_dir).mkdir(parents=True, exist_ok=True)
(Path(out_dir) / 'viz').mkdir(exist_ok=True)
(Path(out_dir) / 'reports').mkdir(exist_ok=True)
rows = [] rows = []
all_reports = [] all_reports = []
@ -463,8 +461,6 @@ def _cmd_eval_run(args) -> int:
from pathlib import Path from pathlib import Path
Path(out_dir).mkdir(parents=True, exist_ok=True) Path(out_dir).mkdir(parents=True, exist_ok=True)
(Path(out_dir) / 'viz').mkdir(exist_ok=True)
(Path(out_dir) / 'reports').mkdir(exist_ok=True)
console = _rich_console() console = _rich_console()
_print_run_plan(console, args, model_spec) _print_run_plan(console, args, model_spec)
for i, name in enumerate(args.datasets): for i, name in enumerate(args.datasets):
@ -523,9 +519,13 @@ def _cmd_eval_run(args) -> int:
report.save(args.out) report.save(args.out)
if out_dir: if out_dir:
_print_phase(console, i + 1, total_runs, name, _print_phase(console, i + 1, total_runs, name,
'writing report and visualization files') 'writing results')
report.save(f'{out_dir}/reports/{name}.report.json') from pathlib import Path as _P
with open(f'{out_dir}/viz/{name}.md', 'w', encoding='utf-8') as f:
bench_dir = _P(out_dir) / name
bench_dir.mkdir(parents=True, exist_ok=True)
report.save(str(bench_dir / 'report.json'))
with open(bench_dir / 'detail.md', 'w', encoding='utf-8') as f:
f.write(render(report, style='md')) f.write(render(report, style='md'))
if args.verbose: if args.verbose:
if not _print_result_panel(console, report, _time.time() - t0): if not _print_result_panel(console, report, _time.time() - t0):
@ -583,7 +583,7 @@ def _cmd_eval_run(args) -> int:
from evalharness.viz import render as _render from evalharness.viz import render as _render
xb = _render(all_reports, style='excel', xb = _render(all_reports, style='excel',
out=f'{out_dir}/viz/results.xlsx') out=f'{out_dir}/summary.xlsx')
print(f'excel -> {xb}', flush=True) print(f'excel -> {xb}', flush=True)
except Exception as e: except Exception as e:
print(f'excel export skipped: {type(e).__name__}: {str(e)[:80]}', print(f'excel export skipped: {type(e).__name__}: {str(e)[:80]}',
@ -593,7 +593,7 @@ def _cmd_eval_run(args) -> int:
if out_dir: if out_dir:
import csv as _csv import csv as _csv
with open(f'{out_dir}/viz/summary.csv', 'w', newline='', encoding='utf-8') as f: with open(f'{out_dir}/summary.csv', 'w', newline='', encoding='utf-8') as f:
w = _csv.writer(f) w = _csv.writer(f)
w.writerow(['benchmark', 'score', 'metric', 'num_samples', w.writerow(['benchmark', 'score', 'metric', 'num_samples',
'time_h', 'time_s', 'extract_fail', 'time_h', 'time_s', 'extract_fail',
@ -618,7 +618,7 @@ def _cmd_eval_run(args) -> int:
'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s', 'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s',
'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] + 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] +
[cats]) [cats])
with open(f'{out_dir}/viz/summary.md', 'w', encoding='utf-8') as f: with open(f'{out_dir}/summary.md', 'w', encoding='utf-8') as f:
import time as _tt import time as _tt
model_names = {r.get('model', '') for r in rows if r.get('model')} model_names = {r.get('model', '') for r in rows if r.get('model')}
head = f"# eval run summary\n\n- model: {', '.join(model_names) or '?'}\n" head = f"# eval run summary\n\n- model: {', '.join(model_names) or '?'}\n"
@ -648,13 +648,13 @@ def _cmd_eval_run(args) -> int:
console.print( console.print(
f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n' f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 {ap}\n' f' 结果 {ap}\n'
f' ├─ reports/<bench>.report.json\n' f' ├─ summary.xlsx (excel 打开总表)\n'
f' ├─ viz/results.xlsx (excel 打开)\n' f' ├─ summary.md / summary.csv\n'
f' └─ viz/summary.md (+ summary.csv)') f' └─ <bench>/report.json + detail.md (每 benchmark 一个目录)')
else: else:
print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n' print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n'
f' 结果 {ap}\n ├─ reports/<bench>.report.json\n' f' 结果 {ap}\n'
f' ├─ viz/results.xlsx\n └─ viz/summary.md') f' ├─ summary.xlsx\n └─ <bench>/report.json + detail.md')
elif rows and rows[0]['ok'] and args.out: elif rows and rows[0]['ok'] and args.out:
_notice('运行结束 · 结果已保存', args.out) _notice('运行结束 · 结果已保存', args.out)
print(f'{ok_n}/{len(rows)} benchmarks ok') print(f'{ok_n}/{len(rows)} benchmarks ok')