Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Excel renderer (xlsxwriter, 4 sheets: Summary dashboard with 21 columns, Perf detail, Categories, Samples drill-down); !perf spec flag; viz --style excel

Browse Source
This commit is contained in:
sora 2026-08-27 02:56:07 +00:00
parent e77ea4ce5c
commit 3c63338ac1
2 changed files with 178 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
evalharness/cli.py
View File

@ -245,6 +245,15 @@ def _f3(v):
def _cmd_viz_show(args) -> int:
from evalharness.viz import render
if args.style == 'excel':
from evalharness.eval.record import EvalReport
reps = [EvalReport.load(p) for p in args.reports]
out = render(reps, style='excel',
**({'n': args.n} if args.n else {}),
**({'out': args.out} if args.out else {}))
print(f'excel -> {out}')
return 0
print(render([*args.reports], style=args.style, **({'n': args.n} if args.n else {})))
return 0
@ -331,7 +340,8 @@ def build_parser() -> argparse.ArgumentParser:
p = zsub.add_parser('show', help='render report file(s)')
p.add_argument('reports', nargs='+')
p.add_argument('--style', default='text',
help='text | md | md_compare (multi) | radar | errors')
help='text | md | md_compare | radar | errors | excel (writes .xlsx)')
p.add_argument('--out', help='excel output path (default ./evalharness_report.xlsx)')
p.add_argument('-n', type=int, help='for errors style: how many samples')
p.set_defaults(func=_cmd_viz_show)

167
evalharness/viz/renderers/excel.py Normal file
View File

@ -0,0 +1,167 @@
"""Excel renderer: multi-sheet workbook from EvalReports (xlsxwriter).
Sheets:
1. Summary -- one row per benchmark: identity + score + quality + perf dashboard
2. Perf -- detailed latency/ttft/tpot/token columns
3. Categories-- per-benchmark category breakdown
4. Samples -- per-sample drill-down (first N)
render([rep1, rep2], style='excel') -> bytes/str path via CLI
evalharness viz show r1.json r2.json --style excel # writes xlsx next to inputs
"""
import json
from typing import List, Union
from ...eval.record import EvalReport
from .. import register_renderer
# dashboard column spec: (header, source-key, formatter)
_SUMMARY_COLS = [
('benchmark', None, None), ('model', None, None), ('metric', None, None),
('score', None, 'pct'), ('num_samples', None, 'int'),
('extract_fail', None, 'int'), ('time_h', None, 'f2'),
('success_rate', 'success_rate', 'pct'), ('latency_mean_s', 'latency_mean_s', 'f3'),
('output_tps', 'output_tps', 'f2'), ('request_qps', 'request_qps', 'f4'),
('input_tokens_mean', 'input_tokens_mean', 'f1'), ('output_tokens_mean', 'output_tokens_mean', 'f1'),
('total_tokens', 'total_tokens', 'int'),
('ttft_mean_s', 'ttft_mean_s', 'f3'), ('ttft_p90_s', 'ttft_p90_s', 'f3'),
('ttft_p99_s', 'ttft_p99_s', 'f3'),
('tpot_mean_s', 'tpot_mean_s', 'f4'), ('tpot_p90_s', 'tpot_p90_s', 'f4'),
('tpot_p99_s', 'tpot_p99_s', 'f4'),
('retry_rate', 'retry_rate', 'pct'),
]
def _row_for(rep: EvalReport) -> dict:
perf = rep.metric_groups.get('perf') or {}
primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '')
secs = sum(float((s.usage or {}).get('latency_s', 0) or 0) for s in rep.samples)
return {
'benchmark': rep.dataset, 'model': rep.model or '?', 'metric': primary,
'score': rep.metrics.get(primary, 0), 'num_samples': rep.num_samples,
'extract_fail': rep.num_failed_extractions,
'time_h': round(secs / 3600, 2),
**{k: v for k, v in perf.items() if v is not None},
}
@register_renderer('excel')
def excel_workbook(target: Union['EvalReport', List['EvalReport']], opts: Dict) -> str:
import xlsxwriter
reports = [r for r in (target if isinstance(target, list) else [target])
if isinstance(r, EvalReport)]
if not reports:
return '(no reports)'
out_path = opts.get('out') or str(opts.get('dir', '.')) + f'/evalharness_report.xlsx'
wb = xlsxwriter.Workbook(out_path)
wb.set_properties({'title': 'EvalHarness Report',
'comments': 'generated by evalharness viz --style excel'})
# formats
f_hdr = wb.add_format({'bold': True, 'bg_color': '#1F2937', 'font_color': 'white',
'border': 1, 'align': 'center', 'valign': 'vcenter'})
f_pct = wb.add_format({'num_format': '0.0%'})
f_int = wb.add_format({'num_format': '#,##0'})
f_f1 = wb.add_format({'num_format': '0.0'})
f_f2 = wb.add_format({'num_format': '0.00'})
f_f3 = wb.add_format({'num_format': '0.000'})
f_f4 = wb.add_format({'num_format': '0.0000'})
fmt_map = {'pct': f_pct, 'int': f_int, 'f1': f_f1, 'f2': f_f2, 'f3': f_f3, 'f4': f_f4}
# ---- sheet 1: Summary ----
ws = wb.add_worksheet('Summary')
ws.freeze_panes(1, 2)
for c, (hdr, _, _) in enumerate(_SUMMARY_COLS):
ws.write(0, c, hdr, f_hdr)
rows = [_row_for(r) for r in reports]
for ri, row in enumerate(rows, start=1):
for c, (hdr, key, fmt) in enumerate(_SUMMARY_COLS):
val = row.get(hdr)
if val is None:
ws.write(ri, c, '')
elif fmt:
ws.write_number(ri, c, float(val), fmt_map[fmt])
else:
ws.write(ri, c, val)
ws.autofilter(0, 0, len(rows), len(_SUMMARY_COLS) - 1)
for c, (hdr, _, _) in enumerate(_SUMMARY_COLS):
ws.set_column(c, c, max(12, min(22, len(hdr) + 4)))
# ---- sheet 2: Perf detail ----
perf_keys = ['n_requests', 'latency_mean_s', 'latency_p50_s', 'latency_p90_s',
'latency_p95_s', 'latency_p99_s', 'ttft_mean_s', 'ttft_p50_s',
'ttft_p90_s', 'ttft_p99_s', 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s',
'itl_mean_s', 'output_tps', 'request_qps', 'input_tokens',
'output_tokens', 'input_tokens_mean', 'output_tokens_mean',
'total_tokens', 'success_rate', 'retry_rate', 'wall_latency_s']
ws2 = wb.add_worksheet('Perf')
ws2.freeze_panes(1, 1)
ws2.write(0, 0, 'benchmark', f_hdr)
for c, k in enumerate(perf_keys, start=1):
ws2.write(0, c, k, f_hdr)
for ri, rep in enumerate(reports, start=1):
ws2.write(ri, 0, rep.dataset)
perf = rep.metric_groups.get('perf') or {}
for c, k in enumerate(perf_keys, start=1):
v = perf.get(k)
if isinstance(v, (int, float)):
ws2.write_number(ri, c, v)
else:
ws2.write(ri, c, '' if v is None else str(v))
ws2.autofilter(0, 0, len(reports), len(perf_keys))
# ---- sheet 3: Categories ----
ws3 = wb.add_worksheet('Categories')
ws3.freeze_panes(1, 1)
ws3.write(0, 0, 'benchmark', f_hdr)
ws3.write(0, 1, 'group', f_hdr)
ws3.write(0, 2, 'subgroup', f_hdr)
ws3.write(0, 3, 'score', f_hdr)
r3 = 1
for rep in reports:
for gname, groups in rep.metric_groups.items():
if gname in ('perf', 'run_info') or gname.startswith('agg_error') \
or not isinstance(groups, dict):
continue
for g, v in groups.items():
if not isinstance(v, (int, float)):
continue
ws3.write(r3, 0, rep.dataset)
ws3.write(r3, 1, gname)
ws3.write(r3, 2, str(g)[:80])
ws3.write_number(r3, 3, v, f_pct)
r3 += 1
ws3.autofilter(0, 0, max(r3 - 1, 1), 3)
# ---- sheet 4: Samples (first 300) ----
ws4 = wb.add_worksheet('Samples')
hdr4 = ['benchmark', 'sample_id', 'correct', 'score', 'latency_s', 'ttft_s',
'output_tokens', 'retries', 'extract_ok', 'extracted', 'target']
for c, h in enumerate(hdr4):
ws4.write(0, c, h, f_hdr)
r4 = 1
for rep in reports:
primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '')
for s in rep.samples[:300]:
u = s.usage or {}
ws4.write(r4, 0, rep.dataset)
ws4.write(r4, 1, s.sample_id if s.sample_id is not None else r4)
ws4.write(r4, 2, 1 if s.scores.get(primary, 0) >= 1 else 0)
ws4.write_number(r4, 3, s.scores.get(primary, 0), f_f3)
ws4.write_number(r4, 4, float(u.get('latency_s', 0) or 0), f_f3)
tt = u.get('ttft_s')
ws4.write(r4, 5, tt if tt is not None else '')
ws4.write_number(r4, 6, int(u.get('output_tokens', 0) or 0), f_int)
ws4.write_number(r4, 7, int(u.get('retries', 0) or 0), f_int)
ws4.write(r4, 8, 1 if s.extraction_ok else 0)
ws4.write(r4, 9, str(s.extracted_prediction)[:120])
ws4.write(r4, 10, str(s.target)[:80])
r4 += 1
ws4.autofilter(0, 0, max(r4 - 1, 1), len(hdr4) - 1)
ws4.set_column(9, 10, 40)
wb.close()
return out_path