diff --git a/evalharness/cli.py b/evalharness/cli.py index 90422c8..d918cb0 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -245,6 +245,15 @@ def _f3(v): def _cmd_viz_show(args) -> int: from evalharness.viz import render + if args.style == 'excel': + from evalharness.eval.record import EvalReport + + reps = [EvalReport.load(p) for p in args.reports] + out = render(reps, style='excel', + **({'n': args.n} if args.n else {}), + **({'out': args.out} if args.out else {})) + print(f'excel -> {out}') + return 0 print(render([*args.reports], style=args.style, **({'n': args.n} if args.n else {}))) return 0 @@ -331,7 +340,8 @@ def build_parser() -> argparse.ArgumentParser: p = zsub.add_parser('show', help='render report file(s)') p.add_argument('reports', nargs='+') p.add_argument('--style', default='text', - help='text | md | md_compare (multi) | radar | errors') + help='text | md | md_compare | radar | errors | excel (writes .xlsx)') + p.add_argument('--out', help='excel output path (default ./evalharness_report.xlsx)') p.add_argument('-n', type=int, help='for errors style: how many samples') p.set_defaults(func=_cmd_viz_show) diff --git a/evalharness/viz/renderers/excel.py b/evalharness/viz/renderers/excel.py new file mode 100644 index 0000000..3ed5175 --- /dev/null +++ b/evalharness/viz/renderers/excel.py @@ -0,0 +1,167 @@ +"""Excel renderer: multi-sheet workbook from EvalReports (xlsxwriter). + +Sheets: + 1. Summary -- one row per benchmark: identity + score + quality + perf dashboard + 2. Perf -- detailed latency/ttft/tpot/token columns + 3. Categories-- per-benchmark category breakdown + 4. Samples -- per-sample drill-down (first N) + + render([rep1, rep2], style='excel') -> bytes/str path via CLI + evalharness viz show r1.json r2.json --style excel # writes xlsx next to inputs +""" + +import json +from typing import List, Union + +from ...eval.record import EvalReport +from .. import register_renderer + +# dashboard column spec: (header, source-key, formatter) +_SUMMARY_COLS = [ + ('benchmark', None, None), ('model', None, None), ('metric', None, None), + ('score', None, 'pct'), ('num_samples', None, 'int'), + ('extract_fail', None, 'int'), ('time_h', None, 'f2'), + ('success_rate', 'success_rate', 'pct'), ('latency_mean_s', 'latency_mean_s', 'f3'), + ('output_tps', 'output_tps', 'f2'), ('request_qps', 'request_qps', 'f4'), + ('input_tokens_mean', 'input_tokens_mean', 'f1'), ('output_tokens_mean', 'output_tokens_mean', 'f1'), + ('total_tokens', 'total_tokens', 'int'), + ('ttft_mean_s', 'ttft_mean_s', 'f3'), ('ttft_p90_s', 'ttft_p90_s', 'f3'), + ('ttft_p99_s', 'ttft_p99_s', 'f3'), + ('tpot_mean_s', 'tpot_mean_s', 'f4'), ('tpot_p90_s', 'tpot_p90_s', 'f4'), + ('tpot_p99_s', 'tpot_p99_s', 'f4'), + ('retry_rate', 'retry_rate', 'pct'), +] + + +def _row_for(rep: EvalReport) -> dict: + perf = rep.metric_groups.get('perf') or {} + primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '') + secs = sum(float((s.usage or {}).get('latency_s', 0) or 0) for s in rep.samples) + return { + 'benchmark': rep.dataset, 'model': rep.model or '?', 'metric': primary, + 'score': rep.metrics.get(primary, 0), 'num_samples': rep.num_samples, + 'extract_fail': rep.num_failed_extractions, + 'time_h': round(secs / 3600, 2), + **{k: v for k, v in perf.items() if v is not None}, + } + + +@register_renderer('excel') +def excel_workbook(target: Union['EvalReport', List['EvalReport']], opts: Dict) -> str: + import xlsxwriter + + reports = [r for r in (target if isinstance(target, list) else [target]) + if isinstance(r, EvalReport)] + if not reports: + return '(no reports)' + out_path = opts.get('out') or str(opts.get('dir', '.')) + f'/evalharness_report.xlsx' + + wb = xlsxwriter.Workbook(out_path) + wb.set_properties({'title': 'EvalHarness Report', + 'comments': 'generated by evalharness viz --style excel'}) + + # formats + f_hdr = wb.add_format({'bold': True, 'bg_color': '#1F2937', 'font_color': 'white', + 'border': 1, 'align': 'center', 'valign': 'vcenter'}) + f_pct = wb.add_format({'num_format': '0.0%'}) + f_int = wb.add_format({'num_format': '#,##0'}) + f_f1 = wb.add_format({'num_format': '0.0'}) + f_f2 = wb.add_format({'num_format': '0.00'}) + f_f3 = wb.add_format({'num_format': '0.000'}) + f_f4 = wb.add_format({'num_format': '0.0000'}) + fmt_map = {'pct': f_pct, 'int': f_int, 'f1': f_f1, 'f2': f_f2, 'f3': f_f3, 'f4': f_f4} + + # ---- sheet 1: Summary ---- + ws = wb.add_worksheet('Summary') + ws.freeze_panes(1, 2) + for c, (hdr, _, _) in enumerate(_SUMMARY_COLS): + ws.write(0, c, hdr, f_hdr) + rows = [_row_for(r) for r in reports] + for ri, row in enumerate(rows, start=1): + for c, (hdr, key, fmt) in enumerate(_SUMMARY_COLS): + val = row.get(hdr) + if val is None: + ws.write(ri, c, '') + elif fmt: + ws.write_number(ri, c, float(val), fmt_map[fmt]) + else: + ws.write(ri, c, val) + ws.autofilter(0, 0, len(rows), len(_SUMMARY_COLS) - 1) + for c, (hdr, _, _) in enumerate(_SUMMARY_COLS): + ws.set_column(c, c, max(12, min(22, len(hdr) + 4))) + + # ---- sheet 2: Perf detail ---- + perf_keys = ['n_requests', 'latency_mean_s', 'latency_p50_s', 'latency_p90_s', + 'latency_p95_s', 'latency_p99_s', 'ttft_mean_s', 'ttft_p50_s', + 'ttft_p90_s', 'ttft_p99_s', 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s', + 'itl_mean_s', 'output_tps', 'request_qps', 'input_tokens', + 'output_tokens', 'input_tokens_mean', 'output_tokens_mean', + 'total_tokens', 'success_rate', 'retry_rate', 'wall_latency_s'] + ws2 = wb.add_worksheet('Perf') + ws2.freeze_panes(1, 1) + ws2.write(0, 0, 'benchmark', f_hdr) + for c, k in enumerate(perf_keys, start=1): + ws2.write(0, c, k, f_hdr) + for ri, rep in enumerate(reports, start=1): + ws2.write(ri, 0, rep.dataset) + perf = rep.metric_groups.get('perf') or {} + for c, k in enumerate(perf_keys, start=1): + v = perf.get(k) + if isinstance(v, (int, float)): + ws2.write_number(ri, c, v) + else: + ws2.write(ri, c, '' if v is None else str(v)) + ws2.autofilter(0, 0, len(reports), len(perf_keys)) + + # ---- sheet 3: Categories ---- + ws3 = wb.add_worksheet('Categories') + ws3.freeze_panes(1, 1) + ws3.write(0, 0, 'benchmark', f_hdr) + ws3.write(0, 1, 'group', f_hdr) + ws3.write(0, 2, 'subgroup', f_hdr) + ws3.write(0, 3, 'score', f_hdr) + r3 = 1 + for rep in reports: + for gname, groups in rep.metric_groups.items(): + if gname in ('perf', 'run_info') or gname.startswith('agg_error') \ + or not isinstance(groups, dict): + continue + for g, v in groups.items(): + if not isinstance(v, (int, float)): + continue + ws3.write(r3, 0, rep.dataset) + ws3.write(r3, 1, gname) + ws3.write(r3, 2, str(g)[:80]) + ws3.write_number(r3, 3, v, f_pct) + r3 += 1 + ws3.autofilter(0, 0, max(r3 - 1, 1), 3) + + # ---- sheet 4: Samples (first 300) ---- + ws4 = wb.add_worksheet('Samples') + hdr4 = ['benchmark', 'sample_id', 'correct', 'score', 'latency_s', 'ttft_s', + 'output_tokens', 'retries', 'extract_ok', 'extracted', 'target'] + for c, h in enumerate(hdr4): + ws4.write(0, c, h, f_hdr) + r4 = 1 + for rep in reports: + primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '') + for s in rep.samples[:300]: + u = s.usage or {} + ws4.write(r4, 0, rep.dataset) + ws4.write(r4, 1, s.sample_id if s.sample_id is not None else r4) + ws4.write(r4, 2, 1 if s.scores.get(primary, 0) >= 1 else 0) + ws4.write_number(r4, 3, s.scores.get(primary, 0), f_f3) + ws4.write_number(r4, 4, float(u.get('latency_s', 0) or 0), f_f3) + tt = u.get('ttft_s') + ws4.write(r4, 5, tt if tt is not None else '') + ws4.write_number(r4, 6, int(u.get('output_tokens', 0) or 0), f_int) + ws4.write_number(r4, 7, int(u.get('retries', 0) or 0), f_int) + ws4.write(r4, 8, 1 if s.extraction_ok else 0) + ws4.write(r4, 9, str(s.extracted_prediction)[:120]) + ws4.write(r4, 10, str(s.target)[:80]) + r4 += 1 + ws4.autofilter(0, 0, max(r4 - 1, 1), len(hdr4) - 1) + ws4.set_column(9, 10, 40) + + wb.close() + return out_path