158 lines
7.3 KiB
Python
158 lines
7.3 KiB
Python
"""HTML renderer: a self-contained dashboard page (no server, no deps).
|
|
|
|
Generates one index.html with sortable metric tables, per-benchmark bars,
|
|
per-sample drill-down and error browser. Consumed by CLI --out-dir.
|
|
"""
|
|
|
|
import html
|
|
import json
|
|
from typing import Dict, List, Union
|
|
|
|
from ...eval.record import EvalReport
|
|
from .. import register_renderer
|
|
|
|
|
|
@register_renderer('html')
|
|
def html_dashboard(target: Union[EvalReport, List[EvalReport]], opts: Dict) -> str:
|
|
reports = target if isinstance(target, list) else [target]
|
|
reports = [r for r in reports if isinstance(r, EvalReport)]
|
|
cards = []
|
|
for rep in reports:
|
|
cards.append(_card(rep))
|
|
rows_json = json.dumps([_row(r) for r in reports], ensure_ascii=False)
|
|
return _PAGE.replace('__ROWS__', rows_json).replace('__CARDS__', '\n'.join(cards))
|
|
|
|
|
|
def _row(rep: EvalReport) -> Dict:
|
|
primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '')
|
|
return {
|
|
'dataset': rep.dataset, 'recipe': rep.recipe, 'model': rep.model,
|
|
'n': rep.num_samples, 'primary': primary,
|
|
'value': rep.metrics.get(primary, 0.0),
|
|
'metrics': {k: v for k, v in rep.metrics.items()},
|
|
'groups': {k: v for k, v in rep.metric_groups.items() if isinstance(v, dict)},
|
|
'extract_fail': rep.num_failed_extractions,
|
|
'samples': [
|
|
{'id': s.sample_id, 'ok': s.extraction_ok, 'extracted': s.extracted_prediction[:200],
|
|
'raw': s.raw_prediction[:300], 'target': str(s.target)[:120],
|
|
'scores': s.scores, 'error': s.error[:200]}
|
|
for s in rep.samples[:200]
|
|
],
|
|
}
|
|
|
|
|
|
def _card(rep: EvalReport) -> str:
|
|
primary = next((k for k in rep.metrics if k != 'extraction_failure_rate'), '')
|
|
value = rep.metrics.get(primary, 0.0)
|
|
pct = f'{value * 100:.1f}%'
|
|
warn = (f'<span class="warn">extraction failures: {rep.num_failed_extractions}'
|
|
f'/{rep.num_samples}</span>') if rep.num_failed_extractions else ''
|
|
return (f'<div class="card"><div class="card-title">{html.escape(rep.dataset)}</div>'
|
|
f'<div class="big">{pct}</div><div class="sub">{html.escape(primary)} · n={rep.num_samples}'
|
|
f' · {html.escape(rep.model or "?")}</div>{warn}</div>')
|
|
|
|
|
|
_PAGE = """<!DOCTYPE html>
|
|
<html lang="zh">
|
|
<head>
|
|
<meta charset="utf-8">
|
|
<title>EvalHarness Dashboard</title>
|
|
<style>
|
|
:root { --bg:#0f1115; --panel:#171a21; --line:#2a2f3a; --fg:#e6e9ef; --dim:#8b93a3;
|
|
--acc:#4f8cff; --ok:#3fb950; --bad:#f85149; --warn:#d29922; }
|
|
* { box-sizing:border-box; }
|
|
body { margin:0; background:var(--bg); color:var(--fg);
|
|
font:14px/1.5 -apple-system,'Segoe UI',Roboto,'PingFang SC',sans-serif; }
|
|
header { padding:18px 24px; border-bottom:1px solid var(--line); }
|
|
h1 { margin:0; font-size:18px; } h1 span { color:var(--dim); font-weight:400; font-size:13px; }
|
|
.wrap { padding:20px 24px; }
|
|
.cards { display:flex; flex-wrap:wrap; gap:12px; margin-bottom:22px; }
|
|
.card { background:var(--panel); border:1px solid var(--line); border-radius:10px;
|
|
padding:14px 16px; min-width:190px; }
|
|
.card-title { color:var(--dim); font-size:12px; text-transform:uppercase; letter-spacing:.05em; }
|
|
.big { font-size:28px; font-weight:700; margin:4px 0; color:var(--acc); }
|
|
.sub { color:var(--dim); font-size:12px; }
|
|
.warn { color:var(--warn); font-size:12px; }
|
|
table { border-collapse:collapse; width:100%; background:var(--panel);
|
|
border:1px solid var(--line); border-radius:10px; overflow:hidden; }
|
|
th, td { text-align:left; padding:8px 12px; border-bottom:1px solid var(--line); }
|
|
th { color:var(--dim); font-size:12px; cursor:pointer; user-select:none;
|
|
text-transform:uppercase; letter-spacing:.04em; background:#1b1f28; }
|
|
th:hover { color:var(--fg); }
|
|
tr:hover td { background:#1c2029; }
|
|
.bar { display:inline-block; height:8px; border-radius:4px; background:var(--acc);
|
|
vertical-align:middle; margin-right:8px; }
|
|
.num { font-variant-numeric:tabular-nums; }
|
|
.tag { display:inline-block; padding:1px 8px; border-radius:10px; font-size:11px;
|
|
background:#222836; color:var(--dim); margin:1px; }
|
|
details { margin-top:14px; } summary { cursor:pointer; color:var(--dim); }
|
|
.sample { border:1px solid var(--line); border-radius:8px; padding:10px 12px;
|
|
margin:8px 0; background:var(--panel); }
|
|
pre { white-space:pre-wrap; word-break:break-all; color:#a8b3c5; font-size:12px;
|
|
background:#10131a; padding:8px; border-radius:6px; margin:6px 0; }
|
|
.ok { color:var(--ok); } .bad { color:var(--bad); }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<header><h1>EvalHarness Dashboard <span>generated by evalharness viz</span></h1></header>
|
|
<div class="wrap">
|
|
<div class="cards">__CARDS__</div>
|
|
<table id="tbl"><thead><tr>
|
|
<th data-k="dataset">dataset</th><th data-k="model">model</th>
|
|
<th data-k="n">n</th><th data-k="primary">metric</th>
|
|
<th data-k="value">value</th><th>share</th><th data-k="extract_fail">extract✗</th>
|
|
</tr></thead><tbody id="tb"></tbody></table>
|
|
<details id="drill"><summary>click a row for per-sample details</summary>
|
|
<div id="drill-body"></div></details>
|
|
</div>
|
|
<script>
|
|
const ROWS = __ROWS__;
|
|
const tb = document.getElementById('tb');
|
|
let sortKey = 'value', sortAsc = false;
|
|
function fmt(v) { return (v === null || v === undefined) ? '—' : (v*100).toFixed(1) + '%'; }
|
|
function render() {
|
|
const rows = [...ROWS].sort((a, b) => {
|
|
const av = a[sortKey], bv = b[sortKey];
|
|
const c = (typeof av === 'number' && typeof bv === 'number') ? av - bv
|
|
: String(av ?? '').localeCompare(String(bv ?? ''));
|
|
return sortAsc ? c : -c;
|
|
});
|
|
tb.innerHTML = rows.map((r, i) => `
|
|
<tr data-i="${i}">
|
|
<td><b>${r.dataset}</b> <span class="tag">${r.recipe}</span></td>
|
|
<td>${r.model || '?'}</td><td class="num">${r.n}</td>
|
|
<td>${r.primary}</td>
|
|
<td class="num"><b>${fmt(r.value)}</b></td>
|
|
<td><span class="bar" style="width:${Math.round(r.value*120)}px"></span></td>
|
|
<td class="num">${r.extract_fail || 0}</td>
|
|
</tr>`).join('');
|
|
tb.querySelectorAll('tr').forEach(tr => tr.onclick = () => drill(rows[+tr.dataset.i]));
|
|
}
|
|
function drill(r) {
|
|
const el = document.getElementById('drill-body');
|
|
el.innerHTML = `<h3>${r.dataset} · ${r.model}</h3>`
|
|
+ Object.entries(r.metrics).map(([k, v]) =>
|
|
`<span class="tag">${k}: ${typeof v === 'number' ? fmt(v) : v}</span>`).join(' ')
|
|
+ Object.entries(r.groups || {}).map(([g, sub]) =>
|
|
`<details><summary>group: ${g}</summary><div>`
|
|
+ Object.entries(sub).map(([k, v]) =>
|
|
`<span class="tag">${k}: ${typeof v === 'number' ? fmt(v) : v}</span>`).join(' ')
|
|
+ '</div></details>').join('')
|
|
+ r.samples.map(s => `
|
|
<div class="sample"><b>#${s.id}</b> ${s.ok ? '<span class="ok">extracted</span>' : '<span class="bad">extract✗</span>'}
|
|
<span class="tag">${JSON.stringify(s.scores)}</span>
|
|
${s.error ? `<div class="bad">${s.error}</div>` : ''}
|
|
<div>target: <b>${s.target}</b></div>
|
|
<div>extracted: <b>${s.extracted || '—'}</b></div>
|
|
<pre>${s.raw}</pre></div>`).join('');
|
|
document.getElementById('drill').open = true;
|
|
}
|
|
document.querySelectorAll('th[data-k]').forEach(th => th.onclick = () => {
|
|
const k = th.dataset.k;
|
|
sortAsc = (k === sortKey) ? !sortAsc : false; sortKey = k; render();
|
|
});
|
|
render();
|
|
</script>
|
|
</body></html>
|
|
"""
|