Replace meaningless trend charts with overall rank and win matrix in results page
This commit is contained in:
sora
parent
28f22c11e3
commit
bde21f6624
@ -9,18 +9,18 @@ from typing import Dict, List, Optional, Tuple
|
||||
|
||||
# Display order for comparison charts
|
||||
CATEGORY_ORDER = [
|
||||
('math', '数学推理', [
|
||||
('math', '数学', [
|
||||
'aime24', 'aime25', 'aime26', 'hmmt26',
|
||||
'imo_answerbench', 'competition_math', 'gsm8k',
|
||||
]),
|
||||
('code', '代码', ['humaneval', 'live_code_bench', 'bigcodebench']),
|
||||
('science', '科学 / 高难推理', ['gpqa_diamond', 'super_gpqa', 'hle']),
|
||||
('knowledge', '知识与通用能力', [
|
||||
('science', '科学', ['gpqa_diamond', 'super_gpqa', 'hle']),
|
||||
('knowledge', '知识', [
|
||||
'mmlu', 'mmlu_pro', 'cmmlu', 'bbh', 'arc',
|
||||
'drop', 'hellaswag', 'winogrande', 'simple_qa', 'trivia_qa',
|
||||
]),
|
||||
('long_context', '长文本', ['longbench_v2', 'openai_mrcr']),
|
||||
('tool_agent', '工具调用 / 智能体', ['bfcl_v3', 'general_fc', 'tau2_bench']),
|
||||
('tool_agent', '智能体', ['bfcl_v3', 'general_fc', 'tau2_bench']),
|
||||
]
|
||||
|
||||
BENCHMARK_ALIAS = {
|
||||
@ -367,6 +367,46 @@ def compare_models(
|
||||
'rows': rows,
|
||||
})
|
||||
|
||||
# Overall aggregates
|
||||
overall = []
|
||||
for s in series:
|
||||
vals = [v for v in s['scores'] if v is not None]
|
||||
overall.append({
|
||||
'folder': s['folder'],
|
||||
'label': s['label'],
|
||||
'display': s['display'],
|
||||
'mean': round(sum(vals) / len(vals), 6) if vals else None,
|
||||
'n': len(vals),
|
||||
})
|
||||
overall.sort(key=lambda x: (x['mean'] if x['mean'] is not None else -1), reverse=True)
|
||||
|
||||
# Win count: how many benchmarks each model ranks #1
|
||||
win_counts = {s['folder']: 0 for s in series}
|
||||
for i, b in enumerate(bench_list):
|
||||
best_score = -1
|
||||
best_folders = []
|
||||
for s in series:
|
||||
v = s['scores'][i]
|
||||
if v is None:
|
||||
continue
|
||||
if v > best_score:
|
||||
best_score = v
|
||||
best_folders = [s['folder']]
|
||||
elif v == best_score:
|
||||
best_folders.append(s['folder'])
|
||||
for f in best_folders:
|
||||
win_counts[f] += 1 / len(best_folders)
|
||||
|
||||
# Head-to-head win matrix
|
||||
folders = [s['folder'] for s in series]
|
||||
win_matrix = {f: {g: 0 for g in folders} for f in folders}
|
||||
for i, b in enumerate(bench_list):
|
||||
scores_here = [(s['folder'], s['scores'][i]) for s in series if s['scores'][i] is not None]
|
||||
for f, sv in scores_here:
|
||||
for g, gv in scores_here:
|
||||
if sv > gv:
|
||||
win_matrix[f][g] += 1
|
||||
|
||||
return {
|
||||
'output_dir': overview['output_dir'],
|
||||
'benchmarks': bench_list,
|
||||
@ -377,6 +417,9 @@ def compare_models(
|
||||
'category_labels': category_labels,
|
||||
'category_series': category_series,
|
||||
'category_ranking': category_ranking,
|
||||
'overall': overall,
|
||||
'win_counts': win_counts,
|
||||
'win_matrix': win_matrix,
|
||||
'matrix': {
|
||||
f: {b: overview['matrix'].get(f, {}).get(b) for b in bench_list}
|
||||
for f in selected
|
||||
|
||||
@ -67,17 +67,15 @@
|
||||
<section class="panel charts-panel">
|
||||
<div class="chart-block">
|
||||
<div class="chart-head">
|
||||
<h2>能力分类走势</h2>
|
||||
<span class="muted">横轴 = 能力域，纵轴 = 该域平均分</span>
|
||||
</div>
|
||||
<div class="chart-wrap">
|
||||
<canvas id="categoryLineChart"></canvas>
|
||||
<h2>综合排名</h2>
|
||||
<span class="muted">所选 benchmark 平均分 + 冠军次数</span>
|
||||
</div>
|
||||
<div id="overallRank" class="rank-list"></div>
|
||||
</div>
|
||||
|
||||
<div class="chart-block">
|
||||
<div class="chart-head">
|
||||
<h2>能力分类柱状对比</h2>
|
||||
<h2>能力分类对比</h2>
|
||||
<span class="muted">各模型在同一能力域的平均得分</span>
|
||||
</div>
|
||||
<div class="chart-wrap">
|
||||
@ -87,17 +85,7 @@
|
||||
|
||||
<div class="chart-block">
|
||||
<div class="chart-head">
|
||||
<h2>得分走势</h2>
|
||||
<span class="muted">横轴 = benchmark，纵轴 = score</span>
|
||||
</div>
|
||||
<div class="chart-wrap">
|
||||
<canvas id="lineChart"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chart-block">
|
||||
<div class="chart-head">
|
||||
<h2>柱状对比</h2>
|
||||
<h2>各 Benchmark 对比</h2>
|
||||
<span class="muted">同 benchmark 下各模型得分</span>
|
||||
</div>
|
||||
<div class="chart-wrap">
|
||||
@ -115,6 +103,19 @@
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chart-block">
|
||||
<div class="chart-head">
|
||||
<h2>胜负矩阵</h2>
|
||||
<span class="muted">行模型在多少 benchmark 上击败列模型</span>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
<table id="winMatrixTable">
|
||||
<thead></thead>
|
||||
<tbody></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="chart-block">
|
||||
<div class="chart-head">
|
||||
<h2>得分表</h2>
|
||||
@ -137,6 +138,6 @@
|
||||
</section>
|
||||
</main>
|
||||
</div>
|
||||
<script src="/static/results.js?v=2"></script>
|
||||
<script src="/static/results.js?v=3"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
const state = {
|
||||
overview: null,
|
||||
compare: null,
|
||||
charts: { line: null, bar: null, radar: null, catLine: null, catBar: null },
|
||||
charts: { bar: null, radar: null, catBar: null },
|
||||
};
|
||||
|
||||
const $ = (id) => document.getElementById(id);
|
||||
@ -197,17 +197,10 @@
|
||||
|
||||
const catLabels = compare.category_labels || [];
|
||||
const catSeries = compare.category_series || [];
|
||||
const catLineCanvas = $('categoryLineChart');
|
||||
const catBarCanvas = $('categoryBarChart');
|
||||
if (!catLabels.length || !catSeries.length) {
|
||||
markEmpty(catLineCanvas, '当前选择下无可用能力分类数据');
|
||||
markEmpty(catBarCanvas, '当前选择下无可用能力分类数据');
|
||||
} else {
|
||||
makeChart('catLine', catLineCanvas, {
|
||||
type: 'line',
|
||||
data: { labels: catLabels, datasets: chartDatasets(catSeries, 'line') },
|
||||
options: { ...common, scales: axisOptions(0) },
|
||||
});
|
||||
makeChart('catBar', catBarCanvas, {
|
||||
type: 'bar',
|
||||
data: { labels: catLabels, datasets: chartDatasets(catSeries, 'bar') },
|
||||
@ -222,14 +215,8 @@
|
||||
}
|
||||
|
||||
if (!labels.length) {
|
||||
markEmpty($('lineChart'), '暂无 benchmark 得分');
|
||||
markEmpty($('barChart'), '暂无 benchmark 得分');
|
||||
} else {
|
||||
makeChart('line', $('lineChart'), {
|
||||
type: 'line',
|
||||
data: { labels, datasets: chartDatasets(compare.series, 'line') },
|
||||
options: { ...common, scales: axisOptions(45) },
|
||||
});
|
||||
makeChart('bar', $('barChart'), {
|
||||
type: 'bar',
|
||||
data: { labels, datasets: chartDatasets(compare.series, 'bar') },
|
||||
@ -306,6 +293,61 @@
|
||||
});
|
||||
}
|
||||
|
||||
function renderWinMatrix(compare) {
|
||||
const table = $('winMatrixTable');
|
||||
const thead = table.querySelector('thead');
|
||||
const tbody = table.querySelector('tbody');
|
||||
thead.innerHTML = '';
|
||||
tbody.innerHTML = '';
|
||||
|
||||
const folders = compare.series.map((s) => s.folder);
|
||||
if (folders.length < 2) {
|
||||
tbody.innerHTML = '<tr><td class="na">至少选择 2 个模型才能生成胜负矩阵</td></tr>';
|
||||
return;
|
||||
}
|
||||
const matrix = compare.win_matrix || {};
|
||||
|
||||
const head = document.createElement('tr');
|
||||
head.innerHTML = `<th>模型</th>${folders.map((f) => `<th>${f}</th>`).join('')}`;
|
||||
thead.appendChild(head);
|
||||
|
||||
folders.forEach((f) => {
|
||||
const tr = document.createElement('tr');
|
||||
const cells = folders.map((g) => {
|
||||
const val = matrix[f]?.[g] ?? 0;
|
||||
return `<td>${val}</td>`;
|
||||
}).join('');
|
||||
tr.innerHTML = `<td class="bench-col">${f}</td>${cells}`;
|
||||
tbody.appendChild(tr);
|
||||
});
|
||||
}
|
||||
|
||||
function renderOverallRank(compare) {
|
||||
const box = $('overallRank');
|
||||
box.innerHTML = '';
|
||||
const overall = compare.overall || [];
|
||||
const wins = compare.win_counts || {};
|
||||
if (!overall.length) {
|
||||
box.innerHTML = '<div class="muted">无数据</div>';
|
||||
return;
|
||||
}
|
||||
overall.forEach((item, idx) => {
|
||||
const card = document.createElement('div');
|
||||
card.className = 'rank-card';
|
||||
const pct = item.mean != null ? (item.mean * 100).toFixed(2) + '%' : '—';
|
||||
const win = wins[item.folder] != null ? wins[item.folder].toFixed(1) : '0';
|
||||
card.innerHTML = `
|
||||
<div class="rank-row">
|
||||
<span class="rank-idx">#${idx + 1}</span>
|
||||
<span class="rank-name">${item.folder}</span>
|
||||
<span class="rank-score">${pct}</span>
|
||||
</div>
|
||||
<small class="muted">${item.n} benches · ${win} 项第一</small>
|
||||
`;
|
||||
box.appendChild(card);
|
||||
});
|
||||
}
|
||||
|
||||
function renderRanking(compare) {
|
||||
const box = $('rankList');
|
||||
box.innerHTML = '';
|
||||
@ -364,6 +406,8 @@
|
||||
state.compare = data;
|
||||
renderCharts(data);
|
||||
renderTable(data);
|
||||
renderWinMatrix(data);
|
||||
renderOverallRank(data);
|
||||
renderRanking(data);
|
||||
setMsg(`已对比 ${data.series.length} 模型 × ${data.benchmarks.length} benches`, 'ok');
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user