662 lines
22 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

(() => {
const state = {
meta: null,
tab: 'scope',
selectionMode: 'suite',
suiteKind: 'builtin',
suite: 'official',
customSuites: {},
activeJobId: null,
pollTimer: null,
logOffset: 0,
activeCategory: 'all',
searchQuery: '',
};
const $ = (id) => document.getElementById(id);
const formMsg = $('formMsg');
const logView = $('logView');
const jobList = $('jobList');
function setMsg(text, type = '') {
formMsg.textContent = text || '';
formMsg.className = `form-msg ${type}`.trim();
}
async function api(path, options) {
const res = await fetch(path, {
headers: { 'Content-Type': 'application/json' },
...options,
});
const data = await res.json().catch(() => ({}));
if (!res.ok) {
const detail = data.detail || res.statusText || 'request failed';
throw new Error(typeof detail === 'string' ? detail : JSON.stringify(detail));
}
return data;
}
function escapeHtml(s) {
return String(s)
.replace(/&/g, '&')
.replace(/</g, '&lt;')
.replace(/>/g, '&gt;')
.replace(/"/g, '&quot;');
}
function fillDefaults(meta) {
const d = meta.defaults;
$('model').value = d.model || '';
$('api_url').value = d.api_url || '';
$('api_key').value = d.api_key || 'EMPTY';
$('seed').value = d.seed;
$('batch_size').value = d.batch_size;
$('thinking').checked = !!d.thinking;
$('dataset_dir').value = d.dataset_dir || '';
$('output_dir').value = d.output_dir || '';
$('config').value = d.config || '';
$('tokenizer_path').value = d.tokenizer_path || '';
$('judge_model').value = d.judge_model || '';
$('judge_api_url').value = d.judge_api_url || '';
$('judge_max_tokens').value = d.judge_max_tokens || '';
state.suite = d.suite || 'official';
state.suiteKind = 'builtin';
state.customSuites = meta.custom_suites || {};
}
// ---- Tabs ----
function setTab(tab) {
state.tab = tab;
document.querySelectorAll('.tabs .tab').forEach((el) => {
el.classList.toggle('active', el.dataset.tab === tab);
});
document.querySelectorAll('.tab-pane').forEach((el) => {
el.classList.toggle('active', el.id === `${tab}Tab`);
});
}
// ---- Mode ----
function setMode(mode) {
state.selectionMode = mode;
document.querySelectorAll('.mode-btn').forEach((el) => {
el.classList.toggle('active', el.dataset.mode === mode);
});
$('suitePane').classList.toggle('hidden', mode !== 'suite');
$('datasetsPane').classList.toggle('hidden', mode !== 'datasets');
updateEstimate();
}
// ---- Suites ----
function formatHours(h) {
if (h == null || h <= 0) return '';
if (h < 1) return `${Math.round(h * 60)}min`;
return `~${h}h`;
}
function renderSuites() {
const box = $('suiteList');
box.innerHTML = '';
const suites = state.meta.suites || {};
const estimates = state.meta.suite_estimates || {};
Object.keys(suites).forEach((name) => {
const info = suites[name];
const count = info.all.length;
const est = estimates[name];
const btn = document.createElement('button');
btn.type = 'button';
btn.className = `suite-card${state.suiteKind === 'builtin' && state.suite === name ? ' active' : ''}`;
btn.innerHTML = `
<strong>${escapeHtml(name)}</strong>
<div class="suite-meta">
<span>${count} benches</span>
${est ? `<span>${formatHours(est)}</span>` : ''}
</div>
`;
btn.addEventListener('click', () => {
state.suiteKind = 'builtin';
state.suite = name;
renderSuites();
renderCustomSuites();
updateEstimate();
});
box.appendChild(btn);
});
}
function renderCustomSuites() {
const box = $('customSuiteList');
const names = Object.keys(state.customSuites || {});
box.innerHTML = '';
$('customSuiteHint').classList.toggle('hidden', names.length > 0);
names.forEach((name) => {
const items = state.customSuites[name] || [];
const wrap = document.createElement('div');
wrap.className = `suite-card${state.suiteKind === 'custom' && state.suite === name ? ' active' : ''}`;
wrap.innerHTML = `
<strong>${escapeHtml(name)}</strong>
<div class="suite-meta">
<span>${items.length} benches</span>
<span>自定义</span>
</div>
<button type="button" class="ghost del">删除</button>
`;
wrap.querySelector('.del').addEventListener('click', async (e) => {
e.stopPropagation();
if (!confirm(`删除自定义组合「${name}」?`)) return;
try {
const data = await api(`/api/custom-suites/${encodeURIComponent(name)}`, { method: 'DELETE' });
state.customSuites = data.suites || {};
if (state.suiteKind === 'custom' && state.suite === name) {
state.suiteKind = 'builtin';
state.suite = state.meta.defaults?.suite || 'official';
}
renderCustomSuites();
renderSuites();
setMsg(`已删除组合: ${name}`, 'ok');
} catch (err) {
setMsg(err.message, 'error');
}
});
wrap.addEventListener('click', (e) => {
if (e.target.closest('.del')) return;
state.suiteKind = 'custom';
state.suite = name;
renderSuites();
renderCustomSuites();
updateEstimate();
});
box.appendChild(wrap);
});
}
// ---- Benchmark picker ----
function benchmarkTags(name) {
const tags = [];
const multi = state.meta.multi_run || {};
const req = state.meta.benchmark_requirements || {};
if (multi[name]) tags.push({ text: `×${multi[name]}`, cls: '' });
if (req[name]) {
if (req[name].sandbox) tags.push({ text: 'sandbox', cls: 'warn' });
else if (req[name].swe) tags.push({ text: 'swe', cls: 'warn' });
else if (req[name].agent) tags.push({ text: 'agent', cls: 'warn' });
}
return tags;
}
function renderBenchmarks() {
const box = $('benchmarkList');
box.innerHTML = '';
const multi = new Set(Object.keys(state.meta.multi_run || {}));
const categories = state.meta.categories || [];
categories.forEach((cat) => {
const section = document.createElement('div');
section.className = 'bench-category';
section.dataset.category = cat.id;
const grid = document.createElement('div');
grid.className = 'benchmark-grid';
cat.items.forEach((name) => {
const label = document.createElement('label');
label.className = 'bench-item';
label.dataset.name = name;
label.dataset.category = cat.id;
const tags = benchmarkTags(name);
label.innerHTML = `
<input type="checkbox" value="${escapeHtml(name)}" />
<span class="name">${escapeHtml(name)}</span>
${tags.map((t) => `<span class="tag ${t.cls}" title="${escapeHtml(t.text)}">${escapeHtml(t.text)}</span>`).join('')}
`;
label.querySelector('input').addEventListener('change', updateSelectedCount);
grid.appendChild(label);
});
if (grid.children.length) {
section.appendChild(grid);
box.appendChild(section);
}
});
updateSelectedCount();
applyFilters();
}
function renderCategoryChips() {
const box = $('categoryFilters');
box.innerHTML = '';
const all = document.createElement('button');
all.type = 'button';
all.className = `chip${state.activeCategory === 'all' ? ' active' : ''}`;
all.textContent = '全部';
all.addEventListener('click', () => { state.activeCategory = 'all'; renderCategoryChips(); applyFilters(); });
box.appendChild(all);
(state.meta.categories || []).forEach((cat) => {
const btn = document.createElement('button');
btn.type = 'button';
btn.className = `chip${state.activeCategory === cat.id ? ' active' : ''}`;
btn.textContent = `${cat.name} (${cat.items.length})`;
btn.addEventListener('click', () => { state.activeCategory = cat.id; renderCategoryChips(); applyFilters(); });
box.appendChild(btn);
});
}
function applyFilters() {
const q = state.searchQuery.toLowerCase().trim();
document.querySelectorAll('.bench-item').forEach((el) => {
const name = el.dataset.name.toLowerCase();
const cat = el.dataset.category;
const matchCat = state.activeCategory === 'all' || cat === state.activeCategory;
const matchSearch = !q || name.includes(q);
el.classList.toggle('hidden-item', !(matchCat && matchSearch));
});
document.querySelectorAll('.bench-category').forEach((cat) => {
const visible = cat.querySelectorAll('.bench-item:not(.hidden-item)').length > 0;
cat.classList.toggle('hidden', !visible);
});
}
function updateSelectedCount() {
const n = [...document.querySelectorAll('#benchmarkList input:checked')].length;
$('selectedCount').textContent = `已选 ${n}`;
updateEstimate();
}
function updateEstimate() {
const el = $('scopeEstimate');
if (state.selectionMode === 'suite') {
const est = (state.meta.suite_estimates || {})[state.suite];
el.textContent = est ? `预计 ${formatHours(est)}` : '';
return;
}
const datasets = [...document.querySelectorAll('#benchmarkList input:checked')].map((el) => el.value);
if (!datasets.length) {
el.textContent = '';
return;
}
const multi = state.meta.multi_run || {};
// rough estimate: single-run bench ~0.5h, multi-run scaled
let hours = 0;
datasets.forEach((d) => {
hours += multi[d] ? multi[d] * 0.5 : 0.5;
});
el.textContent = `预计 ${formatHours(hours)}`;
}
// ---- Command preview ----
function buildCommandPayload() {
const datasets = [...document.querySelectorAll('#benchmarkList input:checked')].map((el) => el.value);
const excludeRaw = state.selectionMode === 'suite' ? '' : '';
const exclude = [];
let selectionMode = state.selectionMode;
let suite = state.suite;
let finalDatasets = datasets;
if (selectionMode === 'suite' && state.suiteKind === 'custom') {
selectionMode = 'datasets';
finalDatasets = [...(state.customSuites[state.suite] || [])];
}
return {
model: $('model').value.trim(),
api_url: $('api_url').value.trim(),
api_key: $('api_key').value.trim() || 'EMPTY',
thinking: $('thinking').checked,
selection_mode: selectionMode,
suite,
datasets: finalDatasets,
exclude,
folder_name: $('folder_name').value.trim() || null,
limit: $('limit').value.trim() || null,
seed: Number($('seed').value || 42),
batch_size: Number($('batch_size').value || 4),
thinking_max_tokens_scale: Number($('thinking_max_tokens_scale').value || 1),
max_tokens_add: Number($('max_tokens_add').value || 0),
dataset_dir: $('dataset_dir').value.trim() || null,
output_dir: $('output_dir').value.trim() || null,
config: $('config').value.trim() || null,
tokenizer_path: $('tokenizer_path').value.trim() || null,
judge_model: $('judge_model').value.trim() || null,
judge_api_url: $('judge_api_url').value.trim() || null,
judge_api_key: $('judge_api_key').value.trim() || null,
judge_max_tokens: $('judge_max_tokens').value ? Number($('judge_max_tokens').value) : null,
write_summary: $('write_summary').value === 'true',
};
}
function updateCommandPreview(cmd) {
$('activeCmd').textContent = (cmd || []).join(' ');
$('copyCmdBtn').disabled = !cmd;
}
// ---- Jobs ----
function statusClass(status) {
return status || 'idle';
}
function renderActive(job) {
const card = $('activeCard');
if (!job) {
card.className = 'active-card idle';
$('activeStatus').className = 'badge idle';
$('activeStatus').textContent = 'idle';
$('activeJobId').textContent = '—';
$('progressBar').style.width = '0%';
$('progressText').textContent = '';
updateCommandPreview(null);
$('stopBtn').disabled = true;
return;
}
card.className = `active-card ${statusClass(job.status)}`;
$('activeStatus').className = `badge ${statusClass(job.status)}`;
$('activeStatus').textContent = job.status;
$('activeJobId').textContent = job.id;
$('stopBtn').disabled = job.status !== 'running';
state.activeJobId = job.id;
updateCommandPreview(job.command);
}
function renderJobs(jobs, activeId) {
jobList.innerHTML = '';
if (!jobs.length) {
jobList.innerHTML = '<div class="muted">暂无历史任务</div>';
return;
}
jobs.forEach((job) => {
const btn = document.createElement('button');
btn.type = 'button';
btn.className = `job-item${job.id === activeId ? ' active' : ''}`;
const model = job.payload?.model || '-';
const mode = job.payload?.thinking ? 'thinking' : 'no-thinking';
const scope = job.payload?.selection_mode === 'datasets'
? `${job.payload.datasets?.length || 0} benches`
: (job.payload?.suite || '-');
btn.innerHTML = `
<div class="job-row">
<strong>${escapeHtml(job.id)}</strong>
<span class="badge ${statusClass(job.status)}">${escapeHtml(job.status)}</span>
</div>
<small>${escapeHtml(model)} · ${mode} · ${escapeHtml(scope)}</small>
`;
btn.addEventListener('click', () => followJob(job.id, true));
jobList.appendChild(btn);
});
}
async function refreshJobs() {
const data = await api('/api/jobs');
renderJobs(data.jobs || [], data.active_job_id);
if (data.active_job_id) {
const active = (data.jobs || []).find((j) => j.id === data.active_job_id);
if (active) renderActive(active);
} else if (state.activeJobId) {
const current = (data.jobs || []).find((j) => j.id === state.activeJobId);
if (current) renderActive(current);
}
return data;
}
function tryParseProgress(text) {
// Look for patterns like "Evaluating[mmlu_pro] 12%| 1450/12032"
const m = text.match(/\[eval\]\s*(\d+)%\|\s*(\d+)\/(\d+)/i);
if (m) {
return { pct: Number(m[1]), current: Number(m[2]), total: Number(m[3]) };
}
const m2 = text.match(/Evaluating\[(\w+)\]\s+(\d+)%\|\s*(\d+)\/(\d+)/);
if (m2) {
return { pct: Number(m2[2]), current: Number(m2[3]), total: Number(m2[4]) };
}
return null;
}
async function pullLogs(reset = false) {
if (!state.activeJobId) return;
if (reset) {
state.logOffset = 0;
logView.textContent = '';
}
const data = await api(`/api/jobs/${state.activeJobId}/logs?offset=${state.logOffset}`);
if (data.content) {
logView.textContent += data.content;
state.logOffset = data.next_offset;
if ($('autoScroll').checked) logView.scrollTop = logView.scrollHeight;
const prog = tryParseProgress(logView.textContent);
if (prog) {
$('progressBar').style.width = `${prog.pct}%`;
$('progressText').textContent = `${prog.pct}% · ${prog.current}/${prog.total}`;
}
}
if (data.status) {
$('activeStatus').className = `badge ${statusClass(data.status)}`;
$('activeStatus').textContent = data.status;
$('activeCard').className = `active-card ${statusClass(data.status)}`;
$('stopBtn').disabled = data.status !== 'running';
if (data.done) {
$('progressBar').style.width = '100%';
$('progressText').textContent = data.status === 'completed' ? '已完成' : `结束: ${data.status}`;
}
}
if (data.done) stopPolling();
}
function stopPolling() {
if (state.pollTimer) {
clearInterval(state.pollTimer);
state.pollTimer = null;
}
}
function startPolling() {
stopPolling();
state.pollTimer = setInterval(async () => {
try {
await pullLogs(false);
await refreshJobs();
} catch (e) {
console.error(e);
}
}, 1200);
}
async function followJob(jobId, resetLog = false) {
const job = await api(`/api/jobs/${jobId}`);
renderActive(job);
await pullLogs(resetLog);
if (job.status === 'running' || job.status === 'queued') startPolling();
else stopPolling();
}
async function createJob() {
setMsg('');
const required = [
['dataset_dir', 'Dataset Dir'],
['output_dir', 'Output Dir'],
['config', 'Config YAML'],
['tokenizer_path', 'Tokenizer Path'],
];
for (const [id, label] of required) {
if (!$(id).value.trim()) {
setMsg(`请填写必填路径: ${label}`, 'error');
setTab('config');
$(id).focus();
return;
}
}
const payload = buildCommandPayload();
if (payload.selection_mode === 'datasets' && !payload.datasets.length) {
setMsg('请至少选择一个 benchmark', 'error');
setTab('scope');
return;
}
$('launchBtn').disabled = true;
try {
const job = await api('/api/jobs', {
method: 'POST',
body: JSON.stringify(payload),
});
setMsg(`任务已启动: ${job.id}`, 'ok');
renderActive(job);
state.logOffset = 0;
logView.textContent = '';
await refreshJobs();
startPolling();
await pullLogs(true);
} catch (err) {
setMsg(err.message, 'error');
} finally {
$('launchBtn').disabled = false;
}
}
// ---- Presets (localStorage) ----
const PRESET_KEY = 'evalstone_path_presets';
function getPresets() {
try {
return JSON.parse(localStorage.getItem(PRESET_KEY) || '[]');
} catch {
return [];
}
}
function savePreset() {
const name = prompt('预设名称');
if (!name) return;
const presets = getPresets();
presets.push({
name,
dataset_dir: $('dataset_dir').value,
output_dir: $('output_dir').value,
config: $('config').value,
tokenizer_path: $('tokenizer_path').value,
});
localStorage.setItem(PRESET_KEY, JSON.stringify(presets));
setMsg('预设已保存', 'ok');
}
function loadPreset() {
const presets = getPresets();
if (!presets.length) {
setMsg('没有保存的预设', 'error');
return;
}
const list = presets.map((p, i) => `${i + 1}. ${p.name}`).join('\n');
const idx = prompt(`选择预设编号:\n${list}`);
const n = Number(idx);
if (!n || n < 1 || n > presets.length) return;
const p = presets[n - 1];
$('dataset_dir').value = p.dataset_dir || '';
$('output_dir').value = p.output_dir || '';
$('config').value = p.config || '';
$('tokenizer_path').value = p.tokenizer_path || '';
setMsg(`已加载预设: ${p.name}`, 'ok');
}
function applyDefaultPreset() {
const d = state.meta.defaults;
$('dataset_dir').value = d.dataset_dir || '';
$('output_dir').value = d.output_dir || '';
$('config').value = d.config || '';
$('tokenizer_path').value = d.tokenizer_path || '';
}
// ---- Init ----
async function init() {
try {
await api('/api/health');
$('healthDot').className = 'dot ok';
$('healthText').textContent = 'online';
} catch (e) {
$('healthDot').className = 'dot bad';
$('healthText').textContent = 'offline';
setMsg(e.message, 'error');
return;
}
state.meta = await api('/api/meta');
fillDefaults(state.meta);
renderSuites();
renderCustomSuites();
renderBenchmarks();
renderCategoryChips();
setMode('suite');
setTab('scope');
updateEstimate();
const jobs = await refreshJobs();
if (jobs.active_job_id) {
await followJob(jobs.active_job_id, true);
} else if (jobs.jobs?.[0]) {
await followJob(jobs.jobs[0].id, true);
}
}
// ---- Events ----
document.querySelectorAll('.tabs .tab').forEach((el) => {
el.addEventListener('click', () => setTab(el.dataset.tab));
});
document.querySelectorAll('.mode-btn').forEach((el) => {
el.addEventListener('click', () => setMode(el.dataset.mode));
});
$('benchSearch').addEventListener('input', (e) => {
state.searchQuery = e.target.value;
applyFilters();
});
$('selectAllBtn').addEventListener('click', () => {
document.querySelectorAll('#benchmarkList input:not(.hidden-item input)').forEach((el) => { el.checked = true; });
updateSelectedCount();
});
$('clearAllBtn').addEventListener('click', () => {
document.querySelectorAll('#benchmarkList input').forEach((el) => { el.checked = false; });
updateSelectedCount();
});
$('saveCustomBtn').addEventListener('click', async () => {
const datasets = [...document.querySelectorAll('#benchmarkList input:checked')].map((el) => el.value);
if (!datasets.length) {
setMsg('请先勾选 benchmark', 'error');
return;
}
const name = prompt('输入自定义组合名称:');
if (!name || !name.trim()) return;
try {
const data = await api('/api/custom-suites', {
method: 'POST',
body: JSON.stringify({ name: name.trim(), datasets }),
});
state.customSuites = data.suites || {};
renderCustomSuites();
state.suiteKind = 'custom';
state.suite = name.trim();
renderSuites();
setMode('suite');
setMsg(`已保存组合: ${name.trim()}`, 'ok');
} catch (err) {
setMsg(err.message, 'error');
}
});
$('refreshJobsBtn').addEventListener('click', () => refreshJobs().catch((e) => setMsg(e.message, 'error')));
$('launchBtn').addEventListener('click', createJob);
$('copyCmdBtn').addEventListener('click', () => {
const text = $('activeCmd').textContent;
if (!text || text === '尚未启动任务') return;
navigator.clipboard.writeText(text).then(() => setMsg('命令已复制', 'ok')).catch(() => setMsg('复制失败', 'error'));
});
$('stopBtn').addEventListener('click', async () => {
if (!state.activeJobId) return;
if (!confirm(`确认停止任务 ${state.activeJobId}`)) return;
try {
const job = await api(`/api/jobs/${state.activeJobId}/stop`, { method: 'POST' });
renderActive(job);
setMsg('任务已停止', 'ok');
await refreshJobs();
await pullLogs(false);
} catch (err) {
setMsg(err.message, 'error');
}
});
$('presetDefaultBtn').addEventListener('click', applyDefaultPreset);
$('presetCurrentBtn').addEventListener('click', savePreset);
$('presetLoadBtn').addEventListener('click', loadPreset);
init();
})();