diff --git a/webui/.gitignore b/webui/.gitignore new file mode 100644 index 0000000..19dbe64 --- /dev/null +++ b/webui/.gitignore @@ -0,0 +1,5 @@ +# Runtime job state / logs — do not commit +data/ +__pycache__/ +*.pyc +.pytest_cache/ diff --git a/webui/requirements.txt b/webui/requirements.txt new file mode 100644 index 0000000..a65ec36 --- /dev/null +++ b/webui/requirements.txt @@ -0,0 +1,3 @@ +fastapi>=0.110.0 +uvicorn>=0.27.0 +pydantic>=2.0.0 diff --git a/webui/results_scan.py b/webui/results_scan.py new file mode 100644 index 0000000..8f08305 --- /dev/null +++ b/webui/results_scan.py @@ -0,0 +1,318 @@ +#!/usr/bin/env python3 +"""Scan EvalScope output directories and aggregate per-model benchmark scores.""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +# Display order for comparison charts +CATEGORY_ORDER = [ + ('math', '数学推理', [ + 'aime24', 'aime25', 'aime26', 'hmmt26', + 'imo_answerbench', 'competition_math', 'gsm8k', + ]), + ('code', '代码', ['humaneval', 'live_code_bench', 'bigcodebench']), + ('science', '科学 / 高难推理', ['gpqa_diamond', 'super_gpqa', 'hle']), + ('knowledge', '知识与通用能力', [ + 'mmlu', 'mmlu_pro', 'cmmlu', 'bbh', 'arc', + 'drop', 'hellaswag', 'winogrande', 'simple_qa', 'trivia_qa', + ]), + ('long_context', '长文本', ['longbench_v2', 'openai_mrcr']), + ('tool_agent', '工具调用 / 智能体', ['bfcl_v3', 'general_fc', 'tau2_bench']), +] + +BENCHMARK_ALIAS = { + 'hle_low': 'hle', +} + +SKIP_DIR_NAMES = { + 'active_time', 'perf_stats_backup', 'predictions_archive', + 'logs', 'configs', 'reports', 'predictions', 'reviews', +} + + +def _is_seed_dir(path: Path) -> bool: + return path.is_dir() and path.name.startswith('seed_') + + +def _is_benchmark_dir(path: Path) -> bool: + if not path.is_dir() or path.name in SKIP_DIR_NAMES: + return False + try: + return any(_is_seed_dir(child) for child in path.iterdir()) + except OSError: + return False + + +def _is_model_folder(path: Path) -> bool: + if not path.is_dir() or path.name.startswith('.'): + return False + try: + return any(_is_benchmark_dir(child) for child in path.iterdir()) + except OSError: + return False + + +def extract_score(report_data: dict) -> Optional[float]: + score = report_data.get('score') + if score is not None: + try: + return float(score) + except (TypeError, ValueError): + pass + for metric in report_data.get('metrics') or []: + if metric.get('name') in ('mean_acc', 'acc', 'accuracy', 'pass@1', 'Score'): + for key in ('score', 'macro_score'): + if metric.get(key) is not None: + try: + return float(metric[key]) + except (TypeError, ValueError): + continue + # fallback: first metric with a score + for metric in report_data.get('metrics') or []: + for key in ('score', 'macro_score'): + if metric.get(key) is not None: + try: + return float(metric[key]) + except (TypeError, ValueError): + continue + return None + + +def _find_report_files(seed_dir: Path, benchmark: str) -> List[Path]: + reports_dir = seed_dir / 'reports' + if not reports_dir.is_dir(): + return [] + preferred = [ + reports_dir / f'{benchmark}.json', + reports_dir / f'{BENCHMARK_ALIAS.get(benchmark, benchmark)}.json', + ] + found = [p for p in preferred if p.is_file()] + if found: + return found + # nested: reports//.json (legacy) + nested = sorted(reports_dir.glob(f'*/*.json')) + sorted(reports_dir.glob('*.json')) + return [p for p in nested if p.is_file()] + + +def _read_report(path: Path) -> Optional[dict]: + try: + return json.loads(path.read_text(encoding='utf-8')) + except Exception: + return None + + +def collect_benchmark_scores(model_dir: Path, benchmark: str) -> Optional[dict]: + bench_dir = model_dir / benchmark + if not bench_dir.is_dir(): + # alias: hle stored as hle_low + for alias_src, alias_dst in BENCHMARK_ALIAS.items(): + if benchmark == alias_dst: + alt = model_dir / alias_src + if alt.is_dir(): + bench_dir = alt + break + else: + return None + + scores: List[float] = [] + runs: List[dict] = [] + model_name = None + num_samples = None + + for seed_dir in sorted(p for p in bench_dir.iterdir() if _is_seed_dir(p)): + for report_path in _find_report_files(seed_dir, benchmark): + data = _read_report(report_path) + if not data: + continue + score = extract_score(data) + if score is None: + continue + scores.append(score) + if model_name is None: + model_name = data.get('model_name') + if num_samples is None: + num_samples = data.get('num') + if num_samples is None: + metrics = data.get('metrics') or [] + if metrics: + num_samples = metrics[0].get('num') + try: + rel = str(report_path.relative_to(model_dir)) + except ValueError: + rel = str(report_path) + runs.append({ + 'seed_dir': seed_dir.name, + 'score': score, + 'report': rel, + }) + break # one report per seed dir + + if not scores: + return None + + avg = sum(scores) / len(scores) + return { + 'benchmark': BENCHMARK_ALIAS.get(benchmark, benchmark), + 'score': round(avg, 6), + 'scores': scores, + 'n_runs': len(scores), + 'num_samples': num_samples, + 'model_name': model_name, + 'runs': runs, + } + + +def list_benchmarks_in_model(model_dir: Path) -> List[str]: + names = [] + for child in sorted(model_dir.iterdir()): + if not _is_benchmark_dir(child): + continue + names.append(BENCHMARK_ALIAS.get(child.name, child.name)) + # unique preserve order + seen = set() + out = [] + for n in names: + if n not in seen: + seen.add(n) + out.append(n) + return out + + +def scan_output_dir(output_dir: Path) -> dict: + output_dir = Path(output_dir) + if not output_dir.is_dir(): + return { + 'output_dir': str(output_dir), + 'models': [], + 'benchmarks': [], + 'categories': [], + 'matrix': {}, + } + + models = [] + all_benchmarks = set() + matrix: Dict[str, Dict[str, dict]] = {} + + for child in sorted(output_dir.iterdir()): + if not _is_model_folder(child): + continue + folder = child.name + bench_names = list_benchmarks_in_model(child) + model_scores: Dict[str, dict] = {} + served_model = None + for bench in bench_names: + info = collect_benchmark_scores(child, bench) + if not info: + continue + model_scores[bench] = info + all_benchmarks.add(bench) + if served_model is None and info.get('model_name'): + served_model = info['model_name'] + + if not model_scores: + continue + + models.append({ + 'folder': folder, + 'model_name': served_model or folder, + 'benchmarks': sorted(model_scores.keys()), + 'n_benchmarks': len(model_scores), + 'path': str(child), + }) + matrix[folder] = model_scores + + # Ordered benchmark list by category, then leftovers + ordered = [] + seen = set() + categories = [] + for cat_id, cat_name, items in CATEGORY_ORDER: + present = [b for b in items if b in all_benchmarks] + if present: + categories.append({'id': cat_id, 'name': cat_name, 'items': present}) + for b in present: + if b not in seen: + seen.add(b) + ordered.append(b) + others = sorted(all_benchmarks - seen) + if others: + categories.append({'id': 'other', 'name': '其他', 'items': others}) + ordered.extend(others) + + # Compact matrix for API: folder -> benchmark -> score summary + compact = {} + for folder, benches in matrix.items(): + compact[folder] = { + b: { + 'score': info['score'], + 'n_runs': info['n_runs'], + 'num_samples': info.get('num_samples'), + 'model_name': info.get('model_name'), + } + for b, info in benches.items() + } + + return { + 'output_dir': str(output_dir), + 'models': models, + 'benchmarks': ordered, + 'categories': categories, + 'matrix': compact, + } + + +def compare_models( + output_dir: Path, + folders: Optional[List[str]] = None, + benchmarks: Optional[List[str]] = None, +) -> dict: + overview = scan_output_dir(output_dir) + available = {m['folder'] for m in overview['models']} + if folders: + selected = [f for f in folders if f in available] + else: + selected = [m['folder'] for m in overview['models']] + + if benchmarks: + bench_list = [b for b in benchmarks if b in overview['benchmarks']] + else: + bench_list = list(overview['benchmarks']) + + series = [] + for folder in selected: + scores = [] + for b in bench_list: + cell = overview['matrix'].get(folder, {}).get(b) + scores.append(cell['score'] if cell else None) + model_meta = next((m for m in overview['models'] if m['folder'] == folder), None) + series.append({ + 'folder': folder, + 'label': model_meta['model_name'] if model_meta else folder, + 'display': folder, + 'scores': scores, + }) + + # Per-benchmark ranking + ranking = [] + for i, b in enumerate(bench_list): + rows = [] + for s in series: + if s['scores'][i] is not None: + rows.append({'folder': s['folder'], 'label': s['label'], 'score': s['scores'][i]}) + rows.sort(key=lambda x: x['score'], reverse=True) + ranking.append({'benchmark': b, 'rows': rows}) + + return { + 'output_dir': overview['output_dir'], + 'benchmarks': bench_list, + 'categories': overview['categories'], + 'models': [m for m in overview['models'] if m['folder'] in selected], + 'series': series, + 'ranking': ranking, + 'matrix': { + f: {b: overview['matrix'].get(f, {}).get(b) for b in bench_list} + for f in selected + }, + } diff --git a/webui/server.py b/webui/server.py new file mode 100644 index 0000000..45cdc8f --- /dev/null +++ b/webui/server.py @@ -0,0 +1,568 @@ +#!/usr/bin/env python3 +"""Local EvalScope launch panel — wraps bash/run.py via FastAPI.""" + +from __future__ import annotations + +import asyncio +import json +import os +import signal +import sys +import uuid +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Dict, List, Optional + +from fastapi import FastAPI, HTTPException +from fastapi.responses import FileResponse, StreamingResponse +from fastapi.staticfiles import StaticFiles +from pydantic import BaseModel, Field + +WEBUI_DIR = Path(__file__).parent.resolve() +PROJECT_ROOT = WEBUI_DIR.parent +BASH_DIR = PROJECT_ROOT / 'bash' +RUN_SCRIPT = BASH_DIR / 'run.py' +STATIC_DIR = WEBUI_DIR / 'static' +DATA_DIR = WEBUI_DIR / 'data' +JOBS_DIR = DATA_DIR / 'jobs' +LOGS_DIR = DATA_DIR / 'logs' + +sys.path.insert(0, str(BASH_DIR)) +import run as run_module # noqa: E402 +import results_scan # noqa: E402 + +JOBS_DIR.mkdir(parents=True, exist_ok=True) +LOGS_DIR.mkdir(parents=True, exist_ok=True) + +DEFAULT_OUTPUT_DIR = Path(run_module.DEFAULT_OUTPUT_DIR) + +app = FastAPI(title='EvalStone Launch Panel', version='1.0.0') + + +# --------------------------------------------------------------------------- +# Job store +# --------------------------------------------------------------------------- + +class JobRecord: + def __init__(self, job_id: str, payload: dict, command: List[str]): + self.id = job_id + self.payload = payload + self.command = command + self.status = 'queued' # queued | running | completed | failed | stopped + self.created_at = datetime.now(timezone.utc).isoformat() + self.started_at: Optional[str] = None + self.finished_at: Optional[str] = None + self.return_code: Optional[int] = None + self.pid: Optional[int] = None + self.log_path = LOGS_DIR / f'{job_id}.log' + self.error: Optional[str] = None + self._proc: Optional[asyncio.subprocess.Process] = None + self._log_fp = None + + def to_dict(self) -> dict: + return { + 'id': self.id, + 'status': self.status, + 'created_at': self.created_at, + 'started_at': self.started_at, + 'finished_at': self.finished_at, + 'return_code': self.return_code, + 'pid': self.pid, + 'command': self.command, + 'payload': self.payload, + 'log_path': str(self.log_path), + 'error': self.error, + } + + def save(self) -> None: + path = JOBS_DIR / f'{self.id}.json' + path.write_text(json.dumps(self.to_dict(), ensure_ascii=False, indent=2), encoding='utf-8') + + +JOBS: Dict[str, JobRecord] = {} +_ACTIVE_JOB_ID: Optional[str] = None +_LOCK = asyncio.Lock() + + +def _load_existing_jobs() -> None: + for path in sorted(JOBS_DIR.glob('*.json'), key=lambda p: p.stat().st_mtime, reverse=True): + try: + data = json.loads(path.read_text(encoding='utf-8')) + job = JobRecord(data['id'], data.get('payload', {}), data.get('command', [])) + job.status = data.get('status', 'unknown') + job.created_at = data.get('created_at', job.created_at) + job.started_at = data.get('started_at') + job.finished_at = data.get('finished_at') + job.return_code = data.get('return_code') + job.pid = data.get('pid') + job.error = data.get('error') + if job.status == 'running': + # Process cannot be resumed after server restart + job.status = 'failed' + job.error = 'Server restarted while job was running' + job.finished_at = datetime.now(timezone.utc).isoformat() + job.save() + JOBS[job.id] = job + except Exception: + continue + + +_load_existing_jobs() + + +# --------------------------------------------------------------------------- +# Request models +# --------------------------------------------------------------------------- + +class LaunchRequest(BaseModel): + model: str = Field(..., min_length=1) + api_url: str = Field(..., min_length=1) + api_key: str = 'EMPTY' + thinking: bool = False + selection_mode: str = 'suite' # suite | datasets + suite: str = 'official' + datasets: List[str] = Field(default_factory=list) + exclude: List[str] = Field(default_factory=list) + folder_name: Optional[str] = None + limit: Optional[str] = None + seed: int = 42 + batch_size: int = 4 + thinking_max_tokens_scale: float = 1.0 + max_tokens_add: int = 0 + dataset_dir: Optional[str] = None + output_dir: Optional[str] = None + config: Optional[str] = None + tokenizer_path: Optional[str] = None + judge_model: Optional[str] = None + judge_api_url: Optional[str] = None + judge_api_key: Optional[str] = None + judge_max_tokens: Optional[int] = None + write_summary: bool = True + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +# Capability-domain categories for the custom benchmark picker. +# Order here is the display order in the UI. +BENCHMARK_CATEGORIES = [ + { + 'id': 'math', + 'name': '数学推理', + 'items': [ + 'aime24', 'aime25', 'aime26', 'hmmt26', + 'imo_answerbench', 'competition_math', 'gsm8k', + ], + }, + { + 'id': 'code', + 'name': '代码', + 'items': ['humaneval', 'live_code_bench', 'bigcodebench'], + }, + { + 'id': 'science', + 'name': '科学 / 高难推理', + 'items': ['gpqa_diamond', 'super_gpqa', 'hle'], + }, + { + 'id': 'knowledge', + 'name': '知识与通用能力', + 'items': [ + 'mmlu', 'mmlu_pro', 'cmmlu', 'bbh', 'arc', + 'drop', 'hellaswag', 'winogrande', 'simple_qa', 'trivia_qa', + ], + }, + { + 'id': 'long_context', + 'name': '长文本', + 'items': ['longbench_v2', 'openai_mrcr'], + }, + { + 'id': 'tool_agent', + 'name': '工具调用 / 智能体', + 'items': ['bfcl_v3', 'general_fc', 'tau2_bench'], + }, +] + + +def _meta() -> dict: + all_benchmarks = sorted( + set(run_module.ALL_MULTI_RUN) + | set(run_module.ALL_SINGLE_RUN) + | set(run_module.ALL_AGENT) + ) + categorized = {b for cat in BENCHMARK_CATEGORIES for b in cat['items']} + other = sorted(set(all_benchmarks) - categorized) + categories = [dict(cat) for cat in BENCHMARK_CATEGORIES] + if other: + categories.append({'id': 'other', 'name': '其他', 'items': other}) + + suites = {} + for name, cfg in run_module.SUITES.items(): + suites[name] = { + 'multi': list(cfg['multi']), + 'single': list(cfg['single']), + 'agent': list(cfg['agent']), + 'all': list(cfg['multi']) + list(cfg['single']) + list(cfg['agent']), + } + return { + 'suites': suites, + 'benchmarks': all_benchmarks, + 'categories': categories, + 'multi_run': run_module.MULTI_RUN_CONFIG, + 'defaults': { + 'model': run_module.DEFAULT_MODEL, + 'api_url': run_module.DEFAULT_API_URL, + 'api_key': 'EMPTY', + 'dataset_dir': run_module.DEFAULT_DATASET_DIR, + 'output_dir': run_module.DEFAULT_OUTPUT_DIR, + 'config': run_module.DEFAULT_CONFIG, + 'tokenizer_path': run_module.DEFAULT_TOKENIZER_PATH, + 'seed': run_module.DEFAULT_SEED, + 'batch_size': run_module.DEFAULT_BATCH_SIZE, + 'thinking': run_module.DEFAULT_ENABLE_THINKING, + 'judge_model': run_module.DEFAULT_JUDGE_MODEL, + 'judge_api_url': run_module.DEFAULT_JUDGE_API_URL, + 'judge_max_tokens': run_module.DEFAULT_JUDGE_MAX_TOKENS, + 'suite': 'official', + }, + 'project_root': str(PROJECT_ROOT), + 'run_script': str(RUN_SCRIPT), + } + + +def build_command(req: LaunchRequest) -> List[str]: + cmd = [ + sys.executable, + str(RUN_SCRIPT), + '--model', req.model, + '--api-url', req.api_url, + '--seed', str(req.seed), + '--batch-size', str(req.batch_size), + ] + # Keep API key in env only; avoid requiring a custom --api-key CLI flag in run.py. + + if req.thinking: + cmd.append('--thinking') + else: + cmd.append('--no-thinking') + + if req.selection_mode == 'datasets': + if not req.datasets: + raise HTTPException(status_code=400, detail='请至少选择一个 benchmark') + cmd.extend(['--datasets', ','.join(req.datasets)]) + else: + if req.suite not in run_module.SUITES: + raise HTTPException(status_code=400, detail=f'未知 suite: {req.suite}') + cmd.extend(['--suite', req.suite]) + + if req.exclude: + cmd.extend(['--exclude', ','.join(req.exclude)]) + + if req.folder_name: + cmd.extend(['--folder-name', req.folder_name]) + if req.limit is not None and str(req.limit).strip() != '': + cmd.extend(['--limit', str(req.limit)]) + if req.thinking_max_tokens_scale != 1.0: + cmd.extend(['--thinking-max-tokens-scale', str(req.thinking_max_tokens_scale)]) + if req.max_tokens_add: + cmd.extend(['--max-tokens-add', str(req.max_tokens_add)]) + if req.dataset_dir: + cmd.extend(['--dataset-dir', req.dataset_dir]) + if req.output_dir: + cmd.extend(['--output-dir', req.output_dir]) + if req.config: + cmd.extend(['--config', req.config]) + if req.tokenizer_path: + cmd.extend(['--tokenizer-path', req.tokenizer_path]) + if req.judge_model: + cmd.extend(['--judge-model', req.judge_model]) + if req.judge_api_url: + cmd.extend(['--judge-api-url', req.judge_api_url]) + if req.judge_api_key: + cmd.extend(['--judge-api-key', req.judge_api_key]) + if req.judge_max_tokens is not None: + cmd.extend(['--judge-max-tokens', str(req.judge_max_tokens)]) + if not req.write_summary: + cmd.append('--no-summary') + + return cmd + + +async def _pump_stdout(job: JobRecord) -> None: + assert job._proc is not None and job._log_fp is not None + assert job._proc.stdout is not None + try: + while True: + line = await job._proc.stdout.readline() + if not line: + break + text = line.decode('utf-8', errors='replace') + if job._log_fp and not job._log_fp.closed: + job._log_fp.write(text) + job._log_fp.flush() + + return_code = await job._proc.wait() + job.return_code = return_code + job.finished_at = datetime.now(timezone.utc).isoformat() + if job.status in ('stopping', 'stopped'): + job.status = 'stopped' + job.error = job.error or 'Stopped by user' + elif job.status == 'running': + if return_code == 0: + job.status = 'completed' + else: + job.status = 'failed' + job.error = f'Process exited with code {return_code}' + finally: + job.pid = None + job._proc = None + if job._log_fp and not job._log_fp.closed: + try: + job._log_fp.close() + except Exception: + pass + job._log_fp = None + job.save() + + global _ACTIVE_JOB_ID + if _ACTIVE_JOB_ID == job.id: + _ACTIVE_JOB_ID = None + + +async def start_job(req: LaunchRequest) -> JobRecord: + global _ACTIVE_JOB_ID + + async with _LOCK: + if _ACTIVE_JOB_ID and _ACTIVE_JOB_ID in JOBS and JOBS[_ACTIVE_JOB_ID].status == 'running': + raise HTTPException(status_code=409, detail=f'已有任务在运行: {_ACTIVE_JOB_ID}') + + cmd = build_command(req) + job_id = datetime.now().strftime('%Y%m%d_%H%M%S') + '_' + uuid.uuid4().hex[:8] + job = JobRecord(job_id, req.model_dump(), cmd) + job.log_path.write_text('', encoding='utf-8') + + env = os.environ.copy() + env['PYTHONUNBUFFERED'] = '1' + if req.api_key and req.api_key != 'EMPTY': + env['OPENAI_API_KEY'] = req.api_key + + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + cwd=str(PROJECT_ROOT), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.STDOUT, + env=env, + start_new_session=True, + ) + except Exception as e: + job.status = 'failed' + job.error = str(e) + job.finished_at = datetime.now(timezone.utc).isoformat() + job.save() + JOBS[job.id] = job + raise HTTPException(status_code=500, detail=f'启动失败: {e}') from e + + job._proc = proc + job.pid = proc.pid + job.status = 'running' + job.started_at = datetime.now(timezone.utc).isoformat() + job._log_fp = open(job.log_path, 'a', encoding='utf-8') + header = ( + f'# job {job.id}\n' + f'# cwd: {PROJECT_ROOT}\n' + f'# cmd: {" ".join(cmd)}\n' + f'# started: {job.started_at}\n' + f'{"=" * 60}\n' + ) + job._log_fp.write(header) + job._log_fp.flush() + job.save() + JOBS[job.id] = job + _ACTIVE_JOB_ID = job.id + asyncio.create_task(_pump_stdout(job)) + return job + + +async def stop_job(job_id: str) -> JobRecord: + job = JOBS.get(job_id) + if not job: + raise HTTPException(status_code=404, detail='任务不存在') + if job.status != 'running' or job._proc is None: + raise HTTPException(status_code=400, detail='任务未在运行') + + proc = job._proc + job.status = 'stopping' + job.error = 'Stopped by user' + if job._log_fp and not job._log_fp.closed: + try: + job._log_fp.write('\n# stop requested by user\n') + job._log_fp.flush() + except Exception: + pass + job.save() + + try: + os.killpg(proc.pid, signal.SIGTERM) + except ProcessLookupError: + pass + except Exception: + proc.terminate() + + try: + await asyncio.wait_for(proc.wait(), timeout=15) + except asyncio.TimeoutError: + try: + os.killpg(proc.pid, signal.SIGKILL) + except Exception: + try: + proc.kill() + except Exception: + pass + + # Final status is finalized by _pump_stdout; wait briefly for it. + for _ in range(20): + if job.status in ('stopped', 'failed', 'completed'): + break + await asyncio.sleep(0.1) + if job.status == 'stopping': + job.status = 'stopped' + job.finished_at = datetime.now(timezone.utc).isoformat() + job.save() + return job + + +# --------------------------------------------------------------------------- +# API routes +# --------------------------------------------------------------------------- + +@app.get('/api/health') +async def health(): + return {'ok': True, 'project_root': str(PROJECT_ROOT)} + + +@app.get('/api/meta') +async def meta(): + return _meta() + + +@app.get('/api/jobs') +async def list_jobs(limit: int = 50): + items = sorted(JOBS.values(), key=lambda j: j.created_at, reverse=True)[:limit] + return {'jobs': [j.to_dict() for j in items], 'active_job_id': _ACTIVE_JOB_ID} + + +@app.get('/api/jobs/{job_id}') +async def get_job(job_id: str): + job = JOBS.get(job_id) + if not job: + raise HTTPException(status_code=404, detail='任务不存在') + return job.to_dict() + + +@app.post('/api/jobs') +async def create_job(req: LaunchRequest): + job = await start_job(req) + return job.to_dict() + + +@app.post('/api/jobs/{job_id}/stop') +async def api_stop_job(job_id: str): + job = await stop_job(job_id) + return job.to_dict() + + +@app.get('/api/jobs/{job_id}/logs') +async def get_logs(job_id: str, offset: int = 0): + job = JOBS.get(job_id) + if not job: + raise HTTPException(status_code=404, detail='任务不存在') + if not job.log_path.exists(): + return {'content': '', 'offset': 0, 'next_offset': 0, 'done': job.status not in ('queued', 'running')} + data = job.log_path.read_bytes() + if offset < 0: + offset = 0 + if offset > len(data): + offset = len(data) + chunk = data[offset:].decode('utf-8', errors='replace') + return { + 'content': chunk, + 'offset': offset, + 'next_offset': len(data), + 'done': job.status not in ('queued', 'running'), + 'status': job.status, + } + + +@app.get('/api/jobs/{job_id}/stream') +async def stream_logs(job_id: str, offset: int = 0): + job = JOBS.get(job_id) + if not job: + raise HTTPException(status_code=404, detail='任务不存在') + + async def event_gen(): + pos = max(0, offset) + while True: + if job.log_path.exists(): + data = job.log_path.read_bytes() + if pos < len(data): + chunk = data[pos:].decode('utf-8', errors='replace') + pos = len(data) + payload = json.dumps({'type': 'log', 'content': chunk, 'offset': pos}, ensure_ascii=False) + yield f'data: {payload}\n\n' + + status_payload = json.dumps({ + 'type': 'status', + 'status': job.status, + 'return_code': job.return_code, + 'offset': pos, + }, ensure_ascii=False) + yield f'data: {status_payload}\n\n' + + if job.status not in ('queued', 'running'): + done_payload = json.dumps({'type': 'done', 'status': job.status, 'offset': pos}, ensure_ascii=False) + yield f'data: {done_payload}\n\n' + break + await asyncio.sleep(0.8) + + return StreamingResponse(event_gen(), media_type='text/event-stream') + + +@app.get('/api/results/overview') +async def results_overview(output_dir: Optional[str] = None): + root = Path(output_dir) if output_dir else DEFAULT_OUTPUT_DIR + return results_scan.scan_output_dir(root) + + +@app.get('/api/results/compare') +async def results_compare( + models: Optional[str] = None, + benchmarks: Optional[str] = None, + output_dir: Optional[str] = None, +): + root = Path(output_dir) if output_dir else DEFAULT_OUTPUT_DIR + folder_list = [x.strip() for x in (models or '').split(',') if x.strip()] or None + bench_list = [x.strip() for x in (benchmarks or '').split(',') if x.strip()] or None + return results_scan.compare_models(root, folders=folder_list, benchmarks=bench_list) + + +@app.get('/') +async def index(): + return FileResponse(STATIC_DIR / 'index.html') + + +@app.get('/results') +async def results_page(): + return FileResponse(STATIC_DIR / 'results.html') + + +app.mount('/static', StaticFiles(directory=str(STATIC_DIR)), name='static') + + +if __name__ == '__main__': + import uvicorn + host = os.environ.get('WEBUI_HOST', '0.0.0.0') + port = int(os.environ.get('WEBUI_PORT', '7860')) + uvicorn.run('server:app', host=host, port=port, reload=False) diff --git a/webui/start.sh b/webui/start.sh new file mode 100755 index 0000000..5454d27 --- /dev/null +++ b/webui/start.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$DIR" + +HOST="${WEBUI_HOST:-0.0.0.0}" +PORT="${WEBUI_PORT:-7860}" + +echo "EvalStone Launch Panel" +echo " project : $(dirname "$DIR")" +echo " listen : http://${HOST}:${PORT}" +echo + +exec python3 -m uvicorn server:app --host "$HOST" --port "$PORT" diff --git a/webui/static/app.js b/webui/static/app.js new file mode 100644 index 0000000..10bc99c --- /dev/null +++ b/webui/static/app.js @@ -0,0 +1,372 @@ +(() => { + const state = { + meta: null, + selectionMode: 'suite', + suite: 'official', + activeJobId: null, + pollTimer: null, + logOffset: 0, + }; + + const $ = (id) => document.getElementById(id); + const form = $('launchForm'); + const formMsg = $('formMsg'); + const logView = $('logView'); + const jobList = $('jobList'); + + function setMsg(text, type = '') { + formMsg.textContent = text || ''; + formMsg.className = `msg ${type}`.trim(); + } + + async function api(path, options) { + const res = await fetch(path, { + headers: { 'Content-Type': 'application/json' }, + ...options, + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) { + const detail = data.detail || res.statusText || 'request failed'; + throw new Error(typeof detail === 'string' ? detail : JSON.stringify(detail)); + } + return data; + } + + function fillDefaults(meta) { + const d = meta.defaults; + $('model').value = d.model || ''; + $('api_url').value = d.api_url || ''; + $('api_key').value = d.api_key || 'EMPTY'; + $('seed').value = d.seed; + $('batch_size').value = d.batch_size; + $('thinking').checked = !!d.thinking; + $('dataset_dir').value = d.dataset_dir || ''; + $('output_dir').value = d.output_dir || ''; + $('config').value = d.config || ''; + $('tokenizer_path').value = d.tokenizer_path || ''; + $('judge_model').value = d.judge_model || ''; + $('judge_api_url').value = d.judge_api_url || ''; + $('judge_max_tokens').value = d.judge_max_tokens || ''; + state.suite = d.suite || 'official'; + } + + function renderSuites() { + const box = $('suiteList'); + box.innerHTML = ''; + const suites = state.meta.suites; + Object.keys(suites).forEach((name) => { + const info = suites[name]; + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = `suite-card${state.suite === name ? ' active' : ''}`; + btn.innerHTML = `${name}${info.all.length} benchmarks
${info.all.slice(0, 6).join(', ')}${info.all.length > 6 ? '…' : ''}
`; + btn.addEventListener('click', () => { + state.suite = name; + renderSuites(); + }); + box.appendChild(btn); + }); + } + + function renderBenchmarks() { + const box = $('benchmarkList'); + box.innerHTML = ''; + const multi = new Set(Object.keys(state.meta.multi_run || {})); + const categories = state.meta.categories || [ + { id: 'all', name: '全部', items: state.meta.benchmarks || [] }, + ]; + + categories.forEach((cat) => { + const section = document.createElement('section'); + section.className = `bench-category cat-${cat.id}`; + section.dataset.category = cat.id; + + const head = document.createElement('div'); + head.className = 'bench-cat-head'; + head.innerHTML = ` +
+ ${cat.name} + ${cat.items.length} +
+
+ + +
+ `; + head.querySelector('.cat-select').addEventListener('click', () => { + section.querySelectorAll('input[type="checkbox"]').forEach((el) => { el.checked = true; }); + updateSelectedCount(); + }); + head.querySelector('.cat-clear').addEventListener('click', () => { + section.querySelectorAll('input[type="checkbox"]').forEach((el) => { el.checked = false; }); + updateSelectedCount(); + }); + section.appendChild(head); + + const grid = document.createElement('div'); + grid.className = 'bench-cat-grid'; + cat.items.forEach((name) => { + const label = document.createElement('label'); + const isMulti = multi.has(name); + label.className = `bench-item${isMulti ? ' multi' : ''}`; + label.title = isMulti ? `multi-run x${state.meta.multi_run[name]}` : cat.name; + label.innerHTML = `${name}`; + if (isMulti) { + const tag = document.createElement('em'); + tag.className = 'run-tag'; + tag.textContent = `×${state.meta.multi_run[name]}`; + label.appendChild(tag); + } + label.querySelector('input').addEventListener('change', updateSelectedCount); + grid.appendChild(label); + }); + section.appendChild(grid); + box.appendChild(section); + }); + updateSelectedCount(); + } + + function updateSelectedCount() { + const n = [...document.querySelectorAll('#benchmarkList input:checked')].length; + $('selectedCount').textContent = `已选 ${n}`; + } + + function setMode(mode) { + state.selectionMode = mode; + document.querySelectorAll('.mode-tabs .tab').forEach((el) => { + el.classList.toggle('active', el.dataset.mode === mode); + }); + $('suitePane').classList.toggle('hidden', mode !== 'suite'); + $('datasetsPane').classList.toggle('hidden', mode !== 'datasets'); + } + + function collectPayload() { + const datasets = [...document.querySelectorAll('#benchmarkList input:checked')].map((el) => el.value); + const excludeRaw = $('exclude').value.trim(); + const exclude = excludeRaw + ? excludeRaw.split(',').map((s) => s.trim()).filter(Boolean) + : []; + + const payload = { + model: $('model').value.trim(), + api_url: $('api_url').value.trim(), + api_key: $('api_key').value.trim() || 'EMPTY', + thinking: $('thinking').checked, + selection_mode: state.selectionMode, + suite: state.suite, + datasets, + exclude, + folder_name: $('folder_name').value.trim() || null, + limit: $('limit').value.trim() || null, + seed: Number($('seed').value || 42), + batch_size: Number($('batch_size').value || 4), + thinking_max_tokens_scale: Number($('thinking_max_tokens_scale').value || 1), + max_tokens_add: Number($('max_tokens_add').value || 0), + dataset_dir: $('dataset_dir').value.trim() || null, + output_dir: $('output_dir').value.trim() || null, + config: $('config').value.trim() || null, + tokenizer_path: $('tokenizer_path').value.trim() || null, + judge_model: $('judge_model').value.trim() || null, + judge_api_url: $('judge_api_url').value.trim() || null, + judge_api_key: $('judge_api_key').value.trim() || null, + judge_max_tokens: $('judge_max_tokens').value + ? Number($('judge_max_tokens').value) + : null, + write_summary: $('write_summary').value === 'true', + }; + return payload; + } + + function statusClass(status) { + return status || 'idle'; + } + + function renderActive(job) { + const card = $('activeCard'); + if (!job) { + card.className = 'active-card idle'; + $('activeStatus').className = 'badge'; + $('activeStatus').textContent = 'idle'; + $('activeJobId').textContent = '—'; + $('activeCmd').textContent = '尚未启动任务'; + $('stopBtn').disabled = true; + return; + } + card.className = `active-card ${statusClass(job.status)}`; + $('activeStatus').className = `badge ${statusClass(job.status)}`; + $('activeStatus').textContent = job.status; + $('activeJobId').textContent = job.id; + $('activeCmd').textContent = (job.command || []).join(' '); + $('stopBtn').disabled = job.status !== 'running'; + state.activeJobId = job.id; + } + + function renderJobs(jobs, activeId) { + jobList.innerHTML = ''; + if (!jobs.length) { + jobList.innerHTML = '
暂无历史任务
'; + return; + } + jobs.forEach((job) => { + const btn = document.createElement('button'); + btn.type = 'button'; + btn.className = 'job-item'; + const model = job.payload?.model || '-'; + const thinking = job.payload?.thinking ? 'thinking' : 'no-thinking'; + btn.innerHTML = ` +
+ ${job.id} + ${job.status} +
+ ${model} · ${thinking} + `; + btn.addEventListener('click', () => followJob(job.id, true)); + if (job.id === activeId) btn.style.borderColor = 'rgba(214,162,74,0.75)'; + jobList.appendChild(btn); + }); + } + + async function refreshJobs() { + const data = await api('/api/jobs'); + renderJobs(data.jobs || [], data.active_job_id); + if (data.active_job_id) { + const active = (data.jobs || []).find((j) => j.id === data.active_job_id); + if (active) renderActive(active); + } else if (state.activeJobId) { + const current = (data.jobs || []).find((j) => j.id === state.activeJobId); + if (current) renderActive(current); + } + return data; + } + + async function pullLogs(reset = false) { + if (!state.activeJobId) return; + if (reset) { + state.logOffset = 0; + logView.textContent = ''; + } + const data = await api(`/api/jobs/${state.activeJobId}/logs?offset=${state.logOffset}`); + if (data.content) { + logView.textContent += data.content; + state.logOffset = data.next_offset; + if ($('autoScroll').checked) logView.scrollTop = logView.scrollHeight; + } + if (data.status) { + $('activeStatus').className = `badge ${statusClass(data.status)}`; + $('activeStatus').textContent = data.status; + $('activeCard').className = `active-card ${statusClass(data.status)}`; + $('stopBtn').disabled = data.status !== 'running'; + } + if (data.done) stopPolling(); + } + + function stopPolling() { + if (state.pollTimer) { + clearInterval(state.pollTimer); + state.pollTimer = null; + } + } + + function startPolling() { + stopPolling(); + state.pollTimer = setInterval(async () => { + try { + await pullLogs(false); + await refreshJobs(); + } catch (e) { + console.error(e); + } + }, 1200); + } + + async function followJob(jobId, resetLog = false) { + const job = await api(`/api/jobs/${jobId}`); + renderActive(job); + await pullLogs(resetLog); + if (job.status === 'running' || job.status === 'queued') startPolling(); + else stopPolling(); + } + + async function init() { + try { + await api('/api/health'); + $('healthDot').className = 'dot ok'; + $('healthText').textContent = 'server online'; + } catch (e) { + $('healthDot').className = 'dot bad'; + $('healthText').textContent = 'server offline'; + setMsg(e.message, 'error'); + return; + } + + state.meta = await api('/api/meta'); + fillDefaults(state.meta); + renderSuites(); + renderBenchmarks(); + setMode('suite'); + + const jobs = await refreshJobs(); + if (jobs.active_job_id) { + await followJob(jobs.active_job_id, true); + } else if (jobs.jobs?.[0]) { + await followJob(jobs.jobs[0].id, true); + } + } + + document.querySelectorAll('.mode-tabs .tab').forEach((el) => { + el.addEventListener('click', () => setMode(el.dataset.mode)); + }); + $('selectAllBtn').addEventListener('click', () => { + document.querySelectorAll('#benchmarkList input').forEach((el) => { el.checked = true; }); + updateSelectedCount(); + }); + $('clearAllBtn').addEventListener('click', () => { + document.querySelectorAll('#benchmarkList input').forEach((el) => { el.checked = false; }); + updateSelectedCount(); + }); + $('refreshJobsBtn').addEventListener('click', () => refreshJobs().catch((e) => setMsg(e.message, 'error'))); + + form.addEventListener('submit', async (e) => { + e.preventDefault(); + setMsg(''); + const payload = collectPayload(); + if (payload.selection_mode === 'datasets' && !payload.datasets.length) { + setMsg('请至少选择一个 benchmark', 'error'); + return; + } + $('launchBtn').disabled = true; + try { + const job = await api('/api/jobs', { + method: 'POST', + body: JSON.stringify(payload), + }); + setMsg(`任务已启动: ${job.id}`, 'ok'); + renderActive(job); + state.logOffset = 0; + logView.textContent = ''; + await refreshJobs(); + startPolling(); + await pullLogs(true); + } catch (err) { + setMsg(err.message, 'error'); + } finally { + $('launchBtn').disabled = false; + } + }); + + $('stopBtn').addEventListener('click', async () => { + if (!state.activeJobId) return; + if (!confirm(`确认停止任务 ${state.activeJobId}?`)) return; + try { + const job = await api(`/api/jobs/${state.activeJobId}/stop`, { method: 'POST' }); + renderActive(job); + setMsg('任务已停止', 'ok'); + await refreshJobs(); + await pullLogs(false); + } catch (err) { + setMsg(err.message, 'error'); + } + }); + + init(); +})(); diff --git a/webui/static/index.html b/webui/static/index.html new file mode 100644 index 0000000..42886c0 --- /dev/null +++ b/webui/static/index.html @@ -0,0 +1,200 @@ + + + + + + EvalStone Launch + + + + + + +
+
+
+ +
+

EvalStone Launch

+

本地评测启动台 · 基于 bash/run.py

+
+
+
+ +
+ + connecting… +
+
+
+ +
+
+
+
+

模型接入

+
+ + + + +
+
+ +
+

Thinking

+
+ + + +
+
+ +
+

评测范围

+
+ + +
+ +
+
+ +
+ + +
+ +
+ 高级参数 +
+ + + + + + + + + + + + +
+
+ +
+ + + +
+
+
+ +
+
+

运行状态

+ +
+ +
+
+ idle + +
+
尚未启动任务
+
+ +
+

实时日志

+ +
+

+
+        
+

历史任务

+
+
+
+
+
+ + + diff --git a/webui/static/results.html b/webui/static/results.html new file mode 100644 index 0000000..430449b --- /dev/null +++ b/webui/static/results.html @@ -0,0 +1,122 @@ + + + + + + EvalStone Results + + + + + + + +
+
+
+ +
+

EvalStone Results

+

多模型同基准对比 · 扫描 output/

+
+
+ +
+ +
+
+
+

数据源

+ +
+ + +
+

+
+ +
+

选择模型(folder)

+
+ + +
+
+
+ +
+

选择 Benchmark

+
+ + +
+
+
+ +
+ +
+
+ +
+
+
+

得分走势

+ 横轴 = benchmark,纵轴 = score +
+
+ +
+
+ +
+
+

柱状对比

+ 同 benchmark 下各模型得分 +
+
+ +
+
+ +
+
+

雷达图

+ 能力轮廓(需 ≥3 个共同 benchmark) +
+
+ +
+
+ +
+
+

得分表

+ 平均值(跨 seed / multi-run) +
+
+ + + +
+
+
+ +
+
+

各 Benchmark 排名

+
+
+
+
+
+
+ + + diff --git a/webui/static/results.js b/webui/static/results.js new file mode 100644 index 0000000..3fdbec4 --- /dev/null +++ b/webui/static/results.js @@ -0,0 +1,342 @@ +(() => { + const state = { + overview: null, + compare: null, + charts: { line: null, bar: null, radar: null }, + }; + + const $ = (id) => document.getElementById(id); + const COLORS = [ + '#d6a24a', '#6fbf8a', '#6aa8d6', '#d86a5b', + '#8b7ec8', '#e0c36a', '#5ec4b0', '#c98a6a', + ]; + + function setMsg(text, type = '') { + const el = $('scanMsg'); + el.textContent = text || ''; + el.className = `msg ${type}`.trim(); + } + + async function api(path) { + const res = await fetch(path); + const data = await res.json().catch(() => ({})); + if (!res.ok) { + throw new Error(data.detail || res.statusText || 'request failed'); + } + return data; + } + + function selectedValues(containerId) { + return [...document.querySelectorAll(`#${containerId} input:checked`)].map((el) => el.value); + } + + function renderModelList(models) { + const box = $('modelList'); + box.innerHTML = ''; + if (!models.length) { + box.innerHTML = '
未发现模型结果目录
'; + return; + } + models.forEach((m, i) => { + const label = document.createElement('label'); + label.className = 'pick-item'; + label.innerHTML = ` + + + ${m.folder} + ${m.model_name} · ${m.n_benchmarks} benches + + + `; + box.appendChild(label); + }); + } + + function renderBenchList(categories, benchmarks) { + const box = $('benchList'); + box.innerHTML = ''; + if (!benchmarks.length) { + box.innerHTML = '
暂无 benchmark 结果
'; + return; + } + const cats = categories && categories.length + ? categories + : [{ id: 'all', name: '全部', items: benchmarks }]; + + cats.forEach((cat) => { + const section = document.createElement('div'); + section.className = `pick-cat cat-${cat.id}`; + section.innerHTML = ` +
+ ${cat.name} + ${cat.items.length} + + +
+ `; + const grid = document.createElement('div'); + grid.className = 'pick-grid'; + cat.items.forEach((name) => { + const label = document.createElement('label'); + label.className = 'pick-chip'; + label.innerHTML = `${name}`; + grid.appendChild(label); + }); + section.querySelector('.cat-all').addEventListener('click', () => { + grid.querySelectorAll('input').forEach((el) => { el.checked = true; }); + }); + section.querySelector('.cat-none').addEventListener('click', () => { + grid.querySelectorAll('input').forEach((el) => { el.checked = false; }); + }); + section.appendChild(grid); + box.appendChild(section); + }); + } + + function destroyCharts() { + Object.keys(state.charts).forEach((k) => { + if (state.charts[k]) { + state.charts[k].destroy(); + state.charts[k] = null; + } + }); + } + + function chartDatasets(series, type) { + return series.map((s, i) => { + const color = COLORS[i % COLORS.length]; + const base = { + label: s.display, + data: s.scores.map((v) => (v == null ? null : Number((v * 100).toFixed(2)))), + borderColor: color, + backgroundColor: type === 'radar' ? color + '33' : color + 'cc', + tension: 0.25, + spanGaps: false, + pointRadius: 4, + pointHoverRadius: 6, + }; + if (type === 'bar') { + return { ...base, borderWidth: 0, borderRadius: 4 }; + } + if (type === 'radar') { + return { ...base, fill: true, borderWidth: 2 }; + } + return { ...base, fill: false, borderWidth: 2 }; + }); + } + + function renderCharts(compare) { + destroyCharts(); + const labels = compare.benchmarks; + const common = { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { + labels: { color: '#c9d4cb', boxWidth: 12, font: { family: 'IBM Plex Sans' } }, + }, + tooltip: { + callbacks: { + label(ctx) { + const v = ctx.parsed.y ?? ctx.parsed.r; + return `${ctx.dataset.label}: ${v == null ? '—' : v.toFixed(2) + '%'}`; + }, + }, + }, + }, + scales: {}, + }; + + state.charts.line = new Chart($('lineChart'), { + type: 'line', + data: { labels, datasets: chartDatasets(compare.series, 'line') }, + options: { + ...common, + scales: { + x: { + ticks: { color: '#92a197', maxRotation: 45, minRotation: 0 }, + grid: { color: 'rgba(51,64,56,0.6)' }, + }, + y: { + min: 0, + max: 100, + ticks: { + color: '#92a197', + callback: (v) => v + '%', + }, + grid: { color: 'rgba(51,64,56,0.6)' }, + title: { display: true, text: 'Score (%)', color: '#92a197' }, + }, + }, + }, + }); + + state.charts.bar = new Chart($('barChart'), { + type: 'bar', + data: { labels, datasets: chartDatasets(compare.series, 'bar') }, + options: { + ...common, + scales: { + x: { + ticks: { color: '#92a197', maxRotation: 45 }, + grid: { display: false }, + }, + y: { + min: 0, + max: 100, + ticks: { color: '#92a197', callback: (v) => v + '%' }, + grid: { color: 'rgba(51,64,56,0.6)' }, + }, + }, + }, + }); + + const radarLabels = labels; + const canRadar = radarLabels.length >= 3 && compare.series.length > 0; + const radarCanvas = $('radarChart'); + if (!canRadar) { + radarCanvas.parentElement.classList.add('empty'); + radarCanvas.parentElement.dataset.empty = '共同 benchmark 不足 3 个,暂不绘制雷达图'; + } else { + radarCanvas.parentElement.classList.remove('empty'); + delete radarCanvas.parentElement.dataset.empty; + state.charts.radar = new Chart(radarCanvas, { + type: 'radar', + data: { labels: radarLabels, datasets: chartDatasets(compare.series, 'radar') }, + options: { + responsive: true, + maintainAspectRatio: false, + plugins: { + legend: { labels: { color: '#c9d4cb', boxWidth: 12 } }, + }, + scales: { + r: { + min: 0, + max: 100, + ticks: { color: '#92a197', backdropColor: 'transparent', stepSize: 20 }, + grid: { color: 'rgba(51,64,56,0.7)' }, + angleLines: { color: 'rgba(51,64,56,0.7)' }, + pointLabels: { color: '#c9d4cb', font: { size: 11 } }, + }, + }, + }, + }); + } + } + + function renderTable(compare) { + const thead = $('scoreTable').querySelector('thead'); + const tbody = $('scoreTable').querySelector('tbody'); + thead.innerHTML = ''; + tbody.innerHTML = ''; + + const head = document.createElement('tr'); + head.innerHTML = `Benchmark${compare.series.map((s) => `${s.display}`).join('')}最佳`; + thead.appendChild(head); + + compare.benchmarks.forEach((b, i) => { + const tr = document.createElement('tr'); + let best = -1; + let bestFolder = '—'; + compare.series.forEach((s) => { + const v = s.scores[i]; + if (v != null && v > best) { + best = v; + bestFolder = s.display; + } + }); + const cells = compare.series.map((s) => { + const v = s.scores[i]; + if (v == null) return '—'; + const pct = (v * 100).toFixed(2); + const isBest = v === best && best >= 0; + return `${pct}%`; + }).join(''); + tr.innerHTML = `${b}${cells}${bestFolder}`; + tbody.appendChild(tr); + }); + } + + function renderRanking(compare) { + const box = $('rankList'); + box.innerHTML = ''; + compare.ranking.forEach((item) => { + const card = document.createElement('div'); + card.className = 'rank-card'; + const rows = item.rows.length + ? item.rows.map((r, idx) => ` +
+ #${idx + 1} + ${r.folder} + ${(r.score * 100).toFixed(2)}% +
`).join('') + : '
无数据
'; + card.innerHTML = `${item.benchmark}${rows}`; + box.appendChild(card); + }); + } + + async function loadOverview() { + const dir = $('outputDir').value.trim(); + const qs = dir ? `?output_dir=${encodeURIComponent(dir)}` : ''; + setMsg('扫描中…'); + try { + const data = await api(`/api/results/overview${qs}`); + state.overview = data; + if (!$('outputDir').value) $('outputDir').value = data.output_dir || ''; + $('scanMeta').textContent = `${data.models.length} 个模型 · ${data.benchmarks.length} 个 benchmark · ${data.output_dir}`; + renderModelList(data.models); + renderBenchList(data.categories, data.benchmarks); + setMsg('扫描完成', 'ok'); + await runCompare(); + } catch (e) { + setMsg(e.message, 'error'); + } + } + + async function runCompare() { + const models = selectedValues('modelList'); + const benchmarks = selectedValues('benchList'); + if (!models.length) { + setMsg('请至少选择一个模型', 'error'); + return; + } + if (!benchmarks.length) { + setMsg('请至少选择一个 benchmark', 'error'); + return; + } + const dir = $('outputDir').value.trim(); + const params = new URLSearchParams({ + models: models.join(','), + benchmarks: benchmarks.join(','), + }); + if (dir) params.set('output_dir', dir); + const data = await api(`/api/results/compare?${params}`); + state.compare = data; + renderCharts(data); + renderTable(data); + renderRanking(data); + setMsg(`已对比 ${data.series.length} 模型 × ${data.benchmarks.length} benches`, 'ok'); + } + + $('reloadBtn').addEventListener('click', () => loadOverview()); + $('compareBtn').addEventListener('click', () => runCompare().catch((e) => setMsg(e.message, 'error'))); + $('modelAllBtn').addEventListener('click', () => { + document.querySelectorAll('#modelList input').forEach((el) => { el.checked = true; }); + }); + $('modelClearBtn').addEventListener('click', () => { + document.querySelectorAll('#modelList input').forEach((el) => { el.checked = false; }); + }); + $('benchAllBtn').addEventListener('click', () => { + document.querySelectorAll('#benchList input').forEach((el) => { el.checked = true; }); + }); + $('benchClearBtn').addEventListener('click', () => { + document.querySelectorAll('#benchList input').forEach((el) => { el.checked = false; }); + }); + + // bootstrap defaults from meta + api('/api/meta').then((meta) => { + $('outputDir').value = meta.defaults?.output_dir || ''; + return loadOverview(); + }).catch(() => loadOverview()); +})(); diff --git a/webui/static/styles.css b/webui/static/styles.css new file mode 100644 index 0000000..7cd9159 --- /dev/null +++ b/webui/static/styles.css @@ -0,0 +1,608 @@ +:root { + --bg0: #121714; + --bg1: #1a211c; + --bg2: #232c26; + --line: #334038; + --text: #e7eee8; + --muted: #92a197; + --accent: #d6a24a; + --accent-2: #6fbf8a; + --danger: #d86a5b; + --shadow: 0 18px 40px rgba(0, 0, 0, 0.28); + --radius: 14px; + --font: "IBM Plex Sans", "PingFang SC", "Noto Sans SC", sans-serif; + --mono: "IBM Plex Mono", ui-monospace, SFMono-Regular, Menlo, monospace; +} + +* { box-sizing: border-box; } +html, body { margin: 0; min-height: 100%; } +body { + font-family: var(--font); + color: var(--text); + background: + radial-gradient(1200px 600px at 10% -10%, rgba(214, 162, 74, 0.16), transparent 55%), + radial-gradient(900px 500px at 100% 0%, rgba(111, 191, 138, 0.12), transparent 50%), + linear-gradient(180deg, #0f1411 0%, var(--bg0) 40%, #101612 100%); +} + +code, pre, .mono { font-family: var(--mono); } + +.page { + max-width: 1440px; + margin: 0 auto; + padding: 28px 24px 40px; +} + +.top { + display: flex; + justify-content: space-between; + align-items: flex-end; + gap: 16px; + margin-bottom: 22px; +} + +.brand { display: flex; gap: 14px; align-items: center; } +.brand-mark { + width: 14px; height: 42px; border-radius: 999px; + background: linear-gradient(180deg, var(--accent), var(--accent-2)); + box-shadow: 0 0 24px rgba(214, 162, 74, 0.35); +} +.brand h1 { + margin: 0; + font-size: 28px; + letter-spacing: 0.02em; + font-weight: 700; +} +.brand p { + margin: 4px 0 0; + color: var(--muted); + font-size: 13px; +} +.brand code { + color: var(--accent); + background: rgba(214, 162, 74, 0.1); + padding: 1px 6px; + border-radius: 6px; +} + +.top-meta { + display: flex; align-items: center; gap: 8px; + color: var(--muted); font-size: 13px; +} +.dot { + width: 9px; height: 9px; border-radius: 50%; + background: #667; + box-shadow: 0 0 0 3px rgba(102, 102, 119, 0.2); +} +.dot.ok { background: var(--accent-2); box-shadow: 0 0 0 3px rgba(111, 191, 138, 0.2); } +.dot.bad { background: var(--danger); box-shadow: 0 0 0 3px rgba(216, 106, 91, 0.2); } + +.layout { + display: grid; + grid-template-columns: minmax(0, 1.15fr) minmax(340px, 0.85fr); + gap: 18px; + align-items: start; +} + +.panel { + background: linear-gradient(180deg, rgba(26, 33, 28, 0.96), rgba(18, 23, 20, 0.96)); + border: 1px solid var(--line); + border-radius: var(--radius); + box-shadow: var(--shadow); + padding: 20px; +} + +.block + .block { margin-top: 22px; } +.block h2, .side-head h2, .log-head h3, .jobs-head h3 { + margin: 0 0 12px; + font-size: 15px; + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; + color: #c7d2c9; +} + +label { + display: flex; + flex-direction: column; + gap: 6px; + font-size: 12px; + color: var(--muted); +} +label span { letter-spacing: 0.03em; } + +input, select, button, summary { + font: inherit; +} +input, select { + width: 100%; + background: var(--bg0); + color: var(--text); + border: 1px solid var(--line); + border-radius: 10px; + padding: 10px 12px; + outline: none; + transition: border-color .15s, box-shadow .15s; +} +input:focus, select:focus { + border-color: rgba(214, 162, 74, 0.7); + box-shadow: 0 0 0 3px rgba(214, 162, 74, 0.15); +} + +.grid-2 { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 12px; +} + +.thinking-row { + display: flex; + flex-wrap: wrap; + gap: 14px 18px; + align-items: end; +} +.thinking-row .inline { min-width: 140px; } + +.switch { + flex-direction: row; + align-items: center; + gap: 10px; + cursor: pointer; + user-select: none; +} +.switch input { display: none; } +.slider { + width: 42px; height: 24px; border-radius: 999px; + background: #2c3630; position: relative; transition: .2s; + border: 1px solid var(--line); +} +.slider::after { + content: ""; + position: absolute; top: 2px; left: 2px; + width: 18px; height: 18px; border-radius: 50%; + background: #c5d0c7; transition: .2s; +} +.switch input:checked + .slider { + background: rgba(214, 162, 74, 0.35); + border-color: rgba(214, 162, 74, 0.7); +} +.switch input:checked + .slider::after { + transform: translateX(18px); + background: var(--accent); +} +.switch-text { color: var(--text); font-size: 14px; } + +.mode-tabs { + display: inline-flex; + background: var(--bg0); + border: 1px solid var(--line); + border-radius: 999px; + padding: 4px; + margin-bottom: 14px; +} +.tab { + border: 0; + background: transparent; + color: var(--muted); + padding: 8px 14px; + border-radius: 999px; + cursor: pointer; +} +.tab.active { + background: rgba(214, 162, 74, 0.18); + color: var(--text); +} + +.suite-list { + display: grid; + grid-template-columns: 1fr 1fr; + gap: 10px; +} +.suite-card { + text-align: left; + border: 1px solid var(--line); + background: var(--bg0); + color: var(--text); + border-radius: 12px; + padding: 12px; + cursor: pointer; + transition: border-color .15s, transform .15s; +} +.suite-card:hover { transform: translateY(-1px); } +.suite-card.active { + border-color: rgba(214, 162, 74, 0.75); + background: rgba(214, 162, 74, 0.08); +} +.suite-card strong { display: block; margin-bottom: 4px; } +.suite-card small { color: var(--muted); line-height: 1.4; } + +.benchmark-list { + display: flex; + flex-direction: column; + gap: 12px; + max-height: 420px; + overflow: auto; + padding-right: 4px; +} +.bench-category { + border: 1px solid var(--line); + border-radius: 12px; + background: rgba(12, 16, 14, 0.55); + padding: 10px; +} +.bench-category.cat-math { border-left: 3px solid #6fbf8a; } +.bench-category.cat-code { border-left: 3px solid #6aa8d6; } +.bench-category.cat-science { border-left: 3px solid #c79a5a; } +.bench-category.cat-knowledge { border-left: 3px solid #9aa4b2; } +.bench-category.cat-long_context { border-left: 3px solid #8b7ec8; } +.bench-category.cat-tool_agent { border-left: 3px solid #d6a24a; } +.bench-category.cat-other { border-left: 3px solid #667; } + +.bench-cat-head { + display: flex; + justify-content: space-between; + align-items: center; + gap: 8px; + margin-bottom: 8px; +} +.bench-cat-title { + display: flex; + align-items: baseline; + gap: 8px; +} +.bench-cat-title strong { + font-size: 13px; + letter-spacing: 0.03em; +} +.bench-cat-actions { display: flex; gap: 4px; } +.bench-cat-grid { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 8px; +} +.bench-item { + display: flex; + align-items: center; + gap: 8px; + padding: 8px 10px; + border-radius: 10px; + border: 1px solid var(--line); + background: var(--bg0); + cursor: pointer; + font-size: 13px; +} +.bench-item input { width: auto; } +.bench-item.multi { border-color: rgba(111, 191, 138, 0.35); } +.bench-item .run-tag { + margin-left: auto; + font-style: normal; + font-size: 11px; + color: #9ee0b2; + opacity: 0.85; +} + +.bench-actions { + display: flex; gap: 8px; align-items: center; margin-bottom: 10px; +} +.muted { color: var(--muted); font-size: 12px; } +.mt { margin-top: 12px; } +.hidden { display: none !important; } + +.advanced { + border: 1px dashed var(--line); + border-radius: 12px; + padding: 10px 14px; +} +.advanced summary { + cursor: pointer; + color: var(--muted); + font-size: 13px; +} + +.actions { + display: flex; + flex-wrap: wrap; + gap: 10px; + align-items: center; + margin-top: 22px; +} +button { + border-radius: 10px; + border: 1px solid var(--line); + padding: 10px 16px; + cursor: pointer; + background: var(--bg2); + color: var(--text); +} +button:disabled { opacity: 0.45; cursor: not-allowed; } +button.primary { + background: linear-gradient(180deg, #e0b05a, #c48c2f); + border-color: #b9852d; + color: #1a1408; + font-weight: 600; +} +button.danger { + background: rgba(216, 106, 91, 0.15); + border-color: rgba(216, 106, 91, 0.45); + color: #ffc2ba; +} +button.ghost { + background: transparent; + padding: 6px 10px; + font-size: 12px; +} +.msg { color: var(--muted); font-size: 13px; } +.msg.error { color: var(--danger); } +.msg.ok { color: var(--accent-2); } + +.side-head, .log-head, .jobs-head { + display: flex; + justify-content: space-between; + align-items: center; +} +.active-card { + border: 1px solid var(--line); + border-radius: 12px; + padding: 12px; + background: var(--bg0); + margin-bottom: 14px; +} +.active-card.running { border-color: rgba(111, 191, 138, 0.55); } +.active-card.failed, .active-card.stopped { border-color: rgba(216, 106, 91, 0.45); } +.active-card.completed { border-color: rgba(214, 162, 74, 0.45); } +.active-title { + display: flex; gap: 10px; align-items: center; margin-bottom: 8px; +} +.badge { + display: inline-flex; + align-items: center; + padding: 2px 8px; + border-radius: 999px; + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.06em; + background: #2a332d; + color: var(--muted); +} +.badge.running { background: rgba(111, 191, 138, 0.18); color: #9ee0b2; } +.badge.completed { background: rgba(214, 162, 74, 0.18); color: #f0d08a; } +.badge.failed, .badge.stopped { background: rgba(216, 106, 91, 0.18); color: #ffb3a8; } + +.cmd, .log-view { + margin: 0; + white-space: pre-wrap; + word-break: break-word; + font-size: 12px; + line-height: 1.5; + color: #c9d4cb; +} +.cmd { color: var(--muted); max-height: 72px; overflow: auto; } +.log-view { + height: 360px; + overflow: auto; + background: #0c100e; + border: 1px solid var(--line); + border-radius: 12px; + padding: 12px; + margin: 8px 0 16px; +} + +.check-inline { + flex-direction: row; + align-items: center; + gap: 6px; + font-size: 12px; +} +.check-inline input { width: auto; } + +.job-list { + display: flex; + flex-direction: column; + gap: 8px; + max-height: 220px; + overflow: auto; +} +.job-item { + text-align: left; + width: 100%; + border: 1px solid var(--line); + background: var(--bg0); + border-radius: 10px; + padding: 10px 12px; + cursor: pointer; +} +.job-item:hover { border-color: rgba(214, 162, 74, 0.45); } +.job-item .row { + display: flex; justify-content: space-between; gap: 8px; margin-bottom: 4px; +} +.job-item small { color: var(--muted); } + +.top-right { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 10px; +} +.nav-tabs { + display: inline-flex; + gap: 4px; + padding: 4px; + border: 1px solid var(--line); + border-radius: 999px; + background: rgba(18, 23, 20, 0.7); +} +.nav-tabs a { + color: var(--muted); + text-decoration: none; + padding: 7px 14px; + border-radius: 999px; + font-size: 13px; +} +.nav-tabs a.active, +.nav-tabs a:hover { + color: var(--text); + background: rgba(214, 162, 74, 0.18); +} + +.results-layout { + display: grid; + grid-template-columns: minmax(280px, 0.85fr) minmax(0, 1.4fr); + gap: 18px; + align-items: start; +} +.filters-panel { position: sticky; top: 16px; } +.pick-list { + display: flex; + flex-direction: column; + gap: 8px; + max-height: 240px; + overflow: auto; +} +.pick-list.bench-pick { max-height: 320px; } +.pick-item { + flex-direction: row; + align-items: center; + gap: 10px; + padding: 10px 12px; + border: 1px solid var(--line); + border-radius: 10px; + background: var(--bg0); + cursor: pointer; +} +.pick-item input { width: auto; } +.pick-main { display: flex; flex-direction: column; gap: 2px; flex: 1; } +.pick-main strong { color: var(--text); font-size: 13px; } +.pick-main small { color: var(--muted); font-size: 11px; } +.swatch { + width: 10px; height: 28px; border-radius: 999px; display: inline-block; +} +.pick-cat { + border: 1px solid var(--line); + border-radius: 10px; + padding: 8px; + background: rgba(12, 16, 14, 0.45); +} +.pick-cat + .pick-cat { margin-top: 8px; } +.pick-cat-head { + display: flex; align-items: center; gap: 8px; margin-bottom: 8px; +} +.pick-cat-head strong { color: var(--text); font-size: 12px; } +.pick-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 6px; +} +.pick-chip { + flex-direction: row; + align-items: center; + gap: 6px; + padding: 6px 8px; + border-radius: 8px; + border: 1px solid var(--line); + background: var(--bg0); + cursor: pointer; + font-size: 12px; + color: var(--text); +} +.pick-chip input { width: auto; } + +.chart-block + .chart-block { margin-top: 22px; } +.chart-wrap { + height: 340px; + background: #0c100e; + border: 1px solid var(--line); + border-radius: 12px; + padding: 12px; + position: relative; +} +.chart-wrap.radar-wrap { height: 380px; } +.chart-wrap.empty::after { + content: attr(data-empty); + position: absolute; + inset: 0; + display: flex; + align-items: center; + justify-content: center; + color: var(--muted); + font-size: 13px; + background: rgba(12, 16, 14, 0.85); + border-radius: 12px; +} + +.table-wrap { + overflow: auto; + border: 1px solid var(--line); + border-radius: 12px; + background: #0c100e; +} +#scoreTable { + width: 100%; + border-collapse: collapse; + font-size: 13px; + font-family: var(--mono); +} +#scoreTable th, #scoreTable td { + padding: 10px 12px; + border-bottom: 1px solid rgba(51, 64, 56, 0.8); + text-align: right; + white-space: nowrap; +} +#scoreTable th:first-child, +#scoreTable td:first-child, +#scoreTable .bench-col { + text-align: left; + position: sticky; + left: 0; + background: #0c100e; +} +#scoreTable th { + color: var(--muted); + font-weight: 500; + background: #121714; +} +#scoreTable td.best { color: #f0d08a; font-weight: 600; } +#scoreTable td.na { color: #5a665e; } +#scoreTable .best-name { color: var(--accent-2); } + +.rank-list { + display: grid; + grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); + gap: 10px; +} +.rank-card { + border: 1px solid var(--line); + border-radius: 12px; + background: var(--bg0); + padding: 12px; +} +.rank-card strong { + display: block; + margin-bottom: 8px; + font-size: 13px; +} +.rank-row { + display: grid; + grid-template-columns: 28px 1fr auto; + gap: 8px; + font-size: 12px; + padding: 4px 0; + color: var(--muted); +} +.rank-idx { color: var(--accent); font-family: var(--mono); } +.rank-name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; color: var(--text); } +.rank-score { font-family: var(--mono); color: #c9d4cb; } + +@media (max-width: 1100px) { + .layout { grid-template-columns: 1fr; } + .results-layout { grid-template-columns: 1fr; } + .filters-panel { position: static; } + .bench-cat-grid { grid-template-columns: repeat(2, minmax(0, 1fr)); } +} +@media (max-width: 720px) { + .grid-2, .suite-list, .bench-cat-grid, .pick-grid { grid-template-columns: 1fr; } + .page { padding: 18px 14px 28px; } + .brand h1 { font-size: 22px; } + .bench-cat-head { flex-wrap: wrap; } + .top { flex-direction: column; align-items: flex-start; } + .top-right { align-items: flex-start; } +}