commit d7547f642aea2cc0ceb594f337c118830d975dfa Author: sora Date: Wed Jul 8 08:56:47 2026 +0000 readme：如下 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..75d4236 --- /dev/null +++ b/.gitignore @@ -0,0 +1,34 @@ +# 本地数据集缓存 +/bash/dataset/* + + +# Python 缓存 +__pycache__/ +*.py[cod] +*.egg-info/ +.pytest_cache/ + +# 环境 +.env +.venv +venv/ +env/ + +# 评测输出结果 +/temp/ +/outputs/ +/output/ + +# 日志 +*.log + +# 密钥/配置文件 +*.secret +*.key +api_keys.json +config_private.py + +# 前端构建产物 +node_modules/ +*.zip +*.tar.gz diff --git a/README.md b/README.md new file mode 100644 index 0000000..b1a8623 --- /dev/null +++ b/README.md @@ -0,0 +1,177 @@ +# Sora Benchmarks + +## 1. 环境安装 + +```bash +conda create -n evalscope python==3.12 -y +conda activate evalscope + +cd evalscope +pip install -e . + +# 安装特定 benchmark 依赖 +pip install 'evalscope[terminal_bench,swe_bench,openai_mrcr]' \ + 'git+https://github.com/sierra-research/tau2-bench@v0.2.0' +pip install 'evalscope[sandbox]' +pip install 'swebench==4.1.0' +``` + +## 2. 常用 Benchmarks + +```python +benchmark_categories = { + "代码与工程": [ + "terminal_bench_v2", + "live_code_bench", + "swe_bench_multilingual_agentic", + "swe_bench_pro", + "swe_bench_verified", + "bigcodebench", + "humaneval", + ], + "推理与数学": [ + "gpqa_diamond", + "hle", + "aime24", + "aime25", + "aime26", + "hmmt26", + "imo_answerbench", + "super_gpqa", + "gsm8k", + "competition_math", + ], + "智能体与工具": [ + "browsecomp", + "mcp_atlas", + "tau2_bench", + ], + "知识与语言理解": [ + "mmlu_pro", + "simple_qa", + "arc", + "bbh", + "cmmlu", + "drop", + "hellaswag", + "mmlu", + "trivia_qa", + "winogrande", + ], + "长上下文": [ + "longbench_v2", + "openai_mrcr", + ], +} +``` + +## 3. 分组运行 + +```python +group_1 = [ + 'terminal_bench_v2', + 'gpqa_diamond', + 'hle', + 'aime24', + 'aime25', + 'mmlu_pro', + 'simple_qa', + 'arc', + 'bbh', + 'browsecomp', +] + +group_2 = [ + 'live_code_bench', + 'swe_bench_pro', + 'aime26', + 'hmmt26', + 'imo_answerbench', + 'super_gpqa', + 'drop', + 'hellaswag', + 'mmlu', + 'openai_mrcr', + 'mcp_atlas', +] + +group_3 = [ + 'swe_bench_multilingual_agentic', + 'swe_bench_verified', + 'bigcodebench', + 'humaneval', + 'gsm8k', + 'competition_math', + 'cmmlu', + 'trivia_qa', + 'winogrande', + 'longbench_v2', + 'tau2_bench', +] +``` + +## 4. 运行示例 + +```python +from evalscope import run_task, TaskConfig + +datasets = [ + 'live_code_bench', + 'swe_bench_pro', + 'aime26', + 'hmmt26', + 'imo_answerbench', + 'super_gpqa', + 'drop', + 'hellaswag', + 'mmlu', + 'openai_mrcr', + 'mcp_atlas', +] + +task_cfg = TaskConfig( + collect_perf=True, + work_dir='/data1/sora/benchmarks/temp/output', + use_cache='/data1/sora/benchmarks/temp/output', + no_timestamp=True, + model='', + api_url='http://localhost:30000/v1', + eval_type='openai_api', + datasets=datasets, + dataset_dir='/data1/sora/benchmarks/bash/datasets', + generation_config={ + 'temperature': 0.0, + 'stream': True, + }, + eval_batch_size=1, + judge_model_args={ + "model_id": "deepseek-v4-flash", + "api_url": "https://api.deepseek.com/v1", + "api_key": "", + "eval_type": "openai_api", + "generation_config": { + "temperature": 0.0, + "max_tokens": 1024 * 10, + }, + } +) + +run_task(task_cfg) +``` + +## 5. 评测结果 + +P800 模型能力评测结果保存在 `temp/output_*` 目录下。 + +## 6. UI 界面 + +```bash +pip install flask sse_starlette +cd evalscope/web +npm install +npm run build +cd /data1/sora/benchmarks +evalscope service +``` + +访问 `http://127.0.0.1:9000/dashboard`。 diff --git a/bash/test.py b/bash/test.py new file mode 100644 index 0000000..fbdf609 --- /dev/null +++ b/bash/test.py @@ -0,0 +1,52 @@ +from evalscope import run_task, TaskConfig +datasets = [ + # 'live_code_bench', + 'aime26', + # 'hmmt26', + # 'imo_answerbench', + # 'super_gpqa', + # 'drop', + # 'hellaswag', + # 'mmlu', + # 'openai_mrcr', + # # 'swe_bench_pro', + # 'mcp_atlas', +] +batch_size_list = [1, 2, 4, 8, 16, 32, 64,128] +for batch_size in batch_size_list: + work_dir = f'/data1/sora/benchmarks/temp/output_{batch_size}' + task_cfg = TaskConfig( + seed=42, + collect_perf= True, + work_dir=work_dir, + use_cache=work_dir, + no_timestamp=True, + model='DeepSeek-V4-Flash-Int8', + api_url='http://localhost:30000/v1', + # api_key='', + eval_type='openai_api', + datasets=datasets, + dataset_dir='/data1/sora/benchmarks/bash/datasets', + generation_config = { + 'temperature': 0.0, + 'stream': True, + "max_tokens": 1024 * 32, + }, + dataset_args={ + 'live_code_bench': { + 'subset_list': ['release_v6'] + } + }, + eval_batch_size = batch_size, + judge_model_args = { + "model_id": "deepseek-v4-pro", + "api_url": "https://api.deepseek.com/v1", + "api_key": "sk-9ed86ef546ca47e3afa7c3b014dea268", + "eval_type": "openai_api", + "generation_config": { + "temperature": 0.0, + "max_tokens": 1024 * 10, + }, + } + ) + run_task(task_cfg) \ No newline at end of file diff --git a/evalscope/.github/ISSUE_TEMPLATE/1-issue-模版-中文版-.md b/evalscope/.github/ISSUE_TEMPLATE/1-issue-模版-中文版-.md new file mode 100644 index 0000000..8c9a042 --- /dev/null +++ b/evalscope/.github/ISSUE_TEMPLATE/1-issue-模版-中文版-.md @@ -0,0 +1,47 @@ +--- +name: 1-Issue 模版（中文版） +about: 模型评测遇到的问题 +title: '' +labels: '' +assignees: '' + +--- + +## 自查清单 + +在提交 issue 之前，请确保您已完成以下步骤: +- [ ] 我已仔细阅读了[相关使用说明文档](https://evalscope.readthedocs.io/zh-cn/latest/get_started/parameters.html) +- [ ] 我已查看了[常见问题解答](https://evalscope.readthedocs.io/zh-cn/latest/get_started/faq.html) +- [ ] 我已搜索并查看了现有的 issues，确认这不是一个重复的问题 + +## 问题描述 + +请简要描述您遇到的问题。 + +## EvalScope 版本（必填） +v0.xx.x + +## 使用的工具 +- [ ] Native / 原生框架 +- [ ] Opencompass backend +- [ ] VLMEvalKit backend +- [ ] RAGEval backend +- [ ] Perf / 模型推理压测工具 +- [ ] Arena / 竞技场模式 + +## 执行的代码或指令 + +请提供您执行的主要代码或指令。 + +## 错误日志 + +请粘贴完整的错误日志或控制台输出。 + +## 运行环境 + +- 操作系统： +- Python版本： + +## 其他信息 + +如果有其他相关信息，请在此处提供。 diff --git a/evalscope/.github/ISSUE_TEMPLATE/2-issue-template--english-version-.md b/evalscope/.github/ISSUE_TEMPLATE/2-issue-template--english-version-.md new file mode 100644 index 0000000..6993258 --- /dev/null +++ b/evalscope/.github/ISSUE_TEMPLATE/2-issue-template--english-version-.md @@ -0,0 +1,47 @@ +--- +name: 2-Issue template (English Version) +about: Issues encountered during model evaluation +title: '' +labels: '' +assignees: '' + +--- + +## Self-Check List + +Before submitting an issue, please ensure you have completed the following steps: +- [ ] I have carefully read the [relevant user documentation](https://evalscope.readthedocs.io/en/latest/get_started/parameters.html) +- [ ] I have reviewed the [Frequently Asked Questions](https://evalscope.readthedocs.io/en/latest/get_started/faq.html) +- [ ] I have searched and reviewed existing issues to confirm this is not a duplicate problem + +## Problem Description + +Please briefly describe the problem you encountered. + +## EvalScope Version (Required) +v0.xx.x + +## Tools Used +- [ ] Native / Native framework +- [ ] Opencompass backend +- [ ] VLMEvalKit backend +- [ ] RAGEval backend +- [ ] Perf / Model inference stress testing tool +- [ ] Arena / Arena mode + +## Executed Code or Instructions + +Please provide the main code or instructions you executed. + +## Error Log + +Please paste the complete error log or console output. + +## Running Environment + +- Operating System: +- Python Version: + +## Additional Information + +If there is any other relevant information, please provide it here. diff --git a/evalscope/.github/ISSUE_TEMPLATE/3-feature-request.md b/evalscope/.github/ISSUE_TEMPLATE/3-feature-request.md new file mode 100644 index 0000000..25aacfb --- /dev/null +++ b/evalscope/.github/ISSUE_TEMPLATE/3-feature-request.md @@ -0,0 +1,24 @@ +--- +name: 3-Feature request +about: Suggest an idea for this project +title: '' +labels: enhancement +assignees: '' + +--- + +## 功能描述 / Feature Description + +请简要描述您希望添加的功能。 / Please briefly describe the feature you would like to request. + +## 需求背景 / Background + +为什么需要这个功能？ / Why is this feature needed? + +## 预期行为 / Expected Behavior + +这个功能应该如何工作？ / How should this feature work? + +## 其他信息 / Additional Information + +还有其他相关信息吗？ / Any other relevant information? diff --git a/evalscope/.github/SECURITY.md b/evalscope/.github/SECURITY.md new file mode 100644 index 0000000..0e74eae --- /dev/null +++ b/evalscope/.github/SECURITY.md @@ -0,0 +1,12 @@ +# Security Policy +The EvalScope team and community take all security bugs in EvalScope seriously. We appreciate your efforts and responsible disclosure and will make every effort to acknowledge your contributions. + +## Reporting a Vulnerability +To report a security issue, please use the **[GitHub Security Advisory "Report a Vulnerability" tab](https://github.com/modelscope/evalscope/security/advisories/new)**. +**Please do not report security vulnerabilities through public GitHub issues.** +You should receive a response within 48 hours. If for some reason you do not, please follow up to ensure we received your original message. +When reporting a vulnerability, please include the following information to help us better understand the issue: +- A description of the vulnerability +- Steps to reproduce the issue +- The potential impact of the vulnerability +- Any known mitigations or workarounds diff --git a/evalscope/.github/copilot-instructions.md b/evalscope/.github/copilot-instructions.md new file mode 120000 index 0000000..be77ac8 --- /dev/null +++ b/evalscope/.github/copilot-instructions.md @@ -0,0 +1 @@ +../AGENTS.md \ No newline at end of file diff --git a/evalscope/.github/workflows/ci_test.yaml b/evalscope/.github/workflows/ci_test.yaml new file mode 100644 index 0000000..94d0352 --- /dev/null +++ b/evalscope/.github/workflows/ci_test.yaml @@ -0,0 +1,71 @@ +name: CI Tests Lite + +on: + push: + branches: + - main + pull_request: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-tests: + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e '.[dev]' + + - name: Create .env file + run: | + echo "DASHSCOPE_API_KEY=${{ secrets.DASHSCOPE_API_KEY }}" > .env + + - name: Run integration tests + run: | + python -m pytest tests/cli/test_all.py::TestRun::test_ci_lite -v -s -p no:warnings + + + perf-tests: + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e '.[dev,perf]' + + - name: Create .env file + run: | + echo "DASHSCOPE_API_KEY=${{ secrets.DASHSCOPE_API_KEY }}" > .env + + - name: Run perf test + run: | + python -m pytest tests/perf/test_perf_basic.py::TestPerfBasic::test_multi_parallel_sweep -v -s -p no:warnings diff --git a/evalscope/.github/workflows/ci_test_full.yaml b/evalscope/.github/workflows/ci_test_full.yaml new file mode 100644 index 0000000..525f6e4 --- /dev/null +++ b/evalscope/.github/workflows/ci_test_full.yaml @@ -0,0 +1,73 @@ +name: CI Tests Full + +on: + push: + branches: + - "release/**" + pull_request: + branches: + - "release/**" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-tests: + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e '.[dev,ifeval,ifbench,multi_if,needle_haystack,arena_hard]' + pip install git+https://github.com/sierra-research/tau-bench + pip install bfcl-eval==2025.10.27.1 + + - name: Create .env file + run: | + echo "DASHSCOPE_API_KEY=${{ secrets.DASHSCOPE_API_KEY }}" > .env + + - name: Run integration tests + run: | + python -m pytest tests/cli/test_all.py::TestRun::test_benchmarks -v -s -p no:warnings + + + perf-tests: + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -e '.[dev,perf]' + + - name: Create .env file + run: | + echo "DASHSCOPE_API_KEY=${{ secrets.DASHSCOPE_API_KEY }}" > .env + + - name: Run perf test + run: | + python -m pytest tests/perf/test_perf_basic.py::TestPerfBasic::test_multi_parallel_sweep -v -s -p no:warnings diff --git a/evalscope/.github/workflows/lint.yml b/evalscope/.github/workflows/lint.yml new file mode 100644 index 0000000..305e1ae --- /dev/null +++ b/evalscope/.github/workflows/lint.yml @@ -0,0 +1,32 @@ +name: Lint Test + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + run-pre-commit: + runs-on: ubuntu-latest + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.10' + + - name: Install pre-commit + run: | + python -m pip install --upgrade pip + pip install pre-commit + + - name: Run pre-commit + run: pre-commit run --all-files diff --git a/evalscope/.github/workflows/publish.yaml b/evalscope/.github/workflows/publish.yaml new file mode 100644 index 0000000..437b76b --- /dev/null +++ b/evalscope/.github/workflows/publish.yaml @@ -0,0 +1,37 @@ +name: release + +on: + push: + tags: + - 'v**' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-publish + cancel-in-progress: true + +jobs: + build-n-publish: + runs-on: ubuntu-22.04 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + #if: startsWith(github.event.ref, 'refs/tags') + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.10 + uses: actions/setup-python@v5 + with: + python-version: '3.10' + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '22' + - name: Build frontend + run: make web-build + - name: Install wheel + run: pip install wheel && pip install -r requirements/framework.txt + - name: Build EvalScope + run: python setup.py sdist bdist_wheel + - name: Publish package to PyPI + run: | + pip install twine + twine upload dist/* --skip-existing -u __token__ -p ${{ secrets.PYPI_TOKEN }} diff --git a/evalscope/.gitignore b/evalscope/.gitignore new file mode 100644 index 0000000..f8e514d --- /dev/null +++ b/evalscope/.gitignore @@ -0,0 +1,184 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +!evalscope/web/src/lib/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +/package +/temp +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +.vscode +.idea + +# custom +*.pkl +*.pkl.json +*.log.json +*.whl +*.tar.gz +*.swp +*.log +*.tar.gz +source.sh +tensorboard.sh +.DS_Store +replace.sh +result.png +result.jpg +result.mp4 +*.itag + +# Pytorch +*.pth +*.pt + +# personal info +private/ + +# others +*.tokenization + +# outputs +outputs/ +evalscope/outputs/ +evalscope/*_temp.py +evalscope/data/ +*/output_res/* + + +/tmp +/data* +*.zip +output/ +/test*.* +*.ttf +*.zip +_build/ +swift.test* +/cache +evalscope/backend/rag_eval/ragas/prompts/chinese +logs +.gradio +*.h5 +CLAUDE.md +Tool/ +docs/zh/learn/ +examples/api_test/ +.qoder +.claude + +# Frontend (evalscope/web) +node_modules/ +evalscope/web/dist/ +evalscope/web/.vite/ +ragas_mini_testset_score.json + +# User benchmark outputs and secrets +benchmark_outputs/ +test_scripts/ +*.secret +*.key +api_keys.json + +# Docker build artifacts (pre-cloned agent sources) +evalscope/agent/external/dockerfiles/hermes-agent-src/ +evalscope/agent/external/dockerfiles/hermes-install.sh diff --git a/evalscope/.pre-commit-config.yaml b/evalscope/.pre-commit-config.yaml new file mode 100644 index 0000000..e650b66 --- /dev/null +++ b/evalscope/.pre-commit-config.yaml @@ -0,0 +1,52 @@ +repos: + - repo: https://github.com/pycqa/flake8.git + rev: 7.3.0 + hooks: + - id: flake8 + exclude: | + (?x)^( + examples/| + docs/| + tests/| + evalscope/utils/utils.py| + evalscope/third_party/| + evalscope/backend/rag_eval/clip_benchmark/tasks| + evalscope/backend/rag_eval/cmteb/tasks| + evalscope/metrics/vision/t2v_metrics + ) + - repo: https://github.com/PyCQA/isort.git + rev: 7.0.0 + hooks: + - id: isort + - repo: https://github.com/google/yapf + rev: v0.43.0 + hooks: + - id: yapf + exclude: | + (?x)^( + examples/| + docs/| + tests/| + evalscope/utils/utils.py| + evalscope/third_party/| + evalscope/backend/rag_eval/clip_benchmark/tasks| + evalscope/backend/rag_eval/cmteb/tasks + ) + - repo: https://github.com/pre-commit/pre-commit-hooks.git + rev: v6.0.0 + hooks: + - id: trailing-whitespace + exclude: thirdparty/|docs/|examples + - id: check-yaml + exclude: thirdparty/|docs/|examples + - id: end-of-file-fixer + exclude: thirdparty/|docs/|examples|.*\.json + - id: requirements-txt-fixer + exclude: thirdparty/|docs/|examples + - id: double-quote-string-fixer + exclude: thirdparty/|docs/|examples|.*\.json|cl_bench_adapter.py + - id: check-merge-conflict + exclude: thirdparty/|docs/|examples + - id: mixed-line-ending + exclude: thirdparty/|docs/|examples + args: ["--fix=lf"] diff --git a/evalscope/AGENTS.md b/evalscope/AGENTS.md new file mode 100644 index 0000000..77f784d --- /dev/null +++ b/evalscope/AGENTS.md @@ -0,0 +1,136 @@ +# AGENTS.md + +EvalScope — LLM evaluation framework with a registry-based plugin architecture. This file is the contract for AI coding agents working in this repo. + +## Setup + +```bash +pip install -e . # basic install +make dev # dev + perf + docs extras + pre-commit +``` + +Python ≥ 3.10 (3.10 / 3.11 / 3.12). Dependencies: `requirements/framework.txt` + `pyproject.toml [project.optional-dependencies]` (extras: `opencompass`, `vlmeval`, `rag`, `perf`, `app`, `aigc`, `sandbox`, `service`, `dev`, `docs`, `all`, plus per-benchmark extras). + +## Build, lint, test + +```bash +make lint # required before commit (yapf + isort + flake8 + basic pre-commit hooks) +pytest tests/cli/test_all.py::TestRun::test_ci_lite -v -s -p no:warnings # CI smoke test +pytest tests/perf/test_perf_basic.py::TestPerfBasic::test_multi_parallel_sweep -v -s # perf +``` + +Commits failing `make lint` are rejected on `main`. + +## Docs generation + +Benchmark detail pages (`docs/{zh,en}/benchmarks/.md`) and meta cache (`evalscope/benchmarks/_meta/.json`) are **auto-generated** from each adapter's `BenchmarkMeta.description` + dataset statistics. Do not hand-edit those files. + +When you add a benchmark or change its `BenchmarkMeta.description`, run: + +```bash +make docs-pipeline BENCHMARK=" " FORCE=1 # update _meta JSON + translate descriptions to zh +make docs-generate # render .md files from _meta +``` + +Targets: `docs-update` (meta only), `docs-update-stats` (+ dataset statistics), `docs-translate` (zh), `docs-pipeline` (stats + translate), `docs-generate` (.md), `docs` (full Sphinx HTML build). + +Conventions: +- `BENCHMARK="a b c"` selects benchmarks; omit for `--all`. +- `FORCE=1` appends `--force` to recompute even if data is cached. +- `WORKERS=N` parallelism (default 4). +- `--translate` calls an LLM; needs `DASHSCOPE_API_KEY` (or equivalent) in env. + +## Quick eval + +```bash +evalscope eval --model Qwen/Qwen2.5-0.5B-Instruct --datasets gsm8k --limit 5 +``` + +```python +from evalscope import run_task, TaskConfig +run_task(TaskConfig(model='Qwen/Qwen2.5-0.5B-Instruct', datasets=['gsm8k'], limit=5)) +``` + +## Code style (enforced) + +- **Line width 120**, 4-space indent, LF endings, trailing newline at EOF. +- **Quotes** governed by `double-quote-string-fixer` hook — follow existing file style; do not mix. +- **f-strings** for formatting (no `%` or `.format()` unless necessary). +- **Imports**: isort with `first_party = evalscope`, groups `STDLIB / THIRDPARTY / LOCALFOLDER`, `multi_line_output=3`. +- **Type hints required** on every function signature. +- **English only** for comments and docstrings. +- **Public APIs need docstrings**; internal helpers only when intent is non-obvious. +- `# TODO:` prefix for pending work. + +| Element | Style | +| --- | --- | +| Class | `PascalCase` | +| Function / variable | `snake_case` | +| Constant | `UPPER_SNAKE_CASE` | +| Private | `_leading_underscore` | +| Handler function | `handle_` prefix | +| Benchmark adapter file | `_adapter.py` | + +**flake8 ignore list** (`setup.cfg`): `F401, F403, F405, F821, W503, E251, W504, F824, F541, E501, E226, E121-E129, E131, E741`. Do not expand — new ignores must be justified in the PR. + +## Design rules + +- **Early returns** over nested conditionals. +- **Minimal changes**: only touch code related to the current task; no drive-by cleanup. +- **Pydantic-first**: cross-module data contracts use Pydantic models. Use `TaskConfig` / `Arguments` for configuration — never raw dicts at module boundaries. +- **Reuse existing patterns**: new benchmarks / models / metrics go through existing registries and adapter base classes — no parallel mechanisms. +- **DRY** but don't over-abstract just to remove minor duplication. + +## Tests + +- Live under `tests/`; files `*test*.py`, classes `Test*`, functions `test_*`. +- New benchmark / model / metric **must** ship a minimal runnable test (pattern: `tests/cli/test_all.py::TestRun::test_ci_lite`). +- Mock external services — no reliance on real network / paid APIs. + +## Architecture pointers + +Don't try to learn the architecture from this file — read these and grep: + +| Topic | Source of truth | +| --- | --- | +| Main flow | `evalscope/run.py` → `evalscope/evaluator/evaluator.py` | +| Config schema | `evalscope/config.py` (`TaskConfig`) | +| Registries | `evalscope/api/registry.py` | +| Benchmark contract | `evalscope/api/benchmark/benchmark.py` (`DataAdapter`, `BenchmarkMeta`) | +| Model layer | `evalscope/api/model/model.py`, `evalscope/models/model_apis.py` | +| CLI dispatch | `evalscope/cli/` | +| Cache schema | `evalscope/api/evaluator/cache.py` | + +**Registry decorators**: `@register_benchmark`, `@register_model_api`, `@register_metric`, `@register_aggregation`, `@register_filter`, `@register_evaluator`. + +**Adapter base classes** (extend, don't reinvent): `DefaultDataAdapter`, `MultiChoiceAdapter`, `VisionLanguageAdapter`, `Text2ImageAdapter`, `ImageEditAdapter`, `NERAdapter`, `AgentAdapter`. Optional capabilities via mixins: `LLMJudgeMixin`, `SandboxMixin`. + +**Non-native backends** live under `evalscope/backend/` (OpenCompass, VLMEvalKit, RAGEval) and are dispatched from `run.py` with their own BackendManager. + +## Adding a benchmark + +1. Create `evalscope/benchmarks//_adapter.py`. +2. Extend `DefaultDataAdapter`, override `record_to_sample()` (and optionally `sample_to_fewshot()`, `extract_answer()`). +3. Decorate with `@register_benchmark(BenchmarkMeta(name=..., ...))`. +4. Auto-discovered by globbing `evalscope/benchmarks/*/**/*_adapter.py`. +5. Add a smoke test. + +## Conventions & gotchas + +- `eval_type`: `openai_api`, `llm_ckpt`, `mock_llm`, `text2image`, `image_editing`. Deprecated aliases: `server` → `openai_api`, `checkpoint` → `llm_ckpt`. +- `limit`: `int` = count, `float` = fraction. +- `repeats`: duplicates items for k-metrics. `generation_config.n` is deprecated and mapped. +- Use `generation_config` for runtime params. `TaskConfig.timeout` / `stream` are deprecated — forwarded with a warning. +- `dataset_args` merges into `BenchmarkMeta._update()` (supports `local_path`, `filters` OrderedDict prepended). +- Models are memoized by `(name, config, base_url, api_key, args)`. +- Use `@thread_safe` for model creation, `run_in_threads_with_progress` for concurrent eval. +- Outputs land in `outputs//{logs,predictions,reviews,reports,configs}/` (see `OutputsStructure`). `use_cache` resumes runs; `rerun_review` recomputes scores only. +- `evalscope app` CLI command is **deprecated** (see `evalscope/cli/start_app.py`) — use `evalscope service` for the Web dashboard. + +## Submission + +```bash +make dev # once +make lint # before every commit +pytest tests/cli/test_all.py::TestRun::test_ci_lite -v -s -p no:warnings +``` diff --git a/evalscope/CONTRIBUTING.md b/evalscope/CONTRIBUTING.md new file mode 100644 index 0000000..df66740 --- /dev/null +++ b/evalscope/CONTRIBUTING.md @@ -0,0 +1,355 @@ +# Contributing to EvalScope + +Thank you for considering contributing to EvalScope! This guide covers everything you need to get started. + +## Table of Contents + +- [Contributing to EvalScope](#contributing-to-evalscope) + - [Table of Contents](#table-of-contents) + - [Quick Start](#quick-start) + - [Development Setup](#development-setup) + - [Backend (Python)](#backend-python) + - [Frontend (React + Vite)](#frontend-react--vite) + - [Full-Stack Development](#full-stack-development) + - [Project Structure](#project-structure) + - [Adding a New Benchmark](#adding-a-new-benchmark) + - [Step 1: Create the adapter directory](#step-1-create-the-adapter-directory) + - [Step 2: Write the adapter](#step-2-write-the-adapter) + - [Key methods you can override](#key-methods-you-can-override) + - [BenchmarkMeta key fields](#benchmarkmeta-key-fields) + - [Step 3: Add extra dependencies (if any)](#step-3-add-extra-dependencies-if-any) + - [Step 4: Update documentation (optional)](#step-4-update-documentation-optional) + - [Verify your benchmark](#verify-your-benchmark) + - [Code Quality](#code-quality) + - [Linting](#linting) + - [Testing](#testing) + - [Git Workflow](#git-workflow) + +--- + +## Quick Start + +```bash +# 1. Fork & clone +git clone https://github.com//evalscope.git +cd evalscope + +# 2. Install in editable mode with dev dependencies +make dev + +# 3. Install pre-commit hooks +pre-commit install +``` + +--- + +## Development Setup + +### Backend (Python) + +EvalScope requires **Python >= 3.10**. + +```bash +# Base install (editable) +pip install -e . + +# With all dev tools +pip install -e '.[dev,perf,docs]' + +# With the web service +pip install -e '.[service]' +``` + +**Run the backend service:** + +```bash +evalscope service --host 0.0.0.0 --port 9000 +``` + +Optional dependency groups (install via `pip install -e '.[]'`): + +| Group | Purpose | Key Packages | +|-------|---------|-------------| +| `dev` | Testing & linting | pytest, pytest-cov | +| `service` | Web dashboard & REST API | flask, plotly | +| `perf` | Performance benchmarking | — | +| `docs` | Documentation build | sphinx | +| `rag` | RAG evaluation | — | +| `aigc` | AIGC evaluation | — | +| `sandbox` | Sandboxed code execution | ms-enclave | + +Some benchmarks have their own extra dependencies (e.g. `pip install -e '.[bfcl]'`). + +### Frontend (React + Vite) + +The dashboard is a React SPA located at `evalscope/web/`. + +```bash +# Install dependencies +make web-install + +# Start dev server (hot reload, proxies API to localhost:9000) +make web-dev + +# Production build +make web-build +``` + +The dev server runs at `http://localhost:5173` and automatically proxies `/api/v1/*` and `/health` to the backend at `http://127.0.0.1:9000`. + +**Tech stack:** React 19 · TypeScript · Vite · Tailwind CSS 4 · React Router · Plotly.js + +### Full-Stack Development + +For the best development experience, run both servers simultaneously: + +```bash +# Terminal 1: Backend +evalscope service --debug + +# Terminal 2: Frontend (hot reload) +make web-dev +``` + +Open `http://localhost:5173` in your browser — changes to frontend code are reflected instantly. + +--- + +## Project Structure + +``` +evalscope/ +├── api/ # Core API: registry, benchmark base classes, dataset, metric, model +│ ├── benchmark/ # DataAdapter, BenchmarkMeta, adapter subclasses +│ ├── dataset/ # Dataset loading, Sample dataclass +│ ├── evaluator/ # TaskState, evaluation loop +│ ├── messages/ # Chat message types +│ ├── metric/ # Score, AggScore, metric registry +│ ├── model/ # Model abstraction (OpenAI-compatible) +│ └── registry.py # register_benchmark(), BENCHMARK_REGISTRY +│ +├── benchmarks/ # All benchmark adapters (auto-discovered) +│ └── / +│ ├── __init__.py +│ └── _adapter.py +│ +├── cli/ # CLI entry points (evalscope eval/perf/service/app) +├── constants.py # Global constants & tags +├── perf/ # Performance benchmarking subsystem +├── report/ # Report generation & visualization +├── service/ # Flask REST API + SPA serving +│ ├── app.py # Flask app factory, run_service() +│ └── blueprints/ # API route handlers (eval, perf, reports) +├── utils/ # Shared utilities (logging, IO, etc.) +└── web/ # React SPA (dashboard UI) + ├── src/ + │ ├── api/ # API client & type definitions + │ ├── components/ # UI components + │ ├── pages/ # Route pages + │ └── i18n/ # Internationalization + └── vite.config.ts +``` + +--- + +## Adding a New Benchmark + +EvalScope uses a **decorator-based registry** pattern. Adding a benchmark requires only two files. + +### Step 1: Create the adapter directory + +``` +evalscope/benchmarks/my_benchmark/ +├── __init__.py # (empty) +└── my_benchmark_adapter.py +``` + +Adapters are **auto-discovered**: any `*_adapter.py` under `evalscope/benchmarks/` is automatically imported at startup, which triggers the `@register_benchmark` decorator. + +### Step 2: Write the adapter + +Choose a base class depending on your benchmark type: + +| Base Class | Use When | +|------------|----------| +| `DefaultDataAdapter` | General text QA, math, coding | +| `MultiChoiceAdapter` | Multiple-choice questions | +| `AgentAdapter` | Function calling, tool use | +| `VisionLanguageAdapter` | Image + text (VQA, etc.) | +| `MultiTurnAdapter` | Multi-turn conversations | +| `Text2ImageAdapter` | Text-to-image generation | +| `NERAdapter` | Named entity recognition | +| `ImageEditAdapter` | Image editing | + +**Minimal example** (text QA benchmark): + +```python +from typing import Any, Dict +from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter +from evalscope.api.dataset import Sample +from evalscope.api.registry import register_benchmark +from evalscope.constants import Tags + +DESCRIPTION = """ +## Overview +Brief description of what this benchmark evaluates. + +## Task Description +- **Task Type**: ... +- **Input**: ... +- **Output**: ... + +## Evaluation Notes +- Default configuration uses **0-shot** evaluation +""" + +@register_benchmark( + BenchmarkMeta( + name='my_benchmark', # unique identifier (snake_case) + pretty_name='MyBenchmark', # display name + dataset_id='org/dataset-name', # ModelScope / HuggingFace dataset ID + tags=[Tags.REASONING], # category tags + description=DESCRIPTION, + subset_list=['default'], # dataset subsets + metric_list=['acc'], # evaluation metrics + eval_split='test', # split to evaluate on + few_shot_num=0, # number of few-shot examples + prompt_template='{question}', # prompt template with placeholders + ) +) +class MyBenchmarkAdapter(DefaultDataAdapter): + + def record_to_sample(self, record: Dict[str, Any]) -> Sample: + """Convert a dataset row to a Sample object.""" + return Sample( + input=record['question'], + target=record['answer'], + ) +``` + +### Key methods you can override + +| Method | Purpose | When to Override | +|--------|---------|------------------| +| `record_to_sample()` | Map dataset row → `Sample` | Always | +| `extract_answer()` | Extract structured answer from model output | When default extraction is insufficient | +| `match_score()` | Custom scoring logic | When `acc` / built-in metrics don't fit | +| `sample_to_fewshot()` | Format a sample as few-shot example | When using few-shot with custom format | +| `_on_inference()` | Custom model interaction | For agent/tool-use benchmarks | + +### BenchmarkMeta key fields + +```python +BenchmarkMeta( + name='...', # Required: unique snake_case ID + dataset_id='...', # Required: remote dataset ID or local path + pretty_name='...', # Display name + tags=[...], # From evalscope.constants.Tags + description='...', # Markdown description for docs + subset_list=['default'], # Dataset subsets + metric_list=['acc'], # Metric names or dicts: [{'acc': {'numeric': True}}] + aggregation='mean', # 'mean', 'pass@k', 'f1', etc. + eval_split='test', # Evaluation split name + train_split='train', # Training split (for few-shot) + few_shot_num=0, # Few-shot count + prompt_template='...', # Prompt template with {placeholders} + filters=OrderedDict(), # Output filters + extra_params={}, # Additional configurable parameters + sandbox_config={}, # Sandboxed execution config (for code benchmarks) + review_timeout=None, # Per-sample timeout in seconds +) +``` + +### Step 3: Add extra dependencies (if any) + +If your benchmark needs additional packages, create a `requirements.txt` in the benchmark directory: + +``` +evalscope/benchmarks/my_benchmark/requirements.txt +``` + +Then register it in `pyproject.toml`: + +```toml +[tool.setuptools.dynamic.optional-dependencies] +my_benchmark = {file = ["evalscope/benchmarks/my_benchmark/requirements.txt"]} +``` + +Users can install via `pip install 'evalscope[my_benchmark]'`. + +### Step 4: Update documentation (optional) + +Run the doc pipeline to auto-generate benchmark documentation: + +```bash +make docs-update BENCHMARK=my_benchmark +make docs-translate BENCHMARK=my_benchmark +make docs-generate +``` + +### Verify your benchmark + +```bash +# Check it's registered +evalscope eval --benchmarks my_benchmark --model dummy --limit 5 + +# Run via service +evalscope service +# Then use the Web dashboard or API: POST /api/v1/eval/invoke +``` + +--- + +## Code Quality + +### Linting + +This project uses **pre-commit** with the following hooks: + +- **flake8** — Python style checker +- **isort** — Import sorting +- **yapf** — Code formatting +- Trailing whitespace, YAML checks, line ending fixes + +```bash +# Run all checks +make lint +# or +pre-commit run --all-files +``` + +### Testing + +```bash +# Run all tests +pytest tests/ + +# Run a specific test +pytest tests/benchmark/test_xxx.py +``` + +--- + +## Git Workflow + +1. **Create a branch** with a descriptive name: + ```bash + git checkout -b feature/my-benchmark + ``` + +2. **Make your changes** and commit with clear messages: + ```bash + git commit -m "feat: add MyBenchmark adapter" + ``` + +3. **Run quality checks** before pushing: + ```bash + pre-commit run --all-files + pytest tests/ + ``` + +4. **Push and open a Pull Request** against the `main` branch. Provide a clear description of your changes. + +--- + +Thank you for your contribution! diff --git a/evalscope/DESIGN.md b/evalscope/DESIGN.md new file mode 100644 index 0000000..23ddd7f --- /dev/null +++ b/evalscope/DESIGN.md @@ -0,0 +1,682 @@ +--- +name: EvalScope Console +colors: + # Brand & Accent — IDENTICAL across both themes. Violet is the brand constant. + accent: "#816DF8" + accent-dark: "#5B3FD6" + accent-dim: "rgba(129,109,248,0.12)" + purple: "#a78bfa" + + # Surface ladder (sunken → elevated) — DARK + bg: "#0c0c1a" + bg-deep: "#09091a" + bg-card: "#12122b" + bg-card2: "#16163a" + surface-glass: "rgba(18,18,43,0.7)" + + # Text (3-step ladder) — DARK + text: "#e2e8f0" + text-muted: "#8896aa" + text-dim: "#7a8195" + on-filled: "#ffffff" + + # Hairline borders — DARK (translucent violet, the near-black bg lets even 10% read) + border: "rgba(129,109,248,0.10)" + border-md: "rgba(129,109,248,0.18)" + border-strong: "rgba(129,109,248,0.28)" + + # Semantic states + success: "#10b981" + warning: "#f59e0b" + danger: "#ef4444" + info: "#60a5fa" + + # Deep saturated pass/fail (boolean badges) + pass: "rgb(45,104,62)" + fail: "rgb(151,31,44)" + + # ──────────────────────────────────────────────────────────────── + # Light theme — warm-cream Console (used when [data-theme="light"]) + # Distinct palette philosophy: warm-neutral surfaces, solid warm-grey + # hairlines (NOT translucent violet), warm ink text. Same violet accent. + # ──────────────────────────────────────────────────────────────── + + # Surface ladder — warm-cream, sunken → elevated + bg-light: "#faf9f5" # warm cream canvas — was cool #f5f6fa + bg-deep-light: "#f0ebe1" # input wells, one step below canvas — was cool #e8eaf2 + bg-card-light: "#ffffff" # pure white — strongest possible contrast against cream canvas + bg-card2-light: "#f5f0e7" # hover / elevated — warm cream-soft, was cool #eef0f7 + surface-glass-light: "rgba(250,249,245,0.80)" # warm-tinted glass — was pure white + + # Accent (unchanged from dark — violet is the brand constant) + accent-light: "#6c57e8" + accent-dim-light: "rgba(108,87,232,0.14)" # slightly stronger on white card + + # Text — warm-ink ladder + text-light: "#141413" # warm near-black — was cool #1a1f2e + text-muted-light: "#6c6a64" # warm grey — was cool #5a6378 + text-dim-light: "#8e8b82" # warm grey — was cool #7c8497 + + # Hairlines — SOLID warm hex, not translucent violet. Three concrete tones. + # Critical: translucent violet at 0.20 alpha composites to near-invisible + # on white cards. Solid warm-grey gives every card a definite boundary. + border-light: "#e6dfd8" # standard hairline — was rgba(violet,0.20) + border-md-light: "#d6cdbe" # emphasized — was rgba(violet,0.30) + border-strong-light: "#c1b6a3" # hover / focus boundary — was rgba(violet,0.40) + + # Compare slot accents (per-model tagging in compare view) + compare-0: "#818cf8" + compare-1: "#34d399" + compare-2: "#fbbf24" + + # Chat-bubble role accents + bubble-user: "#818cf8" + bubble-bot: "#34d399" + bubble-tool: "#fbbf24" + bubble-reasoning: "#34d399" + bubble-system: "rgba(148,163,184,1)" +typography: + display-xl: + fontFamily: System Sans + fontSize: 24px + fontWeight: 700 + letterSpacing: -0.02em + lineHeight: 1.2 + title-md: + fontFamily: System Sans + fontSize: 16px + fontWeight: 700 + letterSpacing: normal + lineHeight: 1.25 + body-sm: + fontFamily: System Sans + fontSize: 14px + fontWeight: 400 + letterSpacing: normal + lineHeight: 1.5 + body-sm-strong: + fontFamily: System Sans + fontSize: 14px + fontWeight: 500 + letterSpacing: normal + lineHeight: 1.5 + body-xs: + fontFamily: System Sans + fontSize: 12px + fontWeight: 400 + letterSpacing: normal + lineHeight: 1.5 + label-xs: + fontFamily: System Sans + fontSize: 12px + fontWeight: 600 + letterSpacing: 0.05em + textTransform: uppercase + lineHeight: 1.4 + table-xs: + fontFamily: System Sans + fontSize: 10px + fontWeight: 600 + letterSpacing: 0.05em + textTransform: uppercase + lineHeight: 1.4 + caption-mono: + fontFamily: System Mono + fontSize: 12px + fontWeight: 400 + letterSpacing: normal + lineHeight: 1.4 + code: + fontFamily: System Mono + fontSize: 13px + fontWeight: 400 + letterSpacing: normal + lineHeight: 1.5 + button-sm: + fontFamily: System Sans + fontSize: 12px + fontWeight: 500 + letterSpacing: normal + lineHeight: 1.4 + button-md: + fontFamily: System Sans + fontSize: 14px + fontWeight: 500 + letterSpacing: normal + lineHeight: 1.4 + button-lg: + fontFamily: System Sans + fontSize: 16px + fontWeight: 500 + letterSpacing: normal + lineHeight: 1.4 +fontFamily: + sans: 'system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif' + mono: 'ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, "Liberation Mono", "DejaVu Sans Mono", "Courier New", monospace' +rounded: + none: 0px + xs: 4px + sm: 8px + md: 12px + lg: 16px + xl: 20px + full: 9999px +spacing: + xs: 4px + sm: 8px + md: 12px + lg: 16px + xl: 20px + 2xl: 24px + 3xl: 32px + 4xl: 48px + 5xl: 64px +shadows: + # Dark theme — single deep drop (works on near-black surfaces). + sm: "0 2px 8px rgba(0,0,0,0.4)" + md: "0 4px 20px rgba(0,0,0,0.55)" + lg: "0 8px 40px rgba(0,0,0,0.6)" + glow: "0 0 20px rgba(129,109,248,0.25)" + glow-soft: "0 0 12px rgba(129,109,248,0.2)" + # Light theme — two-stop stacks tinted with warm-ink (matches text colour), + # not slate. Slate-tinted drops on cream read as a cool-grey smudge and break + # the warm canvas. Warm-ink stays consistent with the rest of the palette. + sm-light: "0 1px 2px rgba(20,20,19,0.04), 0 4px 12px rgba(20,20,19,0.06)" + md-light: "0 4px 16px rgba(20,20,19,0.07), 0 12px 32px rgba(20,20,19,0.05)" + lg-light: "0 12px 24px rgba(20,20,19,0.09), 0 24px 48px rgba(20,20,19,0.07)" + glow-light: "0 0 20px rgba(108,87,232,0.22)" + glow-soft-light: "0 0 12px rgba(108,87,232,0.18)" +gradients: + brand: "linear-gradient(135deg, #816DF8 0%, #a78bfa 100%)" + accent: "linear-gradient(135deg, #0F9C7E 0%, #06b6d4 100%)" + surface: "linear-gradient(135deg, rgba(129,109,248,0.08) 0%, rgba(167,139,250,0.05) 100%)" + kpi-0: "linear-gradient(135deg, #6366f1, #8b5cf6)" + kpi-1: "linear-gradient(135deg, #10b981, #06b6d4)" + kpi-2: "linear-gradient(135deg, #f59e0b, #f97316)" + kpi-3: "linear-gradient(135deg, #ec4899, #8b5cf6)" + nav-hairline: "linear-gradient(90deg, transparent 0%, #816DF8 50%, transparent 100%)" +transition: + fast: "150ms cubic-bezier(0.4, 0, 0.2, 1)" + base: "180ms ease" + slow: "400ms cubic-bezier(0.4, 0, 0.2, 1)" +breakpoints: + sm: 640px + md: 768px + lg: 1024px + xl: 1280px +container: + max-width: 1600px + page-padding-x: 16px + page-padding-y: 20px +score-formula: + foreground: "hsl(score * 120, var(--score-fg-s), var(--score-fg-l)) # dark: 70%/45%, light: 85%/32%" + background: "RGB-interpolated translucent companion, alpha * var(--score-bg-a-mul) # dark: ×1, light: ×1.6" + description: "0 → red, 0.5 → yellow, 1 → green — used for all dynamic score chips, badges, and SVG rings. Saturation, lightness, and bg alpha multiplier are theme-scoped CSS vars so yellow mid-tones stay legible on warm-cream without re-coding the brand HSL. Light theme bumps saturation +15pt to compensate for the lightness compression at hue ≈ 60 (olive)." +--- + +# Design System: EvalScope Console + +## Overview + +EvalScope's web dashboard is a developer-platform brand for **LLM evaluation and benchmarking** — the page is an instrument panel for engineers running evals, written for people who already know the syntax. It earns that posture through **two equally weighted themes** rather than one canonical mode with a translated companion. Both themes share the same vocabulary — same type, same spacing, same radii, same components — but each carries its own surface philosophy. They are two voices of one brand, not one design re-tinted. + +**Dark Console** (default) is a stark dark-indigo system: near-black `{colors.bg}` canvas, ice-cool `{colors.text}` body, a 3-step text ladder. Hairlines are translucent violet at 10-28 % alpha because the near-black bg lets even a faint violet tint read as a definite edge. Shadows are single deep drops at `rgba(0,0,0,0.55)` — near-black eats soft shadows. The mood is *late-night terminal session*: low-light, low-distraction, the violet glow on a primary CTA is the only saturated thing on screen. + +**Warm Console** (light theme) is a warm-cream system: cream `{colors.bg-light}` canvas (`#faf9f5`), pure white `{colors.bg-card-light}` cards, warm-ink `{colors.text-light}` body (`#141413`). Hairlines are **solid warm-grey** (`{colors.border-light}` `#e6dfd8`, `#d6cdbe`, `#c1b6a3`) — *not* translucent violet, because violet at 20 % alpha disappears against a white card and leaves every card boundary undefined. Shadows are two-stop stacks tinted with the same warm-ink as the text, so cards lift off the cream without printing a cool slate smudge. The mood is *morning code review*: high-contrast, restful on the eyes, the violet CTA is the only cool note in an otherwise warm palette. + +The brand constant across both themes is the single violet `{colors.accent}` (`#816DF8` dark / `#6c57e8` light) used for primary CTAs, active nav states, focus rings, and the wordmark accent — plus the dynamic HSL score gradient (`hsl(score × 120, 70%, 45%)`) that maps a 0-1 metric to red → yellow → green. Both signals work over either canvas. Everything else — surface ladder, hairline material, shadow tint, on-canvas text colour — is theme-specific by design, because dark and light surfaces need *different* materials to produce the same hierarchy. + +Type is the second decisive voice and is **theme-agnostic**. The brand uses cross-platform system font stacks (no web font is loaded) — `system-ui, -apple-system, "Segoe UI", Roboto, ...` for narrative and `ui-monospace, SFMono-Regular, Menlo, Consolas, "Liberation Mono", ...` for technical labels. Each OS resolves to its own native UI face. Headlines are sentence-case with `tracking-tight` on display numbers; **all-caps + `tracking-wider`** is reserved for tiny section eyebrows (12 px / 10 px), never headlines. Weight ceiling is **700**; the working set is 400 / 500 / 600 / 700. + +**Key Characteristics:** + +- **Dual-theme parity, not dual-theme translation.** Dark uses translucent violet hairlines on near-black; light uses solid warm-grey hairlines on cream. They produce the same hierarchy through opposite material choices. Theme is persisted to `localStorage` and applied via `data-theme` on `` before first paint to avoid a flash. +- A single violet primary CTA `{colors.accent}` carries every conversion target on both themes, paired with a transparent **ghost** secondary. The brand uses a `{rounded.sm}` 8-px button shape for primary/secondary in the *console* (no marketing pills — this is an in-product surface). +- The primary CTA **glows on hover** with violet at 20-25 % alpha on both themes. That violet glow is the brand's signature interaction — identical animation, identical colour, identical timing across themes. +- Every card section title, form label, and table header sets in `{typography.label-xs}` — 12 px (10 px for tables), `font-semibold`, **UPPERCASE**, `tracking-wider`, muted color. Body and titles stay sentence-case. The contrast between these two voices does most of the hierarchy work. +- A dynamic HSL **score chip** (`hsl(score × 120, 70%, 45%)`) is the second-most-recognizable component after the brand violet — it is how the product communicates pass/fail. Identical formula on both themes; the chip's saturation works over cream and over near-black. +- **Light theme uses solid hex hairlines, not translucent violet.** This is the most important light-theme rule, and the one most often broken on first attempt: a violet alpha overlay disappears into a near-white page, so light surfaces require concrete warm-grey edges (`#e6dfd8` / `#d6cdbe` / `#c1b6a3`) to keep their boundaries. +- A complete domain token set exists for **chat bubbles** (5 semantic roles: user / bot / tool / reasoning / system), **compare slots** (3 per-model accent colors), and **KPI gradients** (4 named gradient pairs) — these are first-class brand tokens, not ad-hoc colors. +- A glassmorphic sticky top-nav (52 px, 12 px backdrop-blur, 1-px violet-to-transparent gradient hairline along the top edge) is the only "marketing-y" flourish the product allows itself. Dark uses translucent indigo glass; light uses translucent cream glass. + +## Colors + +> **Note on dual theme.** Every color token below has a *dark* (default) and *light* value. Hex pairs are listed as `dark / light`. Components reference tokens by name — never by raw hex — so theme switching is free. +> +> **Light values use SOLID hex for hairlines, not translucent violet.** This is the structural difference from earlier light-theme generations and the single most-broken light-theme rule. See `{colors.border-light}` below and *Elevation & Depth* for the reason. + +### Brand & Accent + +The accent family is **identical in spirit on both themes** — a single violet handles every conversion target. Slightly different hex values per theme (the dark violet is brighter to read on near-black; the light violet is a half-step deeper to hold weight against white cards) but the same brand voltage. + +- **Violet** (`{colors.accent}` — `#816DF8` / `#6c57e8`): The single brand color. Primary CTA fill, active nav fill, focus ring, brand-wordmark accent, table-header active-sort color. Used sparingly — should occupy under ~10 % of any screen. +- **Violet Deep** (`{colors.accent-dark}` — `#5B3FD6` / `#4f3ec8`): Hover state for primary CTAs. +- **Lavender** (`{colors.purple}` — `#a78bfa` / `#7c6be0`): The gradient companion stop; the second half of `{gradients.brand}`. +- **Violet Mist** (`{colors.accent-dim}` — `rgba(129,109,248,0.12)` / `rgba(108,87,232,0.14)`): The low-alpha violet used as pill background and focus-ring fill. Light theme runs slightly stronger (0.14 vs 0.12) because white cards need a touch more saturation to read the mist. +- **Violet Glow** (`{shadows.glow}` — `0 0 20px rgba(129,109,248,0.25)` / `0 0 20px rgba(108,87,232,0.22)`): The signature hover halo on primary buttons and active nav. Same effect, same magnitude, on both themes. + +### Surface + +Each theme operates with a 4-step surface ladder, sunken-to-elevated. The **ladder structure is shared**; the **material is different** — dark walks an indigo ladder, light walks a warm-cream ladder. The semantic of each step is the same: `bg-deep` is *below* the page, `bg-card` is the working surface, `bg-card2` is the elevated state. + +- **Page** (`{colors.bg}` — `#0c0c1a` / `#faf9f5`): The page body. Dark = near-black instrument-panel mood. Light = warm cream canvas (Claude-style), deliberately not cool grey-white — the cool variant (`#f5f6fa`) reads as "any other SaaS dashboard" and washes out white cards. +- **Sunken** (`{colors.bg-deep}` — `#09091a` / `#f0ebe1`): One step *below* the page — used for input wells, the deep-well that holds pill-style tab containers, and the icon tile in empty-states. Dark goes deeper-black; light goes warmer-cream. On both themes, inputs read as "wells" because they're one step below the surface that contains them. +- **Card** (`{colors.bg-card}` — `#12122b` / `#ffffff`): The default card / dialog / table surface. Dark = indigo card on near-black page. Light = pure white card on cream canvas — the cream-to-white contrast (ΔL ≈ 7) is what gives a light-theme card its lift, NOT a heavy shadow. +- **Card Elevated** (`{colors.bg-card2}` — `#16163a` / `#f5f0e7`): Hover state for clickable cards and rows; also the inactive-tab fill in pill-tab containers. Light theme's elevated state is warm-cream-soft — the elevated state is darker on dark theme but lighter-than-card-but-warmer on light theme (the white card with a soft-cream hover reads as "depressed into the cream canvas"). +- **Glass** (`{colors.surface-glass}` — `rgba(18,18,43,0.7)` / `rgba(250,249,245,0.80)`): Translucent surface for the sticky top-nav, used with a 12-px backdrop-blur. Light theme uses tinted cream glass (matches the canvas), NOT pure white — white glass on cream reads as a foreign sheet floating in space. + +### Text + +- **Ink** (`{colors.text}` — `#e2e8f0` / `#141413`): All headings, body, table cell values, button labels on non-filled surfaces. Light theme uses warm-near-black (`#141413`, ≈ the same value Claude.com uses) rather than a cool slate (`#1a1f2e`), so the text temperature matches the canvas temperature. +- **Muted** (`{colors.text-muted}` — `#8896aa` / `#6c6a64`): Secondary labels, nav-link inactive text, card-header micro-labels, button "ghost" idle text. *This is also the color section-eyebrow uppercase labels are set in.* Light theme uses warm-grey (`#6c6a64`) rather than cool-slate (`#5a6378`) to stay coherent with the warm canvas. +- **Dim** (`{colors.text-dim}` — `#7a8195` / `#8e8b82`): Lowest-priority text — placeholder text, timestamps in compact rows, table empty-state. **Contrast tuned to ~3.6 : 1** against `{colors.bg-card}` on both themes — sits just above the WCAG AA Large floor (3 : 1), still **below AA Normal (4.5 : 1)**. ⚠️ Reserve for ≥ 14 px non-essential metadata. Light theme uses a warm-grey at the same luminance step as the dark theme's cool-grey — the perceived hierarchy stays identical. +- **On Filled** (`{colors.on-filled}` — `#ffffff` / `#ffffff`): Text on `{colors.accent}` and other saturated fills. Identical on both themes — the violet CTA is dark enough on both that white text holds. + +### Hairlines (the structural difference between themes) + +- **Border** (`{colors.border}` — `rgba(129,109,248,0.10)` / `#e6dfd8`): The default 1-px card / input / divider boundary. **Dark uses translucent violet at 10 % alpha** because the near-black bg-to-card luminance step already does most of the boundary work — the violet hairline just tints it. **Light uses a SOLID warm-grey hex** (`#e6dfd8`, Claude-style cream-hairline) because the white-card-on-cream luminance step is gentle enough that a translucent violet overlay disappears into the page. Borders on light theme are concrete materials, not tints. +- **Border Emphasized** (`{colors.border-md}` — `rgba(129,109,248,0.18)` / `#d6cdbe`): One step stronger — used on form inputs after focus, on the active-state of hover cards, on the boundary between a card and a nested section. +- **Border Strong** (`{colors.border-strong}` — `rgba(129,109,248,0.28)` / `#c1b6a3`): The strongest boundary — used by `{components.card-hover}` on hover lift, and by elevated cards in modal contexts. On both themes this is the "this thing is grabbing attention" hairline. + +### Semantic + +- **Success** (`{colors.success}` — `#10b981` / `#059669`) / **Success Bg** (`rgba(16,185,129,0.08)`) / **Success Border** (`rgba(16,185,129,0.20)`): Confirmed / passed states; success toasts; chat-bot bubble border. +- **Warning** (`{colors.warning}` — `#f59e0b` / `#d97706`) / **Warning Bg** (`rgba(245,158,11,0.08)`) / **Warning Border** (`rgba(245,158,11,0.20)`): Pending / caution; tool-call chat bubbles. +- **Danger** (`{colors.danger}` — `#ef4444` / `#dc2626`) / **Danger Bg** (`rgba(239,68,68,0.08)`) / **Danger Border** (`rgba(239,68,68,0.20)`): Errors, fail badges, validation, destructive actions. +- **Info** (`{colors.info}` — `#60a5fa` / `#3b82f6`): Latency chart series, informational toasts. +- **Pass** (`{colors.pass}` — `rgb(45,104,62)` / `rgb(16,108,55)`) / **Fail** (`{colors.fail}` — `rgb(151,31,44)` / `rgb(180,30,42)`): Deep saturated greens / crimsons for boolean pass/fail badges where the tone needs more weight than the soft semantic family. + +### Score Gradient (Signature) + +The product's emotional core. A 0-1 score maps to **`hsl(score × 120, 70%, 45%)`**: 0 → red, 0.5 → yellow, 1 → green. Used as both foreground and translucent background on score chips, dataset chips, and group-header best-score callouts. This is computed inline (`scoreColor` / `scoreBg` helpers), never stored as a static palette. **Treat the formula as a brand asset** — do not reskin to a 5-step bucket, do not introduce a 4th hue. + +Foreground uses HSL for predictable hue progression; background uses an RGB-interpolated stop pair (rich mid-tones, more saturated yellows) for visual continuity with the existing chip/badge appearance. Do not rewrite the bg to HSL alpha — the mid-tone shift will break existing visual reads. + +**Per-theme legibility knobs**: `hsl(h 70%, 45%)` reads cleanly on the near-black dark canvas but the yellow mid (hue ≈ 60) collapses into warm-cream `#faf9f5` on the light theme. To keep the brand formula intact while fixing legibility, three CSS vars scope per-theme: + +- **`--score-fg-s`** — saturation for the HSL foreground. Dark: `70%`. Light: `85%` (the +15pt bump compensates for the lightness compression at hue ≈ 60; without it, mid scores render as washed-out olive on cream). +- **`--score-fg-l`** — lightness for the HSL foreground. Dark: `45%`. Light: `32%` (darker olive/forest/maroon read cleanly on cream). +- **`--score-bg-a-mul`** — alpha multiplier for the RGB-interpolated bg. Dark: `1`. Light: `1.6` (boosts washed-out yellows so the pill bg is actually visible). + +`scoreColor` / `scoreBg` emit CSS expressions (`hsl(h var(--score-fg-s) var(--score-fg-l))`, `rgb(r g b / calc(α * var(--score-bg-a-mul)))`) — the browser picks the right value per active theme. **Do not** hardcode these in JS or fork separate light/dark helpers. + +**Score Ring** (`{components.score-ring}` — SVG circular progress used in `` and the "Overall Score" callout in ``): the active arc is `stroke={scoreColor(score)}`. Stroke width must be **≥6 px** for the 48 × 48 mini ring and **8 px** for the 72 × 80 summary ring — anything thinner reduces the colored area to the point where the mid-hue olive stops carrying. Background arc uses `var(--border)` for a neutral track. + +### Compare Slots + +Three per-model accent colors used to tag side-by-side model comparisons. Each slot has a `dot`, `border`, `bg`, and `bg-header` tint at ~10-30 % alpha: + +- **Slot 0** (`{compare.0.dot}` — `#818cf8` / `#6c57e8`): Indigo. +- **Slot 1** (`{compare.1.dot}` — `#34d399` / `#0a8a6e`): Mint. +- **Slot 2** (`{compare.2.dot}` — `#fbbf24` / `#d97706`): Amber. + +If a comparison view exceeds 3 models, *do not invent a 4th brand color* — collapse into a numbered legend instead. + +### Chat Bubble Roles + +Five semantic roles, each with a complete 7-token set (`bg`, `bg-hl`, `border`, `border-hl`, `icon-bg`, `icon-border`, `color`): + +- **User** — Indigo family (`#818cf8` color). +- **Bot** — Emerald family (`#34d399` color). +- **Tool** — Amber family (`#fbbf24` color). +- **Reasoning** — Dim emerald (lower-saturation companion to Bot). +- **System** — Slate gray (`rgba(148,163,184,*)`). + +Bubble containers are `{rounded.md}` with the role's tint background and border; hover/highlight states use the `*-hl` variants. + +### KPI Gradients + +Four named linear gradients for the four hero KPI tiles on the dashboard: + +- **Indigo→Violet** (`{gradients.kpi-0}` — `linear-gradient(135deg, #6366f1, #8b5cf6)`) +- **Emerald→Teal** (`{gradients.kpi-1}` — `linear-gradient(135deg, #10b981, #06b6d4)`) +- **Amber→Orange** (`{gradients.kpi-2}` — `linear-gradient(135deg, #f59e0b, #f97316)`) +- **Pink→Violet** (`{gradients.kpi-3}` — `linear-gradient(135deg, #ec4899, #8b5cf6)`) + +Always applied to the 40 × 40 `{rounded.md}` icon tile inside a `{components.kpi-card}`. **Same gradient values on both themes** — they're saturated enough to work over either canvas. + +### Chart Palette (Perf Metrics) + +Four hue tokens used to mark perf-metric series (latency / TTFT / TPOT / token-usage) across the KPI strip, the chart series legends, and the percentile-table accent headers in ``. **The two themes use different RGB values for the same hue** — unlike the KPI gradients, these aren't shared across themes: + +- **Latency** (`{chart.latency}` — `#60a5fa` dark / `#2563eb` light) +- **TTFT** (`{chart.ttft}` — `#34d399` dark / `#047857` light) +- **TPOT** (`{chart.tpot}` — `#a78bfa` dark / `#5b48e0` light) +- **Token** (`{chart.token}` — `#94a3b8` dark / `#5a6378` light) + +**Why the fork**: warm-cream bg (`{colors.bg-card-light}` / `{colors.bg-deep-light}`) compresses cool hues via simultaneous contrast. Mid-tone slate blue / violet that pop on near-black dark canvases read as washed-out lavender on cream. The light palette pulls each hue darker **and** more saturated to restore the data-bearing punch of the same hue family. Same hue identity, different RGB — not a re-tint. + +**KPI strip surface**: the strip in `` uses `{colors.bg-card}` (matching the outer card), not `{colors.bg-deep}`. On light theme, `{colors.bg-deep}` is even warmer than the cards and pushes the chart hues into the warm-on-warm range — losing the contrast that makes the colored numbers carry. Visual separation comes from the border + dividers, not bg differentiation. + +### Brand Gradients (Decorative) + +- **Brand** (`{gradients.brand}` — `linear-gradient(135deg, #816DF8 → #a78bfa)`): For `gradient-text` and large brand moments. +- **Accent** (`{gradients.accent}` — `linear-gradient(135deg, #0F9C7E → #06b6d4)`): For the optional emerald-to-cyan accent text. +- **Surface** (`{gradients.surface}` — `linear-gradient(135deg, rgba(129,109,248,0.08) → rgba(167,139,250,0.05))`): The subtle violet wash layered behind KPI cards via `::before`. + +## Typography + +### Font Family + +Two cross-platform **system font stacks** carry the entire system — each OS resolves to its own native UI face. No `@font-face` is loaded; this is deliberate. Each stack starts with the modern CSS `system-ui` / `ui-monospace` generic family and falls back to named faces for older browsers and per-OS targets: + +1. **System sans** (`{typography.font-sans}` — `system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif`): Every display, body, button, link, and label. Working weights: 400 / 500 / 600 / 700. Resolves to **SF Pro** on macOS / iOS, **Segoe UI** on Windows, **Roboto** on Android / ChromeOS, and the desktop default (Cantarell / Noto / DejaVu) on Linux. +2. **System mono** (`{typography.font-mono}` — `ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, "Liberation Mono", "DejaVu Sans Mono", "Courier New", monospace`): Timestamps, scores, model IDs, and any tabular-numeric data. Weight 400 only. Resolves to **SF Mono** on macOS, **Consolas** on Windows, **Liberation Mono / DejaVu Sans Mono** on Linux. The full chain matters — without the OS-specific named fallbacks, non-mac users land on `Courier New`, breaking the tabular-numeric look. + +Antialiasing is forced (`-webkit-font-smoothing: antialiased`). No web font is loaded — the brand reads as "native developer tool, not marketing site" precisely because of this. The trade-off is per-OS rendering variance; if pixel-identical screenshots across platforms are required, see *Note on Font Substitutes* below. + +### Hierarchy + +| Token | Size | Weight | Tracking | Use | +|---|---|---|---|---| +| `{typography.display-xl}` | 24px | 700 | tight | KPI value, hero number (dashboard `{components.kpi-card}`). | +| `{typography.title-md}` | 16px | 700 | normal | Card-title model name, group-header titles, brand wordmark. | +| `{typography.body-sm}` | 14px | 400 / 500 | normal | Default body text, button-md label, table-cell text, paragraph copy. | +| `{typography.label-xs}` | 12px | 600 | wider | **UPPERCASE** card-section header, form label, badge text — the brand's signature eyebrow. | +| `{typography.body-xs}` | 12px | 400 / 500 | normal | Empty-state hint, pill / badge body, mobile-nav link. | +| `{typography.table-xs}` | 10px | 600 | wider | **UPPERCASE** table-header micro-text — whispers, doesn't shout. | +| `{typography.caption-mono}` | 12px | 400 (mono) | normal | Timestamps, score values, dataset names in chips. | +| `{typography.code}` | 13–14px | 400 (mono) | normal | Log viewer, JSON viewer, terminal-style output. | +| `{typography.button-sm}` | 12px | 500 | normal | `{components.button}` `size="sm"`. | +| `{typography.button-md}` | 14px | 500 | normal | `{components.button}` `size="md"` (default). | +| `{typography.button-lg}` | 16px | 500 | normal | `{components.button}` `size="lg"` — for hero callouts only. | + +### Principles + +- **UPPERCASE + `tracking-wider` is the eyebrow voice — never the headline voice.** It marks "this introduces a region." Card titles, model names, and KPI labels stay sentence-case (or lower-case for the brand wordmark's lowercase "v"). +- **`tracking-tight` is reserved for the display tier** — KPI numbers and the brand wordmark only. It tells the reader "this is set big on purpose." +- **`tabular-nums` + `font-mono` everywhere numbers must align** — KPI values, score chips, timestamps, percentage cells. Treat numbers as data, not prose. +- **Weight 700 is the display ceiling.** No `font-extrabold` / `font-black`. The brand reads as a calmer system because of this. +- **Line-heights inherit Tailwind defaults** (1.5 for body, 1.25 for headings). Don't override globally; rely on padding for vertical rhythm in tight stacks (cards, chips). +- **No web font.** The system stack is the system. Loading Inter or Geist on top would break the "native console" feel. + +### Note on Font Substitutes + +EvalScope uses the OS-native stack, so there are no proprietary faces to substitute. If a future skin needs to enforce a *single* face across all OSes for screenshot consistency: + +- **Sans substitute** — *Inter* (400 / 500 / 600 / 700) is the closest stylistic match to the SF-on-macOS rendering; preserves the geometric / neutral character. +- **Mono substitute** — *JetBrains Mono* (400) at 12–14 px matches the technical voice well; *IBM Plex Mono* is a close second. + +## Layout + +### Spacing System + +- **Base unit**: 4 px (Tailwind's default scale). +- **Tokens** (Tailwind-aligned): + - `{spacing.xs}` 4 px · `{spacing.sm}` 8 px · `{spacing.md}` 12 px · `{spacing.lg}` 16 px · `{spacing.xl}` 20 px · `{spacing.2xl}` 24 px · `{spacing.3xl}` 32 px · `{spacing.4xl}` 48 px · `{spacing.5xl}` 64 px. +- **Page padding**: 16 px horizontal, 20 px vertical (`px-4 py-5`) at all breakpoints — *no responsive expansion*. The 1600 px container does the breathing for us. +- **Card interior padding**: 20 px body (`{spacing.xl}`), 12 px header strips (`{spacing.md} {spacing.xl}`). +- **Inline gap**: `{spacing.md}` (12 px) for component rows inside a card, `{spacing.xl}` (20 px) for inter-section gaps between major page blocks. +- **Pill / chip gap**: `{spacing.sm}` (6-8 px) — tight, scan-friendly. Pills are meant to wrap. + +### Grid & Container + +- **Max width**: `1600px` (`max-w-[1600px]`). Centered (`mx-auto`). Wide enough for 4-up KPI strip + side-by-side comparison; narrow enough that line-length never sprawls. +- **Column patterns**: + - **KPI strip**: 4-up at `lg+`, 2-up below (`grid-cols-2 lg:grid-cols-4`). + - **Compare view**: 2-3 columns at desktop, vertical stack on mobile. + - **Eval timeline**: single column of full-width rows, no grid. + - **Form pairs**: `grid-cols-1 md:grid-cols-2` for label-value pairs. +- **Gutters**: 16 px horizontal at all sizes. + +### Whitespace Philosophy + +Whitespace separates the *bands* — not the components inside a band. Section spacing is generous (`flex flex-col gap-5` → 20 px between major blocks); card interiors are tight (`gap-2` / `gap-3` between rows inside a card). The page reads as engineered — *large gaps + tight interior, never the other way around*. The dark page lets cards float visually without needing margin to assert themselves; the hairline border does the bordering work. + +The brand's voice is **information-rich but uncluttered** — a typical dashboard packs a 4-tile KPI strip, a 30-row sortable eval list, search + sort + view-toggle controls, and the page still feels scannable because: +1. Tokens enforce a 3-step text hierarchy on every row. +2. UPPERCASE eyebrows visually section the layout without horizontal rules. +3. Score chips compress a percentage + benchmark name + color signal into 6 characters of mono. + +### Responsive Strategy + +#### Breakpoints (Tailwind defaults) + +| Name | Width | Key Changes | +|---|---|---| +| Mobile | < 640px | Nav collapses to hamburger drawer; KPI grid drops to 2-up; all controls stack. | +| Small | 640–767px | GitHub icon and locale toggle appear in nav; KPI still 2-up. | +| Tablet | 768–1023px | Nav switches to **icon-only mode** with tooltips; main content full-width. | +| Desktop | 1024–1279px | Full pill-style nav with icon + label; KPI strip goes 4-up. | +| Wide | ≥ 1280px | Container holds at `max-w-[1600px]`; bands stretch but content centers. | + +#### Touch Targets + +The top-nav icon-only buttons (tablet) are 32 × 32 — *under the 44 × 44 WCAG floor*. This is a known compromise for the developer-tool density; on actual touch devices, hit areas are extended via padding. Primary buttons reach ~36 px tall in `md` size and ~44 px in `lg` — meet the floor at `lg`. + +#### Collapsing Strategy + +- **Nav**: Desktop = pill-row with icon + label; tablet = icon-only pills; mobile = logo + hamburger toggling a stacked drop-down (`max-height` animated, 300 ms). +- **KPI strip**: 4-up → 2-up below `lg`. Each tile keeps its `{rounded.md}` 12-px shape and 20-px padding. +- **Eval timeline**: Single column at all sizes — already a vertical list. +- **Forms**: Two-column label/value at `md+`, single column below. +- **Table**: Horizontal scroll wrapper preserves all columns rather than dropping them. The card-shell border keeps the scroll area framed. + +#### Image / Icon Behavior + +- **Iconography**: `lucide-react`, almost always 14–18 px, color inherits `currentColor`. The only non-Lucide mark is the brand SVG in the top-nav (a hand-drawn triangle with an amber check, rendered with `currentColor` so it follows theme). +- **No marketing imagery**: This is a console — no hero photo, no customer-logo strip, no illustrated empty-states. Empty states use a single Lucide icon in a 64 × 64 `{rounded.lg}` deep-well tile. +- **Charts**: Plotly-based, theme-aware — chart series colors are `{colors.chart-*}` tokens (latency / TTFT / TPOT / token). + +## Elevation & Depth + +| Level | Treatment | Use | +|---|---|---| +| **L0 — Flat** | No border, no shadow. | Page body, large empty regions, full-bleed dark sections. | +| **L1 — Hairline** | 1-px `{colors.border}` only. Dark: **translucent violet at 10 % alpha** (the near-black page lets even a faint hairline read; the slight violet tint stays on-brand). Light: **solid warm-grey hex** `#e6dfd8` (translucent violet at any plausible alpha composites to invisible against a white card on cream — solid hex is the only thing that reads). | Default content cards (`{components.card}`), table chrome, form inputs. | +| **L2 — Lifted** | `{shadows.sm}` + L1 hairline. Dark: single deep drop `0 2px 8px rgba(0,0,0,0.4)`. Light: two-stop stack `0 1px 2px rgba(20,20,19,0.04), 0 4px 12px rgba(20,20,19,0.06)` — warm-ink tinted (matches `{colors.text-light}`), not slate. Slate-tinted drops on cream read as a cool-grey smudge against the warm canvas. | KPI cards, path-bar at the top of the dashboard, anything that needs to read as "above the page." | +| **L3 — Floating** | `{shadows.md}` + L1 hairline. Dark `0 4px 20px rgba(0,0,0,0.55)`. Light `0 4px 16px rgba(20,20,19,0.07), 0 12px 32px rgba(20,20,19,0.05)`. | Default for the primary card variants when the page already has heavy chrome. | +| **L4 — Elevated** | `{shadows.lg}` + `{colors.border-strong}`. Dark `0 8px 40px rgba(0,0,0,0.6)`. Light `0 12px 24px rgba(20,20,19,0.09), 0 24px 48px rgba(20,20,19,0.07)`. | Hover state for clickable cards, modal / dialog surfaces, dropdown menus. | +| **L5 — Glow** | `{shadows.glow}` on top of L1-L3. Dark `0 0 20px rgba(129,109,248,0.25)`. Light `0 0 20px rgba(108,87,232,0.22)`. | The signature violet halo — primary button hover, active nav pill, sometimes active tab. Identical magnitude across themes — the glow is the brand's interaction constant. | + +**Brand rule (depth)**: dark and light themes are *not* the same shadow scaled down. Dark surfaces use a **single deep drop** (`rgba(0,0,0,0.4-0.6)`) — the near-black canvas eats soft drops. Light surfaces use a **two-stop stack** tinted with warm-ink `rgba(20,20,19,*)` (the same hex as `{colors.text-light}`), NOT slate `rgba(15,23,42,*)` — the warm-ink tint stays coherent with the cream canvas and the warm-ink body text. Light cards lift through the *combination* of cream-to-white surface contrast + a solid warm-grey hairline + the two-stop warm-ink shadow stack — no single layer carries the weight. + +**Brand rule (hairlines)**: light theme borders are **SOLID HEX**, not translucent violet at any alpha. The earlier light-theme generation used `rgba(108,87,232,0.20-0.40)` and the violet alpha composited to within a few luminance steps of the white card — borders effectively dissolved, especially on outline buttons (`Go to Index`, `Find msg id`) and on input rings (`Score Threshold` field). The current system uses three concrete warm-grey hex values (`#e6dfd8` / `#d6cdbe` / `#c1b6a3`) — each step is a definite material, not a tint. The dark theme keeps translucent violet at 10-28 % because the indigo-bg-to-card luminance delta is already doing most of the boundary work; light theme has no such luminance assist and must rely on the hairline alone. + +### Decorative Depth + +- **Backdrop-blur**: 12 px on `{colors.surface-glass}` for the sticky top-nav. This is the only blur effect in the system. +- **Hairline gradient line**: The top of the nav draws a 1-px `transparent → {colors.accent} → transparent` line at 40 % opacity. The closest the design comes to "decoration." +- **Card lift**: Hover lifts L1/L2 cards `-2px` (`{components.card-hover}`) or `-3px` (`{components.kpi-card}`), simultaneously upgrading their shadow ladder one step. +- **Gradient text**: `.gradient-text` and `.gradient-text-accent` utilities apply `{gradients.brand}` / `{gradients.accent}` to text via `background-clip: text`. Use sparingly — reserved for hero brand moments, never for body or table cells. + +## Shapes + +### Border Radius Scale + +| Token | Value | Use | +|---|---|---| +| `{rounded.none}` | 0px | Full-bleed sections, table cell interiors. | +| `{rounded.xs}` | 4px | `{tokens.radius-xs}` — tightest inline pill (rarely used directly). | +| `{rounded.sm}` | 8px | `{tokens.radius-sm}` — buttons (sm/md), inputs, tabs, view-toggle buttons. Most in-product chrome lives here. | +| `{rounded.md}` | 12px | `{tokens.radius}` — **default card radius**. Cards, KPI tiles, group-header containers, large buttons. | +| `{rounded.lg}` | 16px | `{tokens.radius-lg}` — empty-state icon tile, KPI icon tile. | +| `{rounded.xl}` | 20px | `{tokens.radius-xl}` — reserved (used by occasional rounded-2xl Tailwind class on welcome / empty states). | +| `{rounded.full}` | 9999px | Badges, chips, score pills, mobile-nav icon buttons. | + +**No pill (100-px) shape.** Unlike Vercel's marketing pill, EvalScope is an in-product surface; all CTAs use `{rounded.sm}` 8 px. Pills (`{rounded.full}`) are exclusively for *data* — badges, chips, score indicators. + +### "Photography" — Iconography Geometry + +- **Brand mark**: Hand-drawn SVG triangle with an amber check; rendered inline at 28 × 25 px with `currentColor` so it follows theme. +- **Lucide icons**: 14 px in dense lists, 16 px in nav, 18 px in form controls, 28 px in empty-state hero tiles. Stroke width 1.5–2. +- **KPI icon tile**: 40 × 40, `{rounded.md}`, filled with one of `{gradients.kpi-0..3}`, white icon centered. +- **Chart**: Plotly canvas, no rounded corners; sits inside a `{rounded.md}` card frame. + +## Components + +### Buttons + +The brand operates with three button variants — *all in-product scale*, no marketing pill: + +**`{components.button.primary}`** — the canonical violet CTA. +- Background `{colors.accent}`, text `{colors.on-filled}`, shape `{rounded.sm}` 8 px (md/sm) or `{rounded.md}` 12 px (lg). Hover adds `{shadows.glow}` violet halo + `bg → {colors.accent-dark}`. Press scales to 0.98. **The glow IS the brand interaction.** + +**`{components.button.ghost}`** — the transparent secondary. +- Transparent background → `{colors.bg-card}` on hover → `{colors.bg-card2}` on active. Text `{colors.text}`. Used inside top-nav, toolbars, and inline icon-buttons. + +**`{components.button.outline}`** — the hairline tertiary. +- Transparent background with 1-px `{colors.border-md}` border. Hover swaps both border and text to `{colors.accent}` (no fill). For "alternative action" CTAs. + +**Sizes**: +- `sm` — 12-px text (`{typography.button-sm}`), 6/12 padding, `{rounded.sm}`, ~28 px tall. +- `md` — 14-px text (`{typography.button-md}`), 8/16 padding, `{rounded.sm}`, ~36 px tall *(default)*. +- `lg` — 16-px text (`{typography.button-lg}`), 10/24 padding, `{rounded.md}`, ~44 px tall. + +Disabled = `opacity: 0.5` + `cursor: not-allowed`. Transitions use `{tokens.transition}` (0.18 s ease) consistently — never custom per button. + +### Cards & Containers + +**`{components.card}`** — the canonical card. +- Background `{colors.bg-card}`, text `{colors.text}`, padding 20 px (`{spacing.xl}`), shape `{rounded.md}` 12 px, border 1-px `{colors.border}`, shadow `{shadows.sm}` (L2). Optional **uppercase section header** in a 12 px / 20 px header strip with a bottom border. Collapsible variant rotates a chevron 90 °. + +**`{components.card-hover}`** — utility applied to clickable cards. +- Adds `-2 px translateY` on hover, upgrades shadow to `{shadows.lg}` (L4), strengthens border to `{colors.border-strong}`. All transitions in `{tokens.transition}`. + +**`{components.kpi-card}`** — the dashboard hero metric tile. +- Same chrome as `{components.card}` but layered with `{gradients.surface}` (subtle violet wash, 5-8 % alpha) via `::before`. Hosts a 40 × 40 `{rounded.md}` icon tile filled with `{gradients.kpi-0..3}`. Hover lifts `-3 px` and ramps to L4 shadow. Stagger-animated on first paint (60 ms steps). + +**`{components.card-glass}`** — the glassmorphic surface used by the top-nav. +- `{colors.surface-glass}` background + 12-px backdrop-blur. Always combined with a 1-px hairline border. Reserve for sticky-positioned surfaces — diffuse blur is performance-sensitive. + +**`{components.row-card}`** — borderless button styled as a list row (the eval-run rows on the dashboard). +- Background `{colors.bg-card}`, padding 16-20 px, `{rounded.md}`, 1-px `{colors.border}` border. Hover only changes border tint to `{colors.border-md}` — *no transform* — kept calm for dense lists. + +### Inputs & Forms + +**`{components.input}`** — the canonical text input. +- Background `{colors.bg-deep}` (one step *below* its container — inputs read as "wells"), 1-px `{colors.border}` border, text `{colors.text}` in `{typography.body-sm}`. Focus: border → `{colors.accent}` + 1-px ring in `{colors.accent-dim}`. **Soft halo, never harsh outline.** Padding 8/12, `{rounded.sm}` 8 px. +- Error state: border + ring swap to the danger family; 12 px `{colors.danger}` helper line below. + +**`{components.label}`** — input label. +- 12 px, `font-medium`, **UPPERCASE**, `tracking-wider`, `{colors.text-muted}`, placed above the input with 6-px gap. + +**`{components.select}`** / **`{components.search-input}`** — same chrome as `{components.input}`. The select uses the native ` { setModel(e.target.value); if (errors.model) setErrors((p) => ({ ...p, model: '' })) }} + className={inputClass(errors.model)} + placeholder="Qwen/Qwen2.5-0.5B-Instruct" + /> + + + {/* Datasets with autocomplete */} + +
+ handleDatasetChange(e.target.value)} + onFocus={() => { if (filteredSuggestions.length) setShowSuggestions(true) }} + className={inputClass(errors.datasets)} + placeholder="gsm8k, arc" + /> + {showSuggestions && ( +
+ {filteredSuggestions.map((name) => ( + + ))} +
+ )} +
+ + + + setApiUrl(e.target.value)} className={FORM_INPUT_CLASS} placeholder="http://localhost:8000/v1" /> + + + + setApiKey(e.target.value)} className={FORM_INPUT_CLASS} placeholder="sk-..." /> + + + + setLimit(e.target.value)} className={FORM_INPUT_CLASS} /> + + + + setEvalBatchSize(e.target.value)} className={FORM_INPUT_CLASS} /> + + + + {/* More params toggle */} + + + {showMore && ( + +
+ + setRepeats(e.target.value)} className={FORM_INPUT_CLASS} /> + + + setTimeout_(e.target.value)} className={FORM_INPUT_CLASS} /> + +
+ +
+ + setTemperature(e.target.value)} className={FORM_INPUT_CLASS} /> + + + setTopP(e.target.value)} className={FORM_INPUT_CLASS} /> + + + setMaxTokens(e.target.value)} className={FORM_INPUT_CLASS} /> + + + setTopK(e.target.value)} className={FORM_INPUT_CLASS} /> + +
+ +