# Sora Benchmarks

## 1. 环境安装

```bash
conda create -n evalscope python==3.12 -y
conda activate evalscope

cd evalscope
pip install -e .

# 安装特定 benchmark 依赖
pip install 'evalscope[terminal_bench,swe_bench,openai_mrcr]' \
            'git+https://github.com/sierra-research/tau2-bench@v0.2.0'
pip install 'evalscope[sandbox]'
pip install 'swebench==4.1.0'
```

## 2. 常用 Benchmarks

```python
benchmark_categories = {
    "代码与工程": [
        "terminal_bench_v2",
        "live_code_bench",
        "swe_bench_multilingual_agentic",
        "swe_bench_pro",
        "swe_bench_verified",
        "bigcodebench",
        "humaneval",
    ],
    "推理与数学": [
        "gpqa_diamond",
        "hle",
        "aime24",
        "aime25",
        "aime26",
        "hmmt26",
        "imo_answerbench",
        "super_gpqa",
        "gsm8k",
        "competition_math",
    ],
    "智能体与工具": [
        "browsecomp",
        "mcp_atlas",
        "tau2_bench",
    ],
    "知识与语言理解": [
        "mmlu_pro",
        "simple_qa",
        "arc",
        "bbh",
        "cmmlu",
        "drop",
        "hellaswag",
        "mmlu",
        "trivia_qa",
        "winogrande",
    ],
    "长上下文": [
        "longbench_v2",
        "openai_mrcr",
    ],
}
```

## 3. 分组运行

```python
group_1 = [
    'terminal_bench_v2',
    'gpqa_diamond',
    'hle',
    'aime24',
    'aime25',
    'mmlu_pro',
    'simple_qa',
    'arc',
    'bbh',
    'browsecomp',
]

group_2 = [
    'live_code_bench',
    'swe_bench_pro',
    'aime26',
    'hmmt26',
    'imo_answerbench',
    'super_gpqa',
    'drop',
    'hellaswag',
    'mmlu',
    'openai_mrcr',
    'mcp_atlas',
]

group_3 = [
    'swe_bench_multilingual_agentic',
    'swe_bench_verified',
    'bigcodebench',
    'humaneval',
    'gsm8k',
    'competition_math',
    'cmmlu',
    'trivia_qa',
    'winogrande',
    'longbench_v2',
    'tau2_bench',
]
```

## 4. 运行示例

```python
from evalscope import run_task, TaskConfig

datasets = [
    'live_code_bench',
    'swe_bench_pro',
    'aime26',
    'hmmt26',
    'imo_answerbench',
    'super_gpqa',
    'drop',
    'hellaswag',
    'mmlu',
    'openai_mrcr',
    'mcp_atlas',
]

task_cfg = TaskConfig(
    collect_perf=True,
    work_dir='/data1/sora/benchmarks/temp/output',
    use_cache='/data1/sora/benchmarks/temp/output',
    no_timestamp=True,
    model='',
    api_url='http://localhost:30000/v1',
    eval_type='openai_api',
    datasets=datasets,
    dataset_dir='/data1/sora/benchmarks/bash/datasets',
    generation_config={
        'temperature': 0.0,
        'stream': True,
    },
    eval_batch_size=1,
    judge_model_args={
        "model_id": "deepseek-v4-flash",
        "api_url": "https://api.deepseek.com/v1",
        "api_key": "<YOUR_DEEPSEEK_API_KEY>",
        "eval_type": "openai_api",
        "generation_config": {
            "temperature": 0.0,
            "max_tokens": 1024 * 10,
        },
    }
)

run_task(task_cfg)
```

## 5. 评测结果

P800 模型能力评测结果保存在 `temp/output_*` 目录下。

## 6. UI 界面

```bash
pip install flask sse_starlette
cd evalscope/web
npm install
npm run build
cd /data1/sora/benchmarks
evalscope service
```

访问 `http://127.0.0.1:9000/dashboard`。