178 lines
3.2 KiB
Markdown
178 lines
3.2 KiB
Markdown
# Sora Benchmarks
|
|
|
|
## 1. 环境安装
|
|
|
|
```bash
|
|
conda create -n evalscope python==3.12 -y
|
|
conda activate evalscope
|
|
|
|
cd evalscope
|
|
pip install -e .
|
|
|
|
# 安装特定 benchmark 依赖
|
|
pip install 'evalscope[terminal_bench,swe_bench,openai_mrcr]' \
|
|
'git+https://github.com/sierra-research/tau2-bench@v0.2.0'
|
|
pip install 'evalscope[sandbox]'
|
|
pip install 'swebench==4.1.0'
|
|
```
|
|
|
|
## 2. 常用 Benchmarks
|
|
|
|
```python
|
|
benchmark_categories = {
|
|
"代码与工程": [
|
|
"terminal_bench_v2",
|
|
"live_code_bench",
|
|
"swe_bench_multilingual_agentic",
|
|
"swe_bench_pro",
|
|
"swe_bench_verified",
|
|
"bigcodebench",
|
|
"humaneval",
|
|
],
|
|
"推理与数学": [
|
|
"gpqa_diamond",
|
|
"hle",
|
|
"aime24",
|
|
"aime25",
|
|
"aime26",
|
|
"hmmt26",
|
|
"imo_answerbench",
|
|
"super_gpqa",
|
|
"gsm8k",
|
|
"competition_math",
|
|
],
|
|
"智能体与工具": [
|
|
"browsecomp",
|
|
"mcp_atlas",
|
|
"tau2_bench",
|
|
],
|
|
"知识与语言理解": [
|
|
"mmlu_pro",
|
|
"simple_qa",
|
|
"arc",
|
|
"bbh",
|
|
"cmmlu",
|
|
"drop",
|
|
"hellaswag",
|
|
"mmlu",
|
|
"trivia_qa",
|
|
"winogrande",
|
|
],
|
|
"长上下文": [
|
|
"longbench_v2",
|
|
"openai_mrcr",
|
|
],
|
|
}
|
|
```
|
|
|
|
## 3. 分组运行
|
|
|
|
```python
|
|
group_1 = [
|
|
'terminal_bench_v2',
|
|
'gpqa_diamond',
|
|
'hle',
|
|
'aime24',
|
|
'aime25',
|
|
'mmlu_pro',
|
|
'simple_qa',
|
|
'arc',
|
|
'bbh',
|
|
'browsecomp',
|
|
]
|
|
|
|
group_2 = [
|
|
'live_code_bench',
|
|
'swe_bench_pro',
|
|
'aime26',
|
|
'hmmt26',
|
|
'imo_answerbench',
|
|
'super_gpqa',
|
|
'drop',
|
|
'hellaswag',
|
|
'mmlu',
|
|
'openai_mrcr',
|
|
'mcp_atlas',
|
|
]
|
|
|
|
group_3 = [
|
|
'swe_bench_multilingual_agentic',
|
|
'swe_bench_verified',
|
|
'bigcodebench',
|
|
'humaneval',
|
|
'gsm8k',
|
|
'competition_math',
|
|
'cmmlu',
|
|
'trivia_qa',
|
|
'winogrande',
|
|
'longbench_v2',
|
|
'tau2_bench',
|
|
]
|
|
```
|
|
|
|
## 4. 运行示例
|
|
|
|
```python
|
|
from evalscope import run_task, TaskConfig
|
|
|
|
datasets = [
|
|
'live_code_bench',
|
|
'swe_bench_pro',
|
|
'aime26',
|
|
'hmmt26',
|
|
'imo_answerbench',
|
|
'super_gpqa',
|
|
'drop',
|
|
'hellaswag',
|
|
'mmlu',
|
|
'openai_mrcr',
|
|
'mcp_atlas',
|
|
]
|
|
|
|
task_cfg = TaskConfig(
|
|
collect_perf=True,
|
|
work_dir='/data1/sora/benchmarks/temp/output',
|
|
use_cache='/data1/sora/benchmarks/temp/output',
|
|
no_timestamp=True,
|
|
model='',
|
|
api_url='http://localhost:30000/v1',
|
|
eval_type='openai_api',
|
|
datasets=datasets,
|
|
dataset_dir='/data1/sora/benchmarks/bash/datasets',
|
|
generation_config={
|
|
'temperature': 0.0,
|
|
'stream': True,
|
|
},
|
|
eval_batch_size=1,
|
|
judge_model_args={
|
|
"model_id": "deepseek-v4-flash",
|
|
"api_url": "https://api.deepseek.com/v1",
|
|
"api_key": "<YOUR_DEEPSEEK_API_KEY>",
|
|
"eval_type": "openai_api",
|
|
"generation_config": {
|
|
"temperature": 0.0,
|
|
"max_tokens": 1024 * 10,
|
|
},
|
|
}
|
|
)
|
|
|
|
run_task(task_cfg)
|
|
```
|
|
|
|
## 5. 评测结果
|
|
|
|
P800 模型能力评测结果保存在 `temp/output_*` 目录下。
|
|
|
|
## 6. UI 界面
|
|
|
|
```bash
|
|
pip install flask sse_starlette
|
|
cd evalscope/web
|
|
npm install
|
|
npm run build
|
|
cd /data1/sora/benchmarks
|
|
evalscope service
|
|
```
|
|
|
|
访问 `http://127.0.0.1:9000/dashboard`。
|