From cfa58f869f7989ce718a68436a7792f142a6d611 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Tue, 21 Jul 2026 09:32:49 +0000 Subject: [PATCH] all --- .dockerignore | 26 + .gitignore | 8 +- DOCKER_BUILD.md | 554 ++++++++++++++++++ DOCKER_README.md | 113 ++++ Dockerfile.withcode | 19 + EVALUATION_GUIDE.md | 266 +++++++++ HOWTO_EVAL_NEW_MODEL.md | 216 +++++++ README.md | 177 ------ .../dpv4-int8_nothinking.yaml} | 188 +++--- bash/fill_simpleqa_bfcl.py | 94 +++ bash/generation_config_thinking.yaml | 249 -------- bash/rerun_bigcodebench_review.py | 38 ++ bash/run.py | 518 ++++++++++++---- bash/run_full.py | 74 +++ bash/run_group1.py | 48 ++ bash/run_group2.py | 40 ++ bash/run_group3.py | 35 ++ bash/run_lite.py | 40 ++ bash/run_single.py | 45 ++ bash/test.py | 539 +++++++++++++++-- .../bigcodebench/bigcodebench_adapter.py | 7 +- .../tau_bench/tau2_bench/generation.py | 6 + 22 files changed, 2594 insertions(+), 706 deletions(-) create mode 100644 .dockerignore create mode 100644 DOCKER_BUILD.md create mode 100644 DOCKER_README.md create mode 100644 Dockerfile.withcode create mode 100644 EVALUATION_GUIDE.md create mode 100644 HOWTO_EVAL_NEW_MODEL.md delete mode 100644 README.md rename bash/{generation_config_nothinking.yaml => config/dpv4-int8_nothinking.yaml} (55%) create mode 100644 bash/fill_simpleqa_bfcl.py delete mode 100644 bash/generation_config_thinking.yaml create mode 100644 bash/rerun_bigcodebench_review.py create mode 100644 bash/run_full.py create mode 100644 bash/run_group1.py create mode 100644 bash/run_group2.py create mode 100644 bash/run_group3.py create mode 100644 bash/run_lite.py create mode 100644 bash/run_single.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..62df78c --- /dev/null +++ b/.dockerignore @@ -0,0 +1,26 @@ +datasets/ +datasets_flat/ +output/ +output_*/ +logs/ +docker/ +tau2-bench/ +*.tar +*.tar.gz +*.zip +__pycache__/ +*.pyc +*.pyo +*.pyd +.Python +*.so +.env +.venv/ +venv/ +ENV/ +.git/ +.gitignore +.pytest_cache/ +.mypy_cache/ +*.egg-info/ +.DS_Store diff --git a/.gitignore b/.gitignore index 75d4236..e888d3b 100644 --- a/.gitignore +++ b/.gitignore @@ -18,10 +18,16 @@ env/ /temp/ /outputs/ /output/ +/output* +/tau2-bench +/log +/docker +/datasets # 日志 *.log - +log* +/tools* # 密钥/配置文件 *.secret *.key diff --git a/DOCKER_BUILD.md b/DOCKER_BUILD.md new file mode 100644 index 0000000..b26c9b8 --- /dev/null +++ b/DOCKER_BUILD.md @@ -0,0 +1,554 @@ +# EvalScope Benchmark Docker 镜像构建实录 + +> 本文档记录 `evalscope-complete-py312` Docker 镜像的完整搭建过程、关键参数、踩坑与解决方案，供后续复现和维护参考。 + +--- + +## 1. 目标 + +构建一个**开箱即用**的 Docker 镜像，满足： +- 基于 Python 3.12 +- 内置 evalscope 源码及全部依赖 +- 除 `swe_bench` 系列外，其他 benchmark 无需再安装任何东西即可运行 +- 支持代码执行 sandbox（humaneval、bigcodebench） +- 支持 Agent benchmark（tau2_bench、general_fc） +- 支持 function calling 评测（bfcl_v3） +- 在国内网络环境下可构建 + +--- + +## 2. 基础镜像选择 + +```dockerfile +FROM python:3.12-slim-bookworm +``` + +### 2.1 这句话是什么意思？ + +`FROM` 是 Dockerfile 的第一条指令，意思是：**我要基于哪个镜像开始构建**。 + +`python:3.12-slim-bookworm` 可以拆成三部分： + +| 部分 | 含义 | +|------|------| +| `python` | 镜像名称，官方 Python 镜像 | +| `3.12` | Python 版本号 | +| `slim-bookworm` | 镜像变体标签 | + +`slim` 表示精简版，只保留最基础的东西，体积小。 +`bookworm` 是 Debian 12 的代号，这是 Linux 发行版的一个版本。 + +所以整句意思是：**基于官方 Python 3.12 精简版（Debian 12）镜像来构建我们的环境**。 + +类比理解： +> 就像你要装修房子，`FROM` 就是你选择的一套毛坯房。`python:3.12-slim-bookworm` 就是一套已经通了水电（装了 Python 和 pip）、但还没放家具（没装 evalscope 依赖）的毛坯房。 + +### 2.2 为什么选择这个镜像？ + +- **Python 3.12**：项目主力环境 +- **slim**：体积小，比完整版少几百 MB +- **bookworm**：Debian 12 稳定，apt 包源丰富 +- **已经包含 pip**：不需要自己装 Python + +--- + +## 3. 关键构建参数 + +| 参数 | 值 | 说明 | +|------|-----|------| +| `DEBIAN_FRONTEND=noninteractive` | 环境变量 | 避免 apt 交互式提示 | +| `PYTHONUNBUFFERED=1` | 环境变量 | Python 输出不缓冲 | +| `PIP_NO_CACHE_DIR=1` | 环境变量 | 不保留 pip 缓存，减小镜像 | +| `PYTHONDONTWRITEBYTECODE=1` | 环境变量 | 不生成 .pyc | +| `PYTHONPATH=/opt/evalscope/evalscope` | 环境变量 | 解决 editable install 加载问题 | +| apt 源 | 清华镜像 | 国内加速 | +| pip 源 | 清华镜像 | 国内加速 | +| 基础包 | git/wget/curl/build-essential 等 | 编译依赖 | +| docker.io | apt 安装 | 容器内支持套 Docker 跑 sandbox | + +--- + +## 4. Docker 命令参数详解（新手向） + +如果你是 Docker 新手，先把下面几个概念和参数搞懂，后续命令就不难了。 + +### 4.1 核心概念 + +- **镜像（Image）**：一个只读的模板，相当于一个打包好的环境。比如 `evalscope-complete-py312:latest`。 +- **容器（Container）**：镜像运行起来的实例。你可以把镜像理解为 Class，容器是 Object。 +- **宿主机（Host）**：运行 Docker 的那台物理机/虚拟机。 +- **容器内（Container）**：Docker 容器里面的环境。 + +### 4.2 Dockerfile 常用指令 + +本项目 `Dockerfile.py312` 里用到的指令： + +| 指令 | 作用 | 示例 | +|------|------|------| +| `FROM` | 基础镜像 | `FROM python:3.12-slim-bookworm` | +| `ENV` | 设置环境变量 | `ENV PYTHONPATH=/opt/evalscope/evalscope` | +| `RUN` | 执行命令（每行会产生一层镜像） | `RUN pip install ...` | +| `COPY` | 把宿主机文件复制到镜像里 | `COPY bash/ /opt/evalscope/bash/` | +| `WORKDIR` | 设置工作目录 | `WORKDIR /opt/evalscope` | +| `CMD` | 容器启动时默认执行的命令 | `CMD ["/bin/bash"]` | + +### 4.3 `docker build` 参数 + +```bash +docker build -f Dockerfile.py312 -t evalscope-complete-py312:latest . +``` + +| 参数 | 含义 | +|------|------| +| `-f Dockerfile.py312` | 指定用哪个 Dockerfile（默认是当前目录的 `Dockerfile`） | +| `-t evalscope-complete-py312:latest` | 给构建好的镜像打标签，`name:tag` 格式 | +| `.` | 构建上下文路径，Docker 会把这个目录下的文件传给构建进程 | + +### 4.4 `docker run` 参数 + +这是用得最多的命令： + +```bash +docker run -it --rm \ + --network host \ + -v /data1/sora/evalscope/datasets:/opt/evalscope/datasets \ + -v /data1/sora/evalscope/output:/opt/evalscope/output \ + -v /var/run/docker.sock:/var/run/docker.sock \ + evalscope-complete-py312:latest \ + bash -c "cd /opt/evalscope && python bash/run_lite.py ..." +``` + +| 参数 | 含义 | +|------|------| +| `docker run` | 创建并启动一个容器 | +| `-i` | 交互模式，保持 STDIN 打开 | +| `-t` | 分配一个伪终端，让你能看到彩色输出 | +| `-it` | 上面两个一起用，几乎必加 | +| `--rm` | 容器停止后自动删除，避免垃圾容器堆积 | +| `--network host` | 容器和宿主机共用网络，`localhost` 指向宿主机 | +| `-v 宿主机路径:容器内路径` | 挂载目录/文件，容器内的改动会反映到宿主机 | +| `-e KEY=VALUE` | 设置容器内环境变量 | +| `evalscope-complete-py312:latest` | 要运行的镜像名 | +| `bash -c "..."` | 容器启动后执行的命令 | + +**挂载参数 `-v` 是本项目的核心**： + +```bash +-v /data1/sora/evalscope/datasets:/opt/evalscope/datasets +``` + +意思是：把宿主机的 `/data1/sora/evalscope/datasets` 映射到容器内的 `/opt/evalscope/datasets`。这样容器就能读取宿主机上的数据，评测结果也能写回宿主机。 + +### 4.5 其他常用命令 + +```bash +# 查看本地镜像 +docker images + +# 查看运行中的容器 +docker ps + +# 查看所有容器（包括停止的） +docker ps -a + +# 删除镜像 +docker rmi evalscope-complete-py312:latest + +# 删除容器 +docker rm 容器ID + +# 进入正在运行的容器 +docker exec -it 容器ID /bin/bash + +# 加载 tar.gz 镜像 +docker load -i evalscope-complete-py312.tar.gz + +# 导出镜像为 tar.gz +docker save -o evalscope-complete-py312.tar.gz evalscope-complete-py312:latest +``` + +### 4.6 为什么这个项目不需要 `--gpus all` + +很多 Docker + GPU 的教程会写 `--gpus all`，但**本项目不需要**。 + +原因是： +- evalscope 容器本身只做 API 调用，不跑模型推理。 +- 模型服务（sglang）是在宿主机上启动的。 +- 容器只需要通过网络访问 `http://localhost:30000/v1`。 + +所以用 `--network host` 就够了，不需要把 GPU 分配给容器。 + +--- + +## 5. 构建步骤 + +### 5.1 准备构建上下文 + +#### `tools/docker/` 这个文件夹里都要放什么？ + +``` +tools/docker/ +├── Dockerfile.py312 # 构建配方 +├── evalscope/ # evalscope 源码（从项目根目录同步过来） +├── bash/ # bash 脚本（从项目根目录同步过来） +└── tau2-bench/ # tau2-bench 源码（从项目根目录同步过来） +``` + +**为什么需要同步？** +因为 Docker 构建时只能访问 `tools/docker/` 这个目录下的文件（这叫**构建上下文**）。而项目的源码在 `/data1/sora/evalscope/` 根目录下，所以需要先把需要的部分复制到 `tools/docker/` 里。 + +#### `rsync` 命令是什么意思？ + +```bash +rsync -av --delete --exclude='*.log' --exclude='__pycache__' \ + bash/ tools/docker/bash/ +``` + +拆开看： + +| 参数 | 含义 | +|------|------| +| `rsync` | 文件同步工具 | +| `-a` | 归档模式，保留文件权限、时间等 | +| `-v` | 显示同步了哪些文件 | +| `--delete` | 删除目标目录中源目录没有的文件，保持完全一致 | +| `--exclude='*.log'` | 不同步 `.log` 日志文件 | +| `--exclude='__pycache__'` | 不同步 Python 缓存目录 | +| `bash/` | 源目录（项目根目录的 bash 脚本） | +| `tools/docker/bash/` | 目标目录（构建上下文中的位置） | + +**用人话讲**：把 `bash/` 里的文件同步到 `tools/docker/bash/`，去掉日志和缓存，并且保证两边完全一样。 + +#### 为什么用 rsync 而不用 `cp -r`？ + +因为 `cp -r` 只是简单复制，不会删除目标目录里多余的旧文件。如果用 `cp -r`，以前删除的脚本可能还会留在 `tools/docker/bash/` 里，最终被打包进镜像。 + +### 5.2 构建镜像 + +```bash +cd /data1/sora/evalscope/tools/docker +docker build -f Dockerfile.py312 -t evalscope-complete-py312:latest . +``` + +### 5.3 环境是怎么装进 Docker 里的？ + +很多新手会问：Python 包是怎么装到镜像里的？答案是：**Dockerfile 里的 `RUN pip install ...` 命令会在构建时执行，把包装到镜像里**。 + +整个流程可以概括为： + +``` +毛坯房（python:3.12-slim-bookworm） + ↓ +装修第一层：换国内源、装系统工具（apt） + ↓ +装修第二层：RUN pip install ... 装 Python 包 + ↓ +装修第三层：COPY 项目源码进镜像 + ↓ +装修第四层：把源码用 pip install -e 安装好 + ↓ +装修第五层：设置 PYTHONPATH、权限等收尾工作 + ↓ +完工（evalscope-complete-py312:latest 镜像） +``` + +关键步骤对应 Dockerfile 里的这些行： + +| Dockerfile 行 | 作用 | +|---------------|------| +| `FROM python:3.12-slim-bookworm` | 选毛坯房 | +| `RUN apt-get update && apt-get install ...` | 装系统工具 | +| `RUN pip install openai pandas ...` | 装 Python 包 | +| `COPY evalscope/ /opt/evalscope/evalscope/` | 复制源码 | +| `RUN pip install -e /opt/evalscope/evalscope/` | 安装 evalscope 本身 | +| `ENV PYTHONPATH=/opt/evalscope/evalscope` | 设置环境变量 | + +**`RUN` 和 `COPY` 的区别**： +- `RUN`：执行命令，比如安装软件 +- `COPY`：复制文件，比如把源码复制进去 + +**为什么先 `pip install` 再 `COPY` 源码？** +因为 Docker 有缓存机制。如果依赖没有变化，只有源码变化，Docker 会直接使用之前装好的依赖层，只重新构建 COPY 和之后的层，节省大量时间。 + +### 5.4 导出 tar.gz + +```bash +docker save -o evalscope-complete-py312.tar.gz evalscope-complete-py312:latest +md5sum evalscope-complete-py312.tar.gz > evalscope-complete-py312.tar.gz.md5 +``` + +--- + +## 6. 安装的关键依赖 + +### 6.1 evalscope 核心依赖 + +```bash +pip install \ + openai pandas numpy pyyaml requests tqdm \ + tiktoken transformers \ + scikit-learn matplotlib seaborn plotly \ + jieba nltk rouge-score sacrebleu \ + sympy latex2sympy2_extended pillow \ + docker pexpect pytest \ + tabulate rich jsonlines jsonschema \ + langdetect word2number zhconv \ + modelscope pydantic overrides \ + more_itertools pylatexenc \ + rouge-chinese markdown \ + editdistance dotenv docstring_parser \ + colorlog +``` + +### 6.2 Sandbox 支持 + +```bash +pip install evalscope[sandbox] 2>/dev/null || pip install ms-sandbox 2>/dev/null || true +``` + +`ms-sandbox` 是 ModelScope 的 sandbox 实现，用于 humaneval、bigcodebench 的隔离代码执行。 + +### 6.3 terminal_bench 依赖 + +```bash +pip install "harbor>=0.8.0,<1.0.0" +``` + +### 6.4 tau2-bench 依赖 + +```bash +pip install \ + fastapi uvicorn psutil loguru \ + litellm tenacity deepdiff addict toml + +# 同时把本地 tau2-bench 源码 editable 安装 +pip install -e /opt/evalscope/tools/tau2-bench/ --no-deps +``` + +### 6.5 bigcodebench 评估依赖 + +```bash +pip install \ + tree-sitter tree-sitter-python tempdir termcolor wget gradio-client +``` + +### 6.6 bfcl_v3 依赖 + +```bash +pip install --no-deps bfcl-eval==2025.10.27.1 +pip install \ + anthropic cohere==5.18.0 datamodel-code-generator==0.25.7 \ + faiss-cpu==1.11.0 google-genai==1.24.0 mistralai==1.7.0 \ + networkx==3.3 numpy==1.26.4 google-search-results \ + rank_bm25 html2text boto3 qwen-agent writer-sdk \ + tree-sitter tree-sitter-python tree-sitter-javascript tree-sitter-java +``` + +注意：`bfcl-eval` 默认会装 torch，这里用 `--no-deps` 避免把 torch 拉进来，再手动补缺少的依赖。 + +### 6.7 其他工具 + +```bash +pip install soundfile openpyxl +``` + +--- + +## 7. 踩坑记录 + +### 7.1 editable install 加载失败 + +**现象**： +``` +ImportError: cannot import name 'run_task' from 'evalscope' (unknown location) +``` + +**原因**：evalscope 用 `pip install -e /opt/evalscope/evalscope/` 安装后，editable finder 有时不能正确加载 `__init__.py`。 + +**解决**：在 Dockerfile 末尾设置 `ENV PYTHONPATH=/opt/evalscope/evalscope`。 + +### 7.2 镜像里缺少新脚本 + +**现象**：其他机器拉下来跑 `run_group2.py` 报 `No such file or directory`。 + +**原因**：Docker 镜像构建后，项目根目录又新增了 `run_group1/2/3.py`、`run_1.py` 等文件，但镜像没有重新构建。 + +**解决**：每次修改 bash/ 目录后，必须重新同步到 `tools/docker/bash/`，然后重新 `docker build` 和 `docker save`。 + +### 7.3 镜像体积过大 + +**现象**：安装 bfcl-eval 默认会拉 torch，镜像暴涨几个 GB。 + +**解决**： +```bash +pip install --no-deps bfcl-eval==2025.10.27.1 +``` +再手动安装缺少的非 torch 依赖。 + +### 7.4 litellm 版本冲突 + +**现象**：tau2-bench 要求 `litellm>=1.80.15,<1.82.7`，但 bfcl-eval 可能装更高版本。 + +**解决**：在 Dockerfile 中显式约束： +```bash +pip install "litellm>=1.80.15,<1.82.7" "tenacity>=9.0.0" +``` + +### 7.5 国内网络下载慢/超时 + +**现象**：pip、apt、git clone 都很慢。 + +**解决**： +- apt 换清华源 +- pip 换清华源 +- git+https 安装 tau2-bench 时容易失败，改为本地 COPY 源码后 editable 安装 + +### 7.6 ModelScope 上传 API 参数错误 + +**现象**： +``` +TypeError: HubApi.upload_file() got an unexpected keyword argument 'model_id' +``` + +**原因**：新版 ModelScope SDK 参数名是 `repo_id`，不是 `model_id`。 + +**解决**： +```python +api.upload_file( + repo_id='SoraAmami/evalscope-docker', + path_or_fileobj='evalscope-complete-py312.tar.gz', + path_in_repo='evalscope-complete-py312.tar.gz' +) +``` + +### 7.7 数据集路径重复 + +**现象**：`/data1/sora/evalscope/datasets/datasets/` 下又有一层数据。 + +**原因**：evalscope 会在 `dataset_dir` 后自动拼 `datasets/`。如果把 `dataset_dir` 指向了已经包含 `datasets/` 的路径，就会重复。 + +**解决**：始终让 `dataset_dir` 指向 `datasets/` 的父目录。 + +### 7.8 运行时缺少 tokenizer + +**现象**：`transformers` 提示 `PyTorch was not found`。 + +**原因**：镜像里没有 torch（ intentional，为了减小体积）。 + +**影响**：evalscope 的 tokenizer 加载通常只需要 `transformers` 的 tokenizer 部分，实际运行时通过 API 调用模型，不需要本地 PyTorch。该警告可忽略。 + +--- + +## 8. 镜像验证 + +构建完成后，验证镜像内关键组件： + +```bash +docker run --rm evalscope-complete-py312:latest bash -c " + python -c 'import evalscope; print(evalscope.__file__)' && \ + python -c 'import evalscope.api.agent' && \ + python -c 'import docker' && \ + python -c 'import harbor' && \ + python -c 'import tau2' && \ + python -c 'import bfcl_eval' && \ + python -c 'import soundfile' && \ + ls -la /opt/evalscope/bash/run_1.py /opt/evalscope/bash/run_group*.py +" +``` + +--- + +## 9. 三个评测版本 + +为了兼顾快速验证和完整评测，提供三个版本： + +| 版本 | 预计时间 | 用途 | +|------|----------|------| +| **Lite** | ~2-4h | 快速冒烟，每个能力域 1-2 个 benchmark | +| **Standard** | ~20-28h | 常规能力评测，覆盖主要 benchmark | +| **Full** | ~3-5 天 | 完整评测，全部 benchmark + 多次采样 | + +每个版本都覆盖 5 大能力域：代码生成、推理/数学、知识、长上下文、智能体/工具。 + +### 9.1 Lite 版 + +- 代码：`humaneval` +- 推理/数学：`aime24`、`gsm8k` +- 知识：`mmlu_pro`、`simple_qa` +- 长上下文：`longbench_v2` +- 智能体/工具：`bfcl_v3` + +运行： +```bash +python bash/run_lite.py --model YourModel --api-url http://localhost:30000/v1 --limit 20 +``` + +### 9.2 Standard 版 + +- 代码：`humaneval`、`live_code_bench`、`bigcodebench` +- 推理/数学：`aime24`、`aime25`、`aime26`、`hmmt26`、`gsm8k`、`competition_math`、`bbh`、`drop` +- 知识：`gpqa_diamond`、`mmlu_pro`、`simple_qa`、`super_gpqa`、`mmlu`、`cmmlu`、`arc`、`hellaswag`、`trivia_qa`、`winogrande` +- 长上下文：`longbench_v2`、`openai_mrcr` +- 智能体/工具：`tau2_bench`、`general_fc`、`bfcl_v3` + +运行： +```bash +python bash/run_standard.py --model YourModel --api-url http://localhost:30000/v1 --limit 100 +``` + +### 9.3 Full 版 + +覆盖 `run.py` 中全部 benchmark，使用完整数据集和多次采样配置。 + +运行： +```bash +python bash/run.py --model YourModel --api-url http://localhost:30000/v1 --limit none +``` + +--- + +## 10. 文件清单 + +``` +tools/docker/ +├── Dockerfile.py312 # 镜像构建定义 +├── evalscope-complete-py312.tar.gz # 导出的镜像 +├── evalscope-complete-py312.tar.gz.md5 +├── deploy.sh # 目标机器部署脚本 +├── export_image.sh # 导出脚本 +├── build_*.log # 构建日志 +└── README.md # 使用说明 +``` + +--- + +## 11. 重新构建流程 + +```bash +cd /data1/sora/evalscope + +# 1. 同步最新代码 +rsync -av --delete bash/ tools/docker/bash/ +rsync -av --delete evalscope/ tools/docker/evalscope/ +rsync -av --delete tau2-bench/ tools/docker/tau2-bench/ + +# 2. 构建 +cd tools/docker +docker build -f Dockerfile.py312 -t evalscope-complete-py312:latest . + +# 3. 导出 +docker save -o evalscope-complete-py312.tar.gz evalscope-complete-py312:latest +md5sum evalscope-complete-py312.tar.gz > evalscope-complete-py312.tar.gz.md5 + +# 4. 上传到 ModelScope（可选） +python3 -c " +from modelscope.hub.api import HubApi +api = HubApi() +api.login('ms-3d554a39-6e07-496d-8022-0b0ee64a6389') +api.upload_file( + repo_id='SoraAmami/evalscope-docker', + path_or_fileobj='evalscope-complete-py312.tar.gz', + path_in_repo='evalscope-complete-py312.tar.gz' +) +" +``` diff --git a/DOCKER_README.md b/DOCKER_README.md new file mode 100644 index 0000000..3d7a88b --- /dev/null +++ b/DOCKER_README.md @@ -0,0 +1,113 @@ +# EvalScope Docker 评测快速入口 + +> 详细流程、参数说明、踩坑记录和三个版本（Lite/Standard/Full）的完整命令，请见 `work.md`。 +> +> 本文档只保留最常用的命令，方便复制粘贴。 + +## 快速开始 + +### 0. 配置 Docker 国内镜像加速（可选但强烈建议） + +```bash +sudo mkdir -p /etc/docker +sudo tee /etc/docker/daemon.json <<'EOF' +{ + "registry-mirrors": [ + "https://docker.1ms.run", + "https://docker.m.daocloud.io", + "https://hub-mirror.c.163.com" + ] +} +EOF +sudo systemctl daemon-reload +sudo systemctl restart docker +``` + +### 1. 准备代码与数据 + +代码与 Docker 环境镜像**分开下载**： + +```bash +pip install modelscope +modelscope login --token ms-3d554a39-6e07-496d-8022-0b0ee64a6389 + +# 下载项目代码 +python3 -c " +from modelscope.hub.snapshot_download import snapshot_download +snapshot_download( + 'SoraAmami/evalscope-code', + repo_type='dataset', + cache_dir='/data1/sora/evalscope', + local_dir='/data1/sora/evalscope' +) +" + +# 下载数据集 +python3 -c " +from modelscope.hub.snapshot_download import snapshot_download +snapshot_download( + 'SoraAmami/evalscope-datasets', + repo_type='dataset', + cache_dir='/data1/sora/evalscope', + local_dir='/data1/sora/evalscope' +) +" +``` + +### 2. 加载 Docker 环境镜像 + +```bash +python3 -c " +from modelscope.hub.file_download import model_file_download +model_file_download( + model_id='SoraAmami/evalscope-docker', + file_path='evalscope-complete-py312.tar.gz', + local_dir='/data1/sora/evalscope/docker' +) +" + +docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz +``` + +### 3. 构建 bigcodebench sandbox 镜像 + +```bash +cd /data1/sora/evalscope + +# 如果 Docker Hub 慢，先通过国内镜像拉取上游镜像 +docker pull docker.1ms.run/bigcodebench/bigcodebench-evaluate:latest +docker tag docker.1ms.run/bigcodebench/bigcodebench-evaluate:latest bigcodebench/bigcodebench-evaluate:latest + +# 构建去掉 entrypoint 的 sandbox 镜像 +docker build -t bigcodebench-sandbox:latest docker/bigcodebench_sandbox/ +``` + +### 4. 跑 Standard 版 + +```bash +mkdir -p /data1/sora/evalscope/output_standard +docker run -it --rm \ + --network host \ + -v /data1/sora/evalscope:/opt/evalscope \ + -v /data1/sora/evalscope/datasets:/opt/evalscope/datasets \ + -v /data1/sora/evalscope/output_standard:/opt/evalscope/output_standard \ + -v /var/run/docker.sock:/var/run/docker.sock \ + evalscope-complete-py312:latest \ + bash -c " + cd /opt/evalscope && \ + python bash/run_standard.py \ + --model DeepSeek-V4-Flash-Int8 \ + --api-url http://localhost:30000/v1 \ + --dataset-dir /opt/evalscope \ + --output-dir /opt/evalscope/output_standard + " +``` + +## 更多内容 + +| 需求 | 查看文档 | +|------|----------| +| 完整 Docker 评测流程 | `work.md` 第 3 章 | +| Docker 镜像构建过程 | `DOCKER_BUILD.md` | +| 评测新模型 | `HOWTO_EVAL_NEW_MODEL.md` | +| 三个版本（Lite/Standard/Full）说明 | `work.md` 3.7 节 | diff --git a/Dockerfile.withcode b/Dockerfile.withcode new file mode 100644 index 0000000..6611206 --- /dev/null +++ b/Dockerfile.withcode @@ -0,0 +1,19 @@ +# Evalscope Docker image with project code built-in +# Based on the existing environment image + +FROM evalscope-complete-py312:latest + +# Overwrite /opt/evalscope with updated project code +COPY . /opt/evalscope + +# Remove the old editable install and install local evalscope normally +RUN pip uninstall -y evalscope && \ + pip install /opt/evalscope/evalscope --no-deps + +# Verify run_1.py exists and imports work +RUN test -f /opt/evalscope/bash/run_1.py && \ + echo "run_1.py found" && \ + python3 -c "from evalscope import run_task, TaskConfig; print('evalscope import OK')" + +WORKDIR /opt/evalscope +CMD ["/bin/bash"] diff --git a/EVALUATION_GUIDE.md b/EVALUATION_GUIDE.md new file mode 100644 index 0000000..87b3de5 --- /dev/null +++ b/EVALUATION_GUIDE.md @@ -0,0 +1,266 @@ +# DeepSeek-V4-Flash-INT8 评测经验总结 + +## 1. 环境配置 + +### 1.1 硬件 +- GPU: 8× P800 OAM, 96GB each (XPU) +- 模型: DeepSeek-V4-Flash-INT8 +- 服务: sglang at `http://localhost:30000/v1` +- 上下文长度: 1M tokens (`--context-length 1048576`) + +### 1.2 sglang 启动参数 +```bash +sglang.launch_server \ + --host 0.0.0.0 --port 30000 \ + --model-path /data1/models/DeepSeek-V4-Flash-INT8 \ + --attention-backend nsa --nsa-prefill klxdsa --nsa-decode klxdsa \ + --trust-remote-code --disable-custom-all-reduce \ + --tensor-parallel-size 8 --ep-size 8 \ + --disable-shared-experts-fusion --page-size 64 \ + --mem-fraction-static 0.75 --quantization w8a8_int8 \ + --kv-cache-dtype float16 --disable-piecewise-cuda-graph \ + --cuda-graph-max-bs 16 --context-length 1048576 \ + --watchdog-timeout 3000000 \ + --tool-call-parser deepseekv4 --reasoning-parser deepseek-v4 \ + --constrained-json-disable-any-whitespace \ + --enable-metrics --enable-request-time-stats-logging +``` + +### 1.3 Python 环境 +- conda env: `evalscope` (Python 3.12) +- 正确路径: `/data1/miniconda3/envs/evalscope/bin/python` +- 注意: `conda run -n evalscope` 会覆盖 PYTHONPATH，导致 ms_enclave 找不到 + +## 2. 评测脚本 + +### 2.1 主运行脚本 +- `/data1/sora/evalscope/bash/run.py` - 主评测驱动 +- `/data1/sora/evalscope/bash/run_limit30.py` - limit=100 快速 benchmark +- `/data1/sora/evalscope/bash/run_swe_bench.py` - swe_bench 系列 (limit=3) +- `/data1/sora/evalscope/bash/auto_run.py` - 自动顺序运行 +- `/data1/sora/evalscope/bash/run_longbench_oom_test.py` - 长文本 OOM 测试 + +### 2.2 配置文件 +- `/data1/sora/evalscope/bash/dpv4-int8_nothinking.yaml` - 各数据集生成配置 + +### 2.3 关键参数 +```python +# run.py 顶部可调参数 +DATASETS = [...] # 要评测的数据集列表 +ENABLE_THINKING = False # 是否启用 thinking 模式 +BATCH_SIZE_LIST = [4] # 评测 batch size +LIMIT = 10 # 每个子集最大样本数 (None=全部) +SEED = 42 # 随机种子 +SHUFFLE = True # 是否打乱样本 +``` + +## 3. 各数据集配置 + +### 3.1 温度设置 +- 所有数据集: `temperature: 0.0`（确定性输出） +- temperature=1.0 需要多次运行取平均 + +### 3.2 max_tokens 设置 +| 类别 | 数据集 | max_tokens | +|------|--------|-----------| +| 代码/工程 | terminal_bench_v2 | 262144 | +| 代码/工程 | live_code_bench, bigcodebench, humaneval | 32768-16384 | +| 推理/数学 | aime24/25/26, hmmt26, competition_math, gsm8k | 32768 | +| 推理/数学 | imo_answerbench | 98304 | +| 推理/数学 | gpqa_diamond, hle, super_gpqa | 8192 | +| 知识 | mmlu_pro, simple_qa, arc, bbh, cmmlu, drop, hellaswag, mmlu, trivia_qa, winogrande | 8192-32768 | +| 长上下文 | longbench_v2, openai_mrcr | 8192 | +| 智能体/工具 | browsecomp, mcp_atlas, tau2_bench | 16384-32768 | +| 智能体/工具 | swe_bench 系列 | 32768 | + +### 3.3 Agent 配置 +- swe_bench 系列: `max_steps: 300` +- tau2_bench: `max_steps: 50` +- terminal_bench_v2: `max_turns: 300`, `timeout_multiplier: 6.0` (1小时) + +### 3.4 沙箱配置 +```python +# 需要沙箱的数据集 +SANDBOX_DATASETS = {'humaneval', 'bigcodebench'} + +# 沙箱配置 +SandboxTaskConfig( + enabled=True, + engine='docker', + default_config={ + 'image': 'python:3.11-slim', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + } +) +``` + +## 4. 长文本截断 + +### 4.1 当前配置 +```python +TRUNCATION_CONFIG = { + 'longbench_v2': 32768*4, # 131072 tokens (128K) + 'openai_mrcr': 32768*4, # 131072 tokens (128K) +} +``` + +### 4.2 截断方式 +- Middle-truncation: 保留头部和尾部，截断中间 +- 适用于长上下文 benchmark，保留问题和上下文的关键部分 + +### 4.3 OOM 测试 +- 测试脚本: `run_longbench_oom_test.py` +- 测试 limit: 128K, 256K, 512K, 1M +- 输出目录: `output_longbench_128k/`, `output_longbench_256k/` 等 + +## 5. Docker 镜像 + +### 5.1 需要预拉的镜像 +```bash +# 代码评测 +docker pull docker.1ms.run/bigcodebench/bigcodebench-evaluate:latest + +# Python 沙箱 +docker pull python:3.11-slim +``` + +### 5.2 按样本拉取的镜像（无法预准备） +- **swe_bench 系列**: 每个样本需要不同的 `jefzda/sweap-images:{tag}` + - 每个镜像 2-5GB + - limit=3 需要 ~10GB + - 完整 500 样本需要 ~1.75TB（不现实） + - 建议: limit=1-3 或跳过 + +- **terminal_bench_v2**: 使用统一基础镜像，但任务环境不同 + - 镜像在运行时自动构建/拉取 + - 超时问题常见，需要 `timeout_multiplier: 6.0` + +### 5.3 镜像源 +- Docker Hub 被墙，使用镜像站: + - `docker.1ms.run` + - `docker.m.daocloud.io` + - `docker.1panel.dev` + +### 5.4 磁盘空间管理 +- sglang 镜像: ~58GB（不能删） +- bigcodebench: ~25GB（跑完可删） +- sweap-images: ~21GB（按需删除） +- terminal_bench: ~2GB（可删） + +## 6. 常见问题与解决方案 + +### 6.1 tau2_bench 空响应问题 +**问题**: `UserMessage must have either content or tool_calls` +**解决**: 修改 `tau2-bench/src/tau2/user/user_simulator.py`，增加重试逻辑: +```python +max_retries = 5 +for attempt in range(max_retries): + assistant_message = generate(...) + user_response = assistant_message.content + if user_response and user_response.strip(): + break + if attempt < max_retries - 1: + logger.warning(f"Empty response, retrying...") +``` + +### 6.2 terminal_bench_v2 超时问题 +**问题**: `EnvironmentStartTimeoutError: Environment start timed out after 600.0 seconds` +**解决**: 增加 `timeout_multiplier: 6.0` (1小时) + +### 6.3 swe_bench 镜像拉取慢 +**问题**: 网络慢，镜像大，磁盘满 +**解决**: +- 使用国内镜像站 +- 限制 limit=1-3 +- 定期清理 Docker 镜像 + +### 6.4 模型服务重启 +**问题**: sglang 服务可能崩溃 +**解决**: `/data1/restart_model_service.sh` + +## 7. 结果提取与报告 + +### 7.1 提取脚本 +- `/data1/sora/evalscope/bash/extract_results.py` +- 从 JSON 报告提取 score、perf metrics、percentiles + +### 7.2 Excel 报告 +- `/data1/sora/P800模型能力评测结果_updated.xlsx` +- 包含: score, 时间, tokens, TTFT/TPOT percentiles + +### 7.3 结果合并策略 +- 优先使用 `output_limit100/` 的结果（样本更多） +- 然后使用 `output_swe_bench/` 的结果 +- 最后使用 `output/` 的结果 + +## 8. 评测策略建议 + +### 8.1 快速评测 (1天) +- 使用 limit=100 跑快速 benchmark +- 跳过 swe_bench 系列 +- 长文本使用 128K 截断 + +### 8.2 完整评测 (1周+) +- 所有 benchmark 全量运行 +- swe_bench limit=3-5 +- 长文本测试 256K/512K/1M 边界 + +### 8.3 关键 benchmark +- **推理/数学**: aime24/25/26, competition_math, gpqa_diamond +- **代码**: humaneval, live_code_bench, bigcodebench +- **知识**: mmlu, mmlu_pro, cmmlu +- **长文本**: longbench_v2, openai_mrcr +- **智能体**: browsecomp, tau2_bench + +## 9. 文件清单 + +``` +/data1/sora/evalscope/bash/ +├── run.py # 主运行脚本 +├── run_limit30.py # limit=100 快速运行 +├── run_swe_bench.py # swe_bench 系列 +├── auto_run.py # 自动顺序运行 +├── run_longbench_oom_test.py # 长文本 OOM 测试 +├── extract_results.py # 结果提取 +├── dpv4-int8_nothinking.yaml # 数据集配置 +├── generate_excel.py # Excel 生成 +├── generate_dashboard.py # Dashboard 生成 + +/data1/sora/evalscope/output/ # 主输出目录 +/data1/sora/evalscope/output_limit100/ # limit=100 输出 +/data1/sora/evalscope/output_swe_bench/ # swe_bench 输出 +/data1/sora/evalscope/output_longbench_*/ # 长文本测试输出 + +/data1/sora/P800模型能力评测结果_updated.xlsx # Excel 报告 +``` + +## 10. 运行命令汇总 + +```bash +# 激活环境 +source /data1/miniconda3/bin/activate evalscope + +# 运行主评测 +python /data1/sora/evalscope/bash/run.py + +# 运行 limit=100 快速评测 +python /data1/sora/evalscope/bash/run_limit30.py + +# 运行 swe_bench +python /data1/sora/evalscope/bash/run_swe_bench.py + +# 自动顺序运行 +python /data1/sora/evalscope/bash/auto_run.py + +# 长文本 OOM 测试 +python /data1/sora/evalscope/bash/run_longbench_oom_test.py + +# 提取结果 +python /data1/sora/evalscope/bash/extract_results.py + +# 重启模型服务 +bash /data1/restart_model_service.sh +``` diff --git a/HOWTO_EVAL_NEW_MODEL.md b/HOWTO_EVAL_NEW_MODEL.md new file mode 100644 index 0000000..496b81c --- /dev/null +++ b/HOWTO_EVAL_NEW_MODEL.md @@ -0,0 +1,216 @@ +# 如何评测新模型 + +> 本指南假设模型已通过 sglang/vllm 等部署为 OpenAI 兼容 API，只需修改配置即可用本框架评测。 + +--- + +## 目录 + +1. [最小改动：只换模型名和 API 地址](#最小改动只换模型名和-api-地址) +2. [推荐做法：复制一份配置文件](#推荐做法复制一份配置文件) +3. [配置参数说明](#配置参数说明) +4. [运行命令](#运行命令) +5. [结果收集](#结果收集) + +--- + +## 最小改动：只换模型名和 API 地址 + +如果新模型能力和 `DeepSeek-V4-Flash-INT8` 接近，只想换个名字和 API 地址： + +```bash +cd /data1/sora/evalscope +python bash/run.py \ + --model Your-Model-Name \ + --api-url http://your-api:30000/v1 \ + --dataset-dir /data1/sora/evalscope \ + --output-dir /data1/sora/evalscope/output_your_model \ + --limit none +``` + +> `--limit none` 跑完整数据；测试时用 `--limit 2` 或 `--limit 20`。 + +--- + +## 推荐做法：复制一份配置文件 + +不同模型的最佳 `temperature` / `max_tokens` 可能不同，建议为每个模型单独维护一个 YAML。 + +### 1. 复制默认配置 + +```bash +cd /data1/sora/evalscope/bash +mkdir -p config +cp config/dpv4-int8_nothinking.yaml config/your-model.yaml +``` + +### 2. 按需修改 `config/your-model.yaml` + +下面是重点需要改的字段： + +```yaml +# 示例： reasoning 模型可能需要 temperature=1.0 多测几次求平均 +aime24: + generation_config: + temperature: 1.0 # 多测 benchmark 用 1.0；只测一次用 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 # 根据模型输出长度能力调整 + +# 示例：知识类 benchmark 通常 greedy decoding +mmlu_pro: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 + +# 示例：Agent benchmark 的 judge model +# 如果你不想改 judge model，保持默认 deepseek-v4-pro 即可 +tau2_bench: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 16384 + dataset_args: + extra_params: + user_model: deepseek-v4-pro + api_key: sk-9ed86ef546ca47e3afa7c3b014dea268 + api_base: https://api.deepseek.com/v1 + generation_config: + temperature: 0.0 + max_tokens: 4096 + agent_config: + mode: native + strategy: react + max_steps: 50 +``` + +### 3. 修改 `run.py` 中的截断/tokenizer 配置（长上下文模型必看） + +打开 `bash/run.py`，找到下面几行： + +```python +MODEL_PATH = '/data1/models/DeepSeek-V4-Flash-INT8' +``` + +如果你的新模型本地有 tokenizer 目录，改成对应路径；如果通过 API 跑，这条基本不影响运行，但建议保持和模型一致。 + +再找到： + +```python +TRUNCATION_CONFIG = { + 'longbench_v2': 32768*4, + 'openai_mrcr': 32768*4, +} +``` + +这表示长上下文 benchmark 最多保留 **131072 tokens**（头尾各一半）。 +- 如果新模型 context length 更大且显存足够，可以调大（如 `32768*8`）。 +- 如果显存紧张，可以调小（如 `32768*2` = 65536）。 + +### 4. 修改 judge model（可选） + +如果 tau2_bench 需要换 judge： + +```python +JUDGE_MODEL_ARGS = { + 'model_id': 'deepseek-v4-pro', + 'api_url': 'https://api.deepseek.com/v1', + 'api_key': 'sk-9ed86ef546ca47e3afa7c3b014dea268', + ... +} +``` + +如果不需要 judge（例如只跑非 Agent benchmark），可以忽略。 + +--- + +## 配置参数说明 + +| 参数 | 作用 | 建议 | +|------|------|------| +| `temperature` | 采样温度 | 需要多次采样求平均的 benchmark（aime/hmmt/imo/live_code_bench/humaneval/gpqa）用 `1.0`；其余 greedy `0.0` | +| `max_tokens` | 最大输出长度 | 短输出知识题 `8192`；推理/代码 `32768`；Agent `16384`；具体看模型能力和任务 | +| `top_p` | nucleus sampling | 一般保持 `1.0` | +| `stream` | 流式输出 | 建议 `true`，兼容性好 | +| `agent_config.max_steps` | Agent 最大步数 | 默认 `50`；复杂任务可调大，简单任务可调小 | +| `TRUNCATION_CONFIG` | 长上下文截断 | 根据模型 context length 和显存调整 | + +--- + +## 运行命令 + +### 全量跑 + +```bash +cd /data1/sora/evalscope +python bash/run.py \ + --config bash/config/your-model.yaml \ + --model Your-Model-Name \ + --api-url http://your-api:30000/v1 \ + --dataset-dir /data1/sora/evalscope \ + --output-dir /data1/sora/evalscope/output_your_model \ + --limit none +``` + +### 只跑某个分组 + +```bash +python bash/run_group1.py \ + --config bash/config/your-model.yaml \ + --model Your-Model-Name \ + --api-url http://your-api:30000/v1 \ + --dataset-dir /data1/sora/evalscope \ + --output-dir /data1/sora/evalscope/output_your_model_group1 \ + --limit none +``` + +### Docker 方式 + +```bash +docker run -it --rm \ + --network host \ + -v /data1/sora/evalscope/datasets:/opt/evalscope/datasets \ + -v /data1/sora/evalscope/output_your_model:/opt/evalscope/output_your_model \ + -v /var/run/docker.sock:/var/run/docker.sock \ + evalscope-complete-py312:latest \ + bash -c " + cd /opt/evalscope && \ + python bash/run.py \ + --config bash/config/your-model.yaml \ + --model Your-Model-Name \ + --api-url http://localhost:30000/v1 \ + --dataset-dir /opt/evalscope \ + --output-dir /opt/evalscope/output_your_model \ + --limit none + " +``` + +--- + +## 结果收集 + +评测完成后，结果在 `--output-dir` 目录下： + +``` +output_your_model/ +├── humaneval/seed_42/reports/Your-Model-Name/humaneval.json +├── aime24/seed_42/reports/Your-Model-Name/aime24.json +└── ... +``` + +汇总分数： + +```bash +python3 -c " +import json, glob +for f in sorted(glob.glob('/data1/sora/evalscope/output_your_model/*/seed_*/reports/*/*.json')): + data = json.load(open(f)) + score = data.get('score', data.get('mean_acc', 'N/A')) + print(f'{f}: {score}') +" +``` + +或者用 `bash/update_excel.py`（如果有）把结果写入 Excel。 diff --git a/README.md b/README.md deleted file mode 100644 index b1a8623..0000000 --- a/README.md +++ /dev/null @@ -1,177 +0,0 @@ -# Sora Benchmarks - -## 1. 环境安装 - -```bash -conda create -n evalscope python==3.12 -y -conda activate evalscope - -cd evalscope -pip install -e . - -# 安装特定 benchmark 依赖 -pip install 'evalscope[terminal_bench,swe_bench,openai_mrcr]' \ - 'git+https://github.com/sierra-research/tau2-bench@v0.2.0' -pip install 'evalscope[sandbox]' -pip install 'swebench==4.1.0' -``` - -## 2. 常用 Benchmarks - -```python -benchmark_categories = { - "代码与工程": [ - "terminal_bench_v2", - "live_code_bench", - "swe_bench_multilingual_agentic", - "swe_bench_pro", - "swe_bench_verified", - "bigcodebench", - "humaneval", - ], - "推理与数学": [ - "gpqa_diamond", - "hle", - "aime24", - "aime25", - "aime26", - "hmmt26", - "imo_answerbench", - "super_gpqa", - "gsm8k", - "competition_math", - ], - "智能体与工具": [ - "browsecomp", - "mcp_atlas", - "tau2_bench", - ], - "知识与语言理解": [ - "mmlu_pro", - "simple_qa", - "arc", - "bbh", - "cmmlu", - "drop", - "hellaswag", - "mmlu", - "trivia_qa", - "winogrande", - ], - "长上下文": [ - "longbench_v2", - "openai_mrcr", - ], -} -``` - -## 3. 分组运行 - -```python -group_1 = [ - 'terminal_bench_v2', - 'gpqa_diamond', - 'hle', - 'aime24', - 'aime25', - 'mmlu_pro', - 'simple_qa', - 'arc', - 'bbh', - 'browsecomp', -] - -group_2 = [ - 'live_code_bench', - 'swe_bench_pro', - 'aime26', - 'hmmt26', - 'imo_answerbench', - 'super_gpqa', - 'drop', - 'hellaswag', - 'mmlu', - 'openai_mrcr', - 'mcp_atlas', -] - -group_3 = [ - 'swe_bench_multilingual_agentic', - 'swe_bench_verified', - 'bigcodebench', - 'humaneval', - 'gsm8k', - 'competition_math', - 'cmmlu', - 'trivia_qa', - 'winogrande', - 'longbench_v2', - 'tau2_bench', -] -``` - -## 4. 运行示例 - -```python -from evalscope import run_task, TaskConfig - -datasets = [ - 'live_code_bench', - 'swe_bench_pro', - 'aime26', - 'hmmt26', - 'imo_answerbench', - 'super_gpqa', - 'drop', - 'hellaswag', - 'mmlu', - 'openai_mrcr', - 'mcp_atlas', -] - -task_cfg = TaskConfig( - collect_perf=True, - work_dir='/data1/sora/benchmarks/temp/output', - use_cache='/data1/sora/benchmarks/temp/output', - no_timestamp=True, - model='', - api_url='http://localhost:30000/v1', - eval_type='openai_api', - datasets=datasets, - dataset_dir='/data1/sora/benchmarks/bash/datasets', - generation_config={ - 'temperature': 0.0, - 'stream': True, - }, - eval_batch_size=1, - judge_model_args={ - "model_id": "deepseek-v4-flash", - "api_url": "https://api.deepseek.com/v1", - "api_key": "", - "eval_type": "openai_api", - "generation_config": { - "temperature": 0.0, - "max_tokens": 1024 * 10, - }, - } -) - -run_task(task_cfg) -``` - -## 5. 评测结果 - -P800 模型能力评测结果保存在 `temp/output_*` 目录下。 - -## 6. UI 界面 - -```bash -pip install flask sse_starlette -cd evalscope/web -npm install -npm run build -cd /data1/sora/benchmarks -evalscope service -``` - -访问 `http://127.0.0.1:9000/dashboard`。 diff --git a/bash/generation_config_nothinking.yaml b/bash/config/dpv4-int8_nothinking.yaml similarity index 55% rename from bash/generation_config_nothinking.yaml rename to bash/config/dpv4-int8_nothinking.yaml index f813c9d..e6e89e9 100644 --- a/bash/generation_config_nothinking.yaml +++ b/bash/config/dpv4-int8_nothinking.yaml @@ -1,249 +1,193 @@ -terminal_bench_v2: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - max_steps: 500 - per_step_tokens: 49152 - gpqa_diamond: generation_config: temperature: 1.0 top_p: 1.0 stream: true max_tokens: 8192 - hle: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true - max_tokens: 8192 - + max_tokens: 32768 aime24: generation_config: temperature: 1.0 top_p: 1.0 stream: true - max_tokens: 131072 - + max_tokens: 32768 aime25: generation_config: temperature: 1.0 top_p: 1.0 stream: true - max_tokens: 131072 - + max_tokens: 32768 mmlu_pro: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - simple_qa: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - arc: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - bbh: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true - max_tokens: 8192 - -browsecomp: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 49152 - max_steps: 500 - + max_tokens: 32768 live_code_bench: generation_config: temperature: 1.0 top_p: 1.0 stream: true - max_tokens: 524288 + max_tokens: 32768 dataset_args: subset_list: - - release_v6 - -swe_bench_pro: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 32768 - max_steps: 500 - + - release_v6 aime26: generation_config: temperature: 1.0 top_p: 1.0 stream: true - max_tokens: 131072 - + max_tokens: 32768 hmmt26: generation_config: temperature: 1.0 top_p: 1.0 stream: true - max_tokens: 131072 - + max_tokens: 32768 imo_answerbench: generation_config: temperature: 1.0 top_p: 1.0 stream: true - max_tokens: 524288 - agent_config: - max_steps: 500 - + max_tokens: 32768 super_gpqa: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - drop: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true - max_tokens: 8192 - + max_tokens: 32768 hellaswag: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - mmlu: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - openai_mrcr: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - -mcp_atlas: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 49152 - max_steps: 500 - -swe_bench_multilingual_agentic: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 32768 - max_steps: 500 - -swe_bench_verified: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 32768 - max_steps: 500 - bigcodebench: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true - max_tokens: 524288 - + max_tokens: 32768 humaneval: generation_config: temperature: 1.0 top_p: 1.0 stream: true - max_tokens: 524288 - + max_tokens: 32768 gsm8k: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true - max_tokens: 131072 - + max_tokens: 32768 competition_math: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true - max_tokens: 131072 - + max_tokens: 32768 cmmlu: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - trivia_qa: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - winogrande: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - longbench_v2: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true max_tokens: 8192 - + dataset_args: + subset_list: + - short + - medium + - long tau2_bench: generation_config: - temperature: 1.0 + temperature: 0.0 top_p: 1.0 stream: true - max_tokens: 524288 + max_tokens: 16384 + dataset_args: + extra_params: + user_model: deepseek-v4-pro + api_key: sk-9ed86ef546ca47e3afa7c3b014dea268 + api_base: https://api.deepseek.com/v1 + generation_config: + temperature: 0.0 + max_tokens: 4096 agent_config: - per_step_tokens: 32768 - max_steps: 500 + mode: native + strategy: react + max_steps: 50 +general_fc: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 4096 +bfcl_v3: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 4096 + parallel_tool_calls: true + dataset_args: + extra_params: + is_fc_model: true + underscore_to_dot: true diff --git a/bash/fill_simpleqa_bfcl.py b/bash/fill_simpleqa_bfcl.py new file mode 100644 index 0000000..d9f238c --- /dev/null +++ b/bash/fill_simpleqa_bfcl.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Fill simple_qa (fresh re-run) + bfcl_v3 (official OVERALL) into the Excel. + +simple_qa: old cells had D=0.0005 (cache-resume artifact) and empty perf. + Overwrite C/D/F-Q from the NEW report (real perf + ~1.9h duration). +bfcl_v3 : top-level score 0.6643 is evalscope's macro-average (non-standard). + Official BFCL score = OVERALL subset = 0.569. Update C only. +""" +import json +import glob +import openpyxl + +XLSX = '/data1/sora/P800模型能力评测结果_统一格式_filled.xlsx' +SHEETS = ['2.0-FULL', '2.0-Lite'] + +# --- read reports --- +sq = json.load(open('/data1/sora/evalscope/output/simple_qa/seed_42/reports/DeepSeek-V4-Flash-Int8/simple_qa.json')) +bc = json.load(open(glob.glob('/data1/sora/evalscope/output/bfcl_v3/seed_42.bak/reports/DeepSeek-V4-Flash-Int8/bfcl_v3.json')[0])) + + +def perf(d): + pm = d.get('perf_metrics') or {} + s = pm.get('summary', {}) or {} + u = s.get('usage', {}) or {} + tf = s.get('ttft') or {} + tp = s.get('tpot') or {} + lat = (s.get('latency') or {}).get('mean') + return { + 'score': d.get('score'), + 'duration_h': (d.get('duration_sec') or 0) / 3600, + 'latency': lat, + 'tps': (s.get('throughput') or {}).get('avg_output_tps'), + 'qps': (1.0 / lat) if lat else None, + 'in_tok': (u.get('input_tokens') or {}).get('mean'), + 'out_tok': (u.get('output_tokens') or {}).get('mean'), + 'ttc': u.get('total_tokens_count'), + 'ttft_m': tf.get('mean'), + 'ttft90': tf.get('90%') or tf.get('p90'), + 'ttft99': tf.get('99%') or tf.get('p99'), + 'tpot_m': tp.get('mean'), + 'tpot90': tp.get('90%') or tp.get('p90'), + 'tpot99': tp.get('99%') or tp.get('p99'), + } + + +def bfcl_overall(d): + for metric in d.get('metrics', []): + for cat in metric.get('categories', []): + for sub in cat.get('subsets', []): + nm = sub.get('name') + if nm == 'OVERALL' or (isinstance(nm, list) and 'OVERALL' in nm): + return sub.get('score') + return None + + +P = perf(sq) +BC_OVERALL = bfcl_overall(bc) +print('simple_qa:', {k: (round(v, 4) if isinstance(v, float) else v) for k, v in P.items()}) +print('bfcl_v3 OVERALL:', BC_OVERALL) + +# col -> simple_qa perf key +SQ = {'C': P['score'], 'D': round(P['duration_h'], 4), 'F': P['latency'], 'G': P['tps'], + 'H': P['qps'], 'I': P['in_tok'], 'J': P['out_tok'], 'K': P['ttc'], + 'L': P['ttft_m'], 'M': P['ttft90'], 'N': P['ttft99'], + 'O': P['tpot_m'], 'P': P['tpot90'], 'Q': P['tpot99']} + +wb = openpyxl.load_workbook(XLSX) +for sn in SHEETS: + ws = wb[sn] + for r in range(2, 30): + b = ws.cell(r, 2).value + if b == 'simple_qa': + for col, v in SQ.items(): + if v is not None: + ws[f'{col}{r}'] = int(round(v)) if col == 'K' else round(v, 4) if col in ('C', 'D') else v + print(f'{sn} simple_qa row{r}: C={ws[f"C{r}"].value} D={ws[f"D{r}"].value} F={ws[f"F{r}"].value} K={ws[f"K{r}"].value}') + elif b == 'bfcl_v3' and BC_OVERALL is not None: + ws.cell(r, 3).value = round(BC_OVERALL, 4) + print(f'{sn} bfcl_v3 row{r}: C={ws.cell(r,3).value}') + # recompute total D + tr = next((rr for rr in range(2, ws.max_row + 1) if ws.cell(rr, 2).value == '总计'), None) + if tr: + tot = 0.0 + for rr in range(2, tr): + v = ws.cell(rr, 4).value + try: + tot += float(v) + except (TypeError, ValueError): + pass + ws.cell(tr, 4).value = round(tot, 4) + print(f'{sn} 总计 D = {round(tot, 4)}h') + +wb.save(XLSX) +print('saved ->', XLSX) diff --git a/bash/generation_config_thinking.yaml b/bash/generation_config_thinking.yaml deleted file mode 100644 index 6853ad6..0000000 --- a/bash/generation_config_thinking.yaml +++ /dev/null @@ -1,249 +0,0 @@ -terminal_bench_v2: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - max_steps: 500 - per_step_tokens: 49152 - -gpqa_diamond: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -hle: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -aime24: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -aime25: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -mmlu_pro: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -simple_qa: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -arc: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -bbh: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -browsecomp: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 49152 - max_steps: 500 - -live_code_bench: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - dataset_args: - subset_list: - - release_v6 - -swe_bench_pro: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 32768 - max_steps: 500 - -aime26: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -hmmt26: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -imo_answerbench: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - max_steps: 500 - -super_gpqa: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -drop: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -hellaswag: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -mmlu: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -openai_mrcr: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -mcp_atlas: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 49152 - max_steps: 500 - -swe_bench_multilingual_agentic: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 32768 - max_steps: 500 - -swe_bench_verified: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 32768 - max_steps: 500 - -bigcodebench: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - -humaneval: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - -gsm8k: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -competition_math: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -cmmlu: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -trivia_qa: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -winogrande: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -longbench_v2: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 131072 - -tau2_bench: - generation_config: - temperature: 1.0 - top_p: 1.0 - stream: true - max_tokens: 524288 - agent_config: - per_step_tokens: 32768 - max_steps: 500 diff --git a/bash/rerun_bigcodebench_review.py b/bash/rerun_bigcodebench_review.py new file mode 100644 index 0000000..c73a66b --- /dev/null +++ b/bash/rerun_bigcodebench_review.py @@ -0,0 +1,38 @@ +"""Re-run only the review stage for bigcodebench using existing predictions. + +The upstream `bigcodebench/bigcodebench-evaluate` image has an ENTRYPOINT that +runs the evaluator and exits, which kills the ms_enclave sandbox containers. +We use the locally built `bigcodebench-sandbox:latest` image (same libraries, +entrypoint dropped, runs `tail -f /dev/null`) and re-score the cached +predictions without regenerating them. +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +import run as run_mod +from evalscope import run_task + +DATASET_NAME = 'bigcodebench' +BATCH_SIZE = 4 + +if DATASET_NAME not in run_mod.DATASET_CONFIGS: + raise SystemExit(f'{DATASET_NAME} not found in config') + +ds_cfg = run_mod.DATASET_CONFIGS[DATASET_NAME] +task_cfg = run_mod.build_task_config( + dataset_name=DATASET_NAME, + ds_cfg=ds_cfg, + batch_size=BATCH_SIZE, + enable_thinking=run_mod.ENABLE_THINKING, + seed=run_mod.SEED, + run_idx=0, +) +task_cfg.rerun_review = True + +print(f'Re-running review for {DATASET_NAME}') +print(f'Cache/work dir: {task_cfg.work_dir}') +print(f'Sandbox config: {task_cfg.sandbox.default_config}') + +run_task(task_cfg) +print('Done.') diff --git a/bash/run.py b/bash/run.py index 7136bee..35a7ce1 100644 --- a/bash/run.py +++ b/bash/run.py @@ -1,161 +1,402 @@ -# cd /data1/p800_deploy_scripts/DeepSeek-V4-Flash-xsglang-20260511-deploy -# docker rm -f /deepseek-v4-flash-xsglang-0511 -# bash /data1/p800_deploy_scripts/DeepSeek-V4-Flash-xsglang-20260511-deploy/scripts/docker_run.sh -# bash scripts/run_server.sh - -import os import yaml from copy import deepcopy +from pathlib import Path +import sys +import os from evalscope import run_task, TaskConfig from evalscope.api.agent import NativeAgentConfig +from evalscope.config import SandboxTaskConfig + +SCRIPT_DIR = Path(__file__).parent.resolve() +PROJECT_ROOT = SCRIPT_DIR.parent # ============================================================ -# 1. 用户可配置项 +# ★★★ 必改参数 (USER CONFIG) ★★★ +# 每次评测新模型前，只需要检查/修改以下参数。 +# 也可以通过命令行覆盖：--model / --api-url / --dataset-dir / +# --output-dir / --limit / --config # ============================================================ -CONFIG_PATH = '/data1/benchmark/evalscope/bash/generation_config_nothinking.yaml' -# 显式开关：True 表示开启模型的 thinking 能力，False 表示关闭 +# 模型名（served model name） +MODEL = 'DeepSeek-V4-Flash-Int8' + +# 模型服务地址（OpenAI 兼容 API） +API_URL = 'http://localhost:30000/v1' + +# 本地数据集根目录（提前下载好的 datasets 目录） +DATASET_DIR = str(PROJECT_ROOT / 'datasets') + +# 评测输出目录（每个 benchmark 单独子目录） +OUTPUT_DIR = str(PROJECT_ROOT / 'output') + +# 采样数量上限；None 表示跑全部样本 +LIMIT = None + +# 每个 benchmark 的生成参数配置文件（max_tokens / temperature 等） +CONFIG = None + +# 是否开启 thinking 模式（sglang chat_template_kwargs.thinking） ENABLE_THINKING = False -# 选择本次实际要跑的数据集 -DATASETS = [ - # 'aime26', - # 'terminal_bench_v2', - # 'gpqa_diamond', - # 'hle', - # 'aime24', - # 'aime25', - # 'mmlu_pro', - # 'simple_qa', - # 'arc', - # 'bbh', - # 'browsecomp', - # 'live_code_bench', - # 'swe_bench_pro', - # 'hmmt26', - # 'imo_answerbench', - # 'super_gpqa', - # 'drop', - # 'hellaswag', - # 'mmlu', - # 'openai_mrcr', - # 'mcp_atlas', - # 'swe_bench_multilingual_agentic', - # 'swe_bench_verified', - 'bigcodebench', - # 'humaneval', - # 'gsm8k', - # 'competition_math', - # 'cmmlu', - # 'trivia_qa', - # 'winogrande', - # 'longbench_v2', - # 'tau2_bench', -] +# 测试哪些 benchmark：见下方 DATASETS = _multi_run_order + _single_run_order + _agent_order +# 想只跑部分 benchmark 时，注释掉对应行即可。 +MODEL_PATH = '/data1/models/DeepSeek-V4-Flash-INT8' -# 公共配置（所有数据集共享） -COMMON_CONFIG = { - 'seed': 42, - 'collect_perf': True, - 'no_timestamp': True, - 'model': 'DeepSeek-V4-Flash-Int8', - 'api_url': 'http://localhost:30000/v1', - # 'api_key': '', +# Judge model for benchmarks that require LLM-as-judge (simple_qa, tau2_bench, etc.) +# Vectron endpoint with DeepSeek-V4-Pro +JUDGE_MODEL_ARGS = { + 'model_id': 'DeepSeek/DeepSeek-V4-Pro', + 'api_url': 'https://api.vectron.meta-stone.com/v1', + 'api_key': 'sk-dbd8a665f7634081b87ec409c7636500', 'eval_type': 'openai_api', - 'dataset_dir': '/data1/benchmark/bash/datasets', - 'judge_model_args': { - 'model_id': 'deepseek-v4-pro', - 'api_url': 'https://api.deepseek.com/v1', - 'api_key': 'sk-9ed86ef546ca47e3afa7c3b014dea268', - 'eval_type': 'openai_api', - 'generation_config': { - 'temperature': 0.0, - 'max_tokens': 1024 * 10, - }, + 'generation_config': { + 'temperature': 0.0, + 'max_tokens': 10240, }, } -# 数学类任务集合 +TRUNCATION_CONFIG = { + 'longbench_v2': 32768*4, + 'openai_mrcr': 32768*4, +} +# Combine all datasets in order (shortest estimated time first) +# 1. Multi-run small benchmarks (temperature=1.0 for sampling diversity) +_multi_run_order = [ + # 'humaneval', # ~164 samples x 3 runs = 492 + # 'live_code_bench', # ~100 samples x 5 runs = 500 + # 'aime26', # ~30 samples x 12 runs = 360 + # 'aime24', # ~30 samples x 12 runs = 360 + # 'aime25', # ~30 samples x 12 runs = 360 + # 'gpqa_diamond', # ~198 samples x 2 runs = 396 + # 'imo_answerbench', # ~120 samples x 4 runs = 480 + # 'hmmt26', # ~30 samples x 12 runs = 360 +] + +# 2. Single-run benchmarks (temperature=0.0 greedy) +_single_run_order = [ + # 'arc', + # 'bfcl_v3', # function calling: greedy decoding + # 'winogrande', + # 'competition_math', + # 'gsm8k', + # 'hellaswag', + # 'bigcodebench', + # 'drop', + # 'bbh', + # 'openai_mrcr', + # 'longbench_v2', + # 'mmlu', + # 'cmmlu', + # 'simple_qa', + # 'mmlu_pro', + 'hle', + # 'trivia_qa', +] + +# 3. Agent / tool benchmarks (last) +_agent_order = [ + # 'tau2_bench', + # 'general_fc', +] +# ============================================================ +# Command line argument overrides (不需要修改) +# ============================================================ + +for i, arg in enumerate(sys.argv): + if arg == '--limit' and i + 1 < len(sys.argv): + limit_val = sys.argv[i + 1] + if limit_val.lower() == 'none' or limit_val.lower() == 'all': + LIMIT = None + else: + LIMIT = int(limit_val) + elif arg == '--model' and i + 1 < len(sys.argv): + MODEL = sys.argv[i + 1] + elif arg == '--api-url' and i + 1 < len(sys.argv): + API_URL = sys.argv[i + 1] + elif arg == '--dataset-dir' and i + 1 < len(sys.argv): + DATASET_DIR = sys.argv[i + 1] + elif arg == '--output-dir' and i + 1 < len(sys.argv): + OUTPUT_DIR = sys.argv[i + 1] + elif arg == '--config' and i + 1 < len(sys.argv): + CONFIG = sys.argv[i + 1] + +# Benchmarks that require sandboxed code execution +SANDBOX_DATASETS = {'humaneval', 'bigcodebench'} +SANDBOX_CONFIGS = { + 'bigcodebench': { + 'image': 'bigcodebench-sandbox:latest', + 'working_dir': '/tmp', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }, + 'humaneval': { + 'image': 'python:3.11-slim', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }, +} + +# ============================================================ +# User-tunable parameters +# ============================================================ + +# Fixed seed for reproducibility. With temperature > 0 the model still samples +# randomly, so running N times with the same seed still yields variance. +SEED = 42 + +# Benchmarks to run multiple times with temperature=1.0. +# Format: {benchmark_name: num_runs} +# Number of runs chosen so total samples ≈ 400-500 per benchmark. +MULTI_RUN_CONFIG = { + # ~30 samples each -> 12 runs = ~360 samples + 'aime24': 12, + 'aime25': 12, + 'aime26': 12, + 'hmmt26': 12, + # ~100 samples -> 5 runs = ~500 samples + 'live_code_bench': 5, + # ~120 samples -> 4 runs = ~480 samples + 'imo_answerbench': 4, + # ~164 samples -> 3 runs = ~492 samples + 'humaneval': 3, + # ~198 samples -> 2 runs = ~396 samples + 'gpqa_diamond': 2, +} + +# Single-run benchmarks (temperature=0.0 greedy) +SINGLE_RUN_DATASETS = [ + 'bigcodebench', + 'bfcl_v3', # function calling: greedy decoding + 'competition_math', + 'gsm8k', + 'hle', + 'super_gpqa', + 'arc', + 'bbh', + 'cmmlu', + 'drop', + 'hellaswag', + 'mmlu', + 'mmlu_pro', + 'simple_qa', + 'trivia_qa', + 'winogrande', + 'openai_mrcr', # long-context, run once + 'longbench_v2', # long-context, run once +] + +# Agent / tool benchmarks (temperature=0.0 greedy, run once) +AGENT_DATASETS = [ + 'tau2_bench', + 'general_fc', +] + + + +DATASETS = _multi_run_order + _single_run_order + _agent_order + +# ENABLE_THINKING 已移至文件头部「必改参数」区 +BATCH_SIZE_LIST = [4] +# LIMIT is parsed from command line: --limit N +SHUFFLE = True + +# ============================================================ +# Fixed configuration +# ============================================================ + +CONFIG_PATH = str(Path(CONFIG) if CONFIG else SCRIPT_DIR / 'config' / 'dpv4-int8_nothinking.yaml') +if not Path(CONFIG_PATH).exists(): + raise FileNotFoundError(f'Config file not found: {CONFIG_PATH}') + + MATH_DATASETS = { 'aime24', 'aime25', 'aime26', 'hmmt26', 'gsm8k', 'competition_math', 'imo_answerbench', } -# 数学类任务 prompt 模板 MATH_PROMPT_TEMPLATE = ( "{question}\n" "Please reason step by step, and put your final answer within \\boxed{{}}." ) -batch_size_list = [16] - - -# ============================================================ -# 2. 加载 YAML 配置 -# ============================================================ with open(CONFIG_PATH, 'r', encoding='utf-8') as f: DATASET_CONFIGS = yaml.safe_load(f) # ============================================================ -# 3. 辅助函数 +# Middle-truncation helpers # ============================================================ + +_TOKENIZER = None + + +def get_tokenizer(): + global _TOKENIZER + if _TOKENIZER is None: + from transformers import AutoTokenizer + try: + # Try local path first + _TOKENIZER = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True) + except Exception: + # Fallback to HuggingFace model ID + _TOKENIZER = AutoTokenizer.from_pretrained('deepseek-ai/DeepSeek-V4-Flash', trust_remote_code=True) + return _TOKENIZER + + +def truncate_middle(text: str, max_tokens: int) -> str: + if max_tokens <= 0: + return text + tokenizer = get_tokenizer() + token_ids = tokenizer.encode(text, add_special_tokens=False) + if len(token_ids) <= max_tokens: + return text + keep_head = max_tokens // 2 + keep_tail = max_tokens - keep_head + truncated_ids = token_ids[:keep_head] + token_ids[-keep_tail:] + return tokenizer.decode(truncated_ids, skip_special_tokens=True) + + +def _patch_adapters_for_truncation(): + from evalscope.benchmarks.longbench_v2.longbench_v2_adapter import LongBenchV2Adapter + from evalscope.benchmarks.openai_mrcr.openai_mrcr_adapter import OpenAIMRCRAdapter + + # Patch LongBenchV2Adapter.format_prompt_template + _orig_longbench_format = LongBenchV2Adapter.format_prompt_template + def _patched_longbench_format(self, sample): + max_tok = TRUNCATION_CONFIG.get('longbench_v2') + if max_tok and sample.metadata and 'context' in sample.metadata: + sample.metadata['context'] = truncate_middle(sample.metadata['context'], max_tok) + return _orig_longbench_format(self, sample) + LongBenchV2Adapter.format_prompt_template = _patched_longbench_format + + # Patch OpenAIMRCRAdapter.record_to_sample with needle-aware truncation. + # MRCR is a long chat history with needles hidden at desired_msg_index. + # We keep the head, tail, and a window around the needle, and truncate + # each kept message if it is still too long. This preserves the retrieval + # task while fitting GPU memory. + _orig_mrcr_record = OpenAIMRCRAdapter.record_to_sample + def _patched_mrcr_record(self, record): + import json + max_total_tok = TRUNCATION_CONFIG.get('openai_mrcr') + per_msg_max_tok = 8192 + if max_total_tok and 'prompt' in record: + try: + prompt_data = json.loads(record['prompt']) + if not isinstance(prompt_data, list) or len(prompt_data) == 0: + return _orig_mrcr_record(self, record) + + tokenizer = get_tokenizer() + total_tok = sum( + len(tokenizer.encode(msg.get('content', '') if isinstance(msg, dict) else '', add_special_tokens=False)) + for msg in prompt_data + ) + if total_tok <= max_total_tok: + return _orig_mrcr_record(self, record) + + desired_idx = record.get('desired_msg_index', 0) + if not isinstance(desired_idx, int) or desired_idx < 0 or desired_idx >= len(prompt_data): + desired_idx = 0 + + n = len(prompt_data) + keep = set() + # Head and tail context + keep.update(range(min(2, n))) + keep.update(range(max(0, n - 2), n)) + # Window around the needle + window = 2 + keep.update(range(max(0, desired_idx - window), min(n, desired_idx + window + 1))) + keep = sorted(keep) + + new_prompt = [] + for idx in keep: + msg = prompt_data[idx] + if isinstance(msg, dict): + msg = dict(msg) + content = msg.get('content', '') + if len(tokenizer.encode(content, add_special_tokens=False)) > per_msg_max_tok: + msg['content'] = truncate_middle(content, per_msg_max_tok) + new_prompt.append(msg) + + record = dict(record) + record['prompt'] = json.dumps(new_prompt) + except (json.JSONDecodeError, TypeError): + pass + return _orig_mrcr_record(self, record) + OpenAIMRCRAdapter.record_to_sample = _patched_mrcr_record + + +_patch_adapters_for_truncation() + + +# ============================================================ +# Helpers +# ============================================================ + def configure_thinking(generation_config: dict, enable: bool) -> dict: - """显式开启或关闭 generation_config 中的 thinking 能力。""" + extra_body = generation_config.get('extra_body', {}) + chat_template_kwargs = extra_body.get('chat_template_kwargs', {}) if enable: - generation_config['thinking'] = {'type': 'enabled'} + chat_template_kwargs['thinking'] = True else: - generation_config.pop('thinking', None) + chat_template_kwargs.pop('thinking', None) + if chat_template_kwargs: + extra_body['chat_template_kwargs'] = chat_template_kwargs + if extra_body: + generation_config['extra_body'] = extra_body return generation_config def build_agent_config(agent_cfg: dict) -> NativeAgentConfig: - """从 YAML 的 agent_config 构造 NativeAgentConfig。 - - YAML 里可能包含 strategy 未声明的字段（如 per_step_tokens）， - 这些字段会被放到 kwargs 里，透传给 Agent Strategy。 - """ agent_cfg = deepcopy(agent_cfg or {}) - - # NativeAgentConfig 已知的顶层字段 known_fields = {'mode', 'strategy', 'tools', 'max_steps', 'mcp_servers', 'environment', 'environment_extra'} kwargs = agent_cfg.pop('kwargs', {}) - - # 把未知字段收敛进 kwargs for key in list(agent_cfg.keys()): if key not in known_fields: kwargs[key] = agent_cfg.pop(key) - if kwargs: agent_cfg['kwargs'] = kwargs - return NativeAgentConfig(**agent_cfg) -def build_task_config(dataset_name: str, ds_cfg: dict, batch_size: int, enable_thinking: bool) -> TaskConfig: - """根据数据集配置构造单个 TaskConfig。""" - work_dir = f'/data1/benchmark/temp/output_{batch_size}/{dataset_name}' +def build_task_config(dataset_name: str, ds_cfg: dict, batch_size: int, enable_thinking: bool, seed: int, run_idx: int = 0) -> TaskConfig: + # Each run gets its own work_dir so use_cache does not reuse predictions + # across repeated samples. This is required for temperature=1.0 multi-run + # benchmarks to actually measure variance. + if run_idx > 0: + work_dir = Path(OUTPUT_DIR) / dataset_name / f'seed_{seed}_run_{run_idx}' + else: + work_dir = Path(OUTPUT_DIR) / dataset_name / f'seed_{seed}' + work_dir.mkdir(parents=True, exist_ok=True) + work_dir = str(work_dir) - # 构造 generation_config，并显式处理 thinking 开关 generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking) - - # 构造 dataset_args：仅对应当前数据集 dataset_args = deepcopy(ds_cfg.get('dataset_args', {})) + dataset_args.setdefault('shuffle', True) - # 数学类任务覆盖 prompt_template if dataset_name in MATH_DATASETS: dataset_args['prompt_template'] = MATH_PROMPT_TEMPLATE dataset_args_dict = {dataset_name: dataset_args} - # 构造 agent_config（如果 YAML 里配了的话） agent_config = None if 'agent_config' in ds_cfg: agent_config = build_agent_config(ds_cfg['agent_config']) return TaskConfig( - **COMMON_CONFIG, + model=MODEL, + api_url=API_URL, + eval_type='openai_api', + dataset_dir=DATASET_DIR, + judge_model_args=JUDGE_MODEL_ARGS, + seed=seed, + limit=LIMIT, + collect_perf=True, + no_timestamp=True, work_dir=work_dir, use_cache=work_dir, datasets=[dataset_name], @@ -163,26 +404,87 @@ def build_task_config(dataset_name: str, ds_cfg: dict, batch_size: int, enable_t dataset_args=dataset_args_dict, agent_config=agent_config, eval_batch_size=batch_size, + sandbox=SandboxTaskConfig( + enabled=True, + engine='docker', + default_config=SANDBOX_CONFIGS.get(dataset_name, { + 'image': 'python:3.11-slim', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }) + ) if dataset_name in SANDBOX_DATASETS else None, ) # ============================================================ -# 4. 主循环 +# Main loop # ============================================================ -if __name__ == '__main__': - for batch_size in batch_size_list: - task_cfg_list = [] - for dataset_name in DATASETS: +def main(): + print(f"Config: {CONFIG_PATH}") + print(f"Model: {MODEL}") + print(f"API URL: {API_URL}") + print(f"Dataset Dir: {DATASET_DIR}") + print(f"Output Dir: {OUTPUT_DIR}") + print(f"Limit: {LIMIT if LIMIT is not None else 'ALL'}") + print(f"Total benchmarks: {len(DATASETS)}") + print("="*60) + + for batch_size in BATCH_SIZE_LIST: + # 1. Run multi-run benchmarks (temperature=1.0, same seed, variance from sampling) + for dataset_name in _multi_run_order: if dataset_name not in DATASET_CONFIGS: - raise ValueError( - f"Dataset '{dataset_name}' 不在 {CONFIG_PATH} 中，" - f"请先在 YAML 里补充它的 generation_config。" - ) - + print(f"WARNING: {dataset_name} not in YAML config, skipping") + continue ds_cfg = DATASET_CONFIGS[dataset_name] - task_cfg = build_task_config(dataset_name, ds_cfg, batch_size, ENABLE_THINKING) - task_cfg_list.append(task_cfg) + num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1) + for run_idx in range(num_runs): + print(f"\n{'='*60}") + print(f"Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={SEED})") + print(f"{'='*60}") + task_cfg = build_task_config(dataset_name, ds_cfg, batch_size, ENABLE_THINKING, SEED, run_idx=run_idx) + try: + run_task(task_cfg) + except Exception as e: + print(f"ERROR in {dataset_name} (run {run_idx + 1}): {e}") + continue - # run_task 支持 list，会串行执行每个数据集的评测 - run_task(task_cfg_list) + # 2. Run single-run benchmarks (temperature=0.0 greedy) + for dataset_name in _single_run_order: + if dataset_name not in DATASET_CONFIGS: + print(f"WARNING: {dataset_name} not in YAML config, skipping") + continue + ds_cfg = DATASET_CONFIGS[dataset_name] + print(f"\n{'='*60}") + print(f"Running: {dataset_name} (seed={SEED})") + print(f"{'='*60}") + task_cfg = build_task_config(dataset_name, ds_cfg, batch_size, ENABLE_THINKING, SEED) + try: + run_task(task_cfg) + except Exception as e: + print(f"ERROR in {dataset_name}: {e}") + continue + + # 3. Run agent benchmarks + for dataset_name in _agent_order: + if dataset_name not in DATASET_CONFIGS: + print(f"WARNING: {dataset_name} not in YAML config, skipping") + continue + ds_cfg = DATASET_CONFIGS[dataset_name] + print(f"\n{'='*60}") + print(f"Running: {dataset_name} (seed={SEED})") + print(f"{'='*60}") + task_cfg = build_task_config(dataset_name, ds_cfg, batch_size, ENABLE_THINKING, SEED) + try: + run_task(task_cfg) + except Exception as e: + print(f"ERROR in {dataset_name}: {e}") + continue + + print("\nAll benchmarks done!") + + +if __name__ == '__main__': + main() diff --git a/bash/run_full.py b/bash/run_full.py new file mode 100644 index 0000000..462c952 --- /dev/null +++ b/bash/run_full.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +""" +Full evaluation suite: ~3-5 days +Runs all benchmarks with full datasets and multiple runs for stability. +Use --limit none (default) for the complete evaluation. +""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).parent)) + +import run as run_module + +# Use the default full configuration from run.py +# (already ordered by estimated runtime) +run_module._multi_run_order = [ + 'humaneval', + 'live_code_bench', + 'aime26', + 'aime24', + 'aime25', + 'gpqa_diamond', + 'imo_answerbench', + 'hmmt26', +] + +run_module._single_run_order = [ + 'arc', + 'bfcl_v3', + 'winogrande', + 'competition_math', + 'gsm8k', + 'hellaswag', + 'bigcodebench', + 'drop', + 'bbh', + 'openai_mrcr', + 'longbench_v2', + 'mmlu', + 'cmmlu', + 'super_gpqa', + 'simple_qa', + 'mmlu_pro', + 'hle', + 'trivia_qa', +] + +run_module._agent_order = [ + 'tau2_bench', + 'general_fc', +] + +run_module.DATASETS = ( + run_module._multi_run_order + + run_module._single_run_order + + run_module._agent_order +) + +# Full: many runs for statistical stability +run_module.MULTI_RUN_CONFIG = { + 'aime24': 12, + 'aime25': 12, + 'aime26': 12, + 'hmmt26': 12, + 'live_code_bench': 5, + 'imo_answerbench': 4, + 'humaneval': 3, + 'gpqa_diamond': 2, +} + +if __name__ == '__main__': + print('Full benchmarks:', run_module.DATASETS) + print('Estimated time: ~3-5 days with --limit none') + run_module.main() diff --git a/bash/run_group1.py b/bash/run_group1.py new file mode 100644 index 0000000..aa0377a --- /dev/null +++ b/bash/run_group1.py @@ -0,0 +1,48 @@ +""" +Group 1: Code + Reasoning + short Knowledge + Agent (est. ~18-22h) +Run on machine 1. +""" +from pathlib import Path +import sys + +# Ensure we can import run.py from the same directory inside or outside Docker +sys.path.insert(0, str(Path(__file__).parent)) + +import run as run_module + +# Override dataset lists for group 1 +run_module._multi_run_order = [ + 'humaneval', + 'live_code_bench', + 'aime26', + 'aime24', + 'aime25', + 'gpqa_diamond', + 'imo_answerbench', + 'hmmt26', +] + +run_module._single_run_order = [ + 'arc', + 'winogrande', + 'competition_math', + 'gsm8k', + 'hellaswag', + 'bigcodebench', +] + +run_module._agent_order = [ + 'bfcl_v3', + 'tau2_bench', +] + +run_module.DATASETS = ( + run_module._multi_run_order + + run_module._single_run_order + + run_module._agent_order +) + +if __name__ == '__main__': + print('Group 1 benchmarks:', run_module.DATASETS) + print('Estimated time: ~18-22h') + run_module.main() diff --git a/bash/run_group2.py b/bash/run_group2.py new file mode 100644 index 0000000..fd33035 --- /dev/null +++ b/bash/run_group2.py @@ -0,0 +1,40 @@ +""" +Group 2: Knowledge + Long-context + Agent (est. ~20-24h) +Run on machine 2. +""" +from pathlib import Path +import sys + +# Ensure we can import run.py from the same directory inside or outside Docker +sys.path.insert(0, str(Path(__file__).parent)) + +import run as run_module + +# Override dataset lists for group 2 +run_module._multi_run_order = [] + +run_module._single_run_order = [ + 'drop', + 'bbh', + 'openai_mrcr', + 'longbench_v2', + 'mmlu', + 'cmmlu', + 'super_gpqa', + 'simple_qa', +] + +run_module._agent_order = [ + 'general_fc', +] + +run_module.DATASETS = ( + run_module._multi_run_order + + run_module._single_run_order + + run_module._agent_order +) + +if __name__ == '__main__': + print('Group 2 benchmarks:', run_module.DATASETS) + print('Estimated time: ~20-24h') + run_module.main() diff --git a/bash/run_group3.py b/bash/run_group3.py new file mode 100644 index 0000000..8917309 --- /dev/null +++ b/bash/run_group3.py @@ -0,0 +1,35 @@ +""" +Group 3: Long-running Knowledge benchmarks (est. ~35-40h) +Run on machine 3. +Note: trivia_qa and hle are inherently slow; consider using --limit to control time. +""" +from pathlib import Path +import sys + +# Ensure we can import run.py from the same directory inside or outside Docker +sys.path.insert(0, str(Path(__file__).parent)) + +import run as run_module + +# Override dataset lists for group 3 +run_module._multi_run_order = [] + +run_module._single_run_order = [ + 'mmlu_pro', + 'hle', + 'trivia_qa', +] + +run_module._agent_order = [] + +run_module.DATASETS = ( + run_module._multi_run_order + + run_module._single_run_order + + run_module._agent_order +) + +if __name__ == '__main__': + print('Group 3 benchmarks:', run_module.DATASETS) + print('Estimated time: ~35-40h (trivia_qa and hle are slow)') + print('Tip: use --limit 500 to cap runtime if needed') + run_module.main() diff --git a/bash/run_lite.py b/bash/run_lite.py new file mode 100644 index 0000000..1b923ea --- /dev/null +++ b/bash/run_lite.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 +""" +Lite evaluation suite: ~2-4h +Covers all 5 capability domains with 1-2 benchmarks each. +Use --limit 20 (default) for quick smoke testing. +""" +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).parent)) + +import run as run_module + +run_module._multi_run_order = [ + 'aime24', # reasoning + 'humaneval', # code +] + +run_module._single_run_order = [ + 'gsm8k', # reasoning + 'mmlu_pro', # knowledge + 'simple_qa', # knowledge + 'longbench_v2', # long-context +] + +run_module._agent_order = [ + 'bfcl_v3', # tool/function calling +] + +run_module.DATASETS = ( + run_module._multi_run_order + + run_module._single_run_order + + run_module._agent_order +) + + +if __name__ == '__main__': + print('Lite benchmarks:', run_module.DATASETS) + run_module.main() + diff --git a/bash/run_single.py b/bash/run_single.py new file mode 100644 index 0000000..13b57f5 --- /dev/null +++ b/bash/run_single.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Run a single benchmark for quick smoke tests. + +Usage: + python bash/run_single.py bigcodebench --limit 1 + python bash/run_single.py bfcl_v3 --limit 5 --model MODEL --api-url URL +""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent)) +import run as run_mod +from evalscope import run_task + +if len(sys.argv) < 2: + print('Usage: python bash/run_single.py [--limit N] [--model M] [--api-url U]') + sys.exit(1) + +dataset_name = sys.argv[1] +limit = None +for i, arg in enumerate(sys.argv): + if arg == '--limit' and i + 1 < len(sys.argv): + v = sys.argv[i + 1] + limit = None if v.lower() in ('none', 'all') else int(v) + elif arg == '--model' and i + 1 < len(sys.argv): + run_mod.MODEL = sys.argv[i + 1] + elif arg == '--api-url' and i + 1 < len(sys.argv): + run_mod.API_URL = sys.argv[i + 1] + +if dataset_name not in run_mod.DATASET_CONFIGS: + raise SystemExit(f'{dataset_name} not in config') + +run_mod.LIMIT = limit +ds_cfg = run_mod.DATASET_CONFIGS[dataset_name] +task_cfg = run_mod.build_task_config( + dataset_name=dataset_name, + ds_cfg=ds_cfg, + batch_size=4, + enable_thinking=run_mod.ENABLE_THINKING, + seed=run_mod.SEED, + run_idx=0, +) +print(f'Running {dataset_name} with limit={limit}') +run_task(task_cfg) +print('Done.') diff --git a/bash/test.py b/bash/test.py index fbdf609..f955845 100644 --- a/bash/test.py +++ b/bash/test.py @@ -1,52 +1,497 @@ +import yaml +from copy import deepcopy +from pathlib import Path +import sys +import os + from evalscope import run_task, TaskConfig -datasets = [ - # 'live_code_bench', - 'aime26', - # 'hmmt26', - # 'imo_answerbench', - # 'super_gpqa', - # 'drop', - # 'hellaswag', - # 'mmlu', - # 'openai_mrcr', - # # 'swe_bench_pro', - # 'mcp_atlas', +from evalscope.api.agent import NativeAgentConfig +from evalscope.config import SandboxTaskConfig + +SCRIPT_DIR = Path(__file__).parent.resolve() +PROJECT_ROOT = SCRIPT_DIR.parent + + +# ============================================================ +# ★★★ 必改参数 (USER CONFIG) ★★★ +# 每次评测新模型前，只需要检查/修改以下参数。 +# 也可以通过命令行覆盖：--model / --api-url / --dataset-dir / +# --output-dir / --limit / --config +# ============================================================ + +# 模型名（served model name） +MODEL = 'DeepSeek-V4-Flash-Int8' + +# 模型服务地址（OpenAI 兼容 API） +API_URL = 'http://localhost:30000/v1' + +# 本地数据集根目录（提前下载好的 datasets 目录） +DATASET_DIR = str(PROJECT_ROOT / 'datasets') + +# 评测输出目录（每个 benchmark 单独子目录） +OUTPUT_DIR = str(PROJECT_ROOT / 'output') + +# 采样数量上限；None 表示跑全部样本 +LIMIT = None + +# 每个 benchmark 的生成参数配置文件（max_tokens / temperature 等） +CONFIG = None + +# 是否开启 thinking 模式（sglang chat_template_kwargs.thinking） +ENABLE_THINKING = False + +# 测试哪些 benchmark：见下方 DATASETS = _multi_run_order + _single_run_order + _agent_order +# 想只跑部分 benchmark 时，注释掉对应行即可。 + +# ============================================================ +# Command line argument overrides (不需要修改) +# ============================================================ + +for i, arg in enumerate(sys.argv): + if arg == '--limit' and i + 1 < len(sys.argv): + limit_val = sys.argv[i + 1] + if limit_val.lower() == 'none' or limit_val.lower() == 'all': + LIMIT = None + else: + LIMIT = int(limit_val) + elif arg == '--model' and i + 1 < len(sys.argv): + MODEL = sys.argv[i + 1] + elif arg == '--api-url' and i + 1 < len(sys.argv): + API_URL = sys.argv[i + 1] + elif arg == '--dataset-dir' and i + 1 < len(sys.argv): + DATASET_DIR = sys.argv[i + 1] + elif arg == '--output-dir' and i + 1 < len(sys.argv): + OUTPUT_DIR = sys.argv[i + 1] + elif arg == '--config' and i + 1 < len(sys.argv): + CONFIG = sys.argv[i + 1] + +# Benchmarks that require sandboxed code execution +SANDBOX_DATASETS = {'humaneval', 'bigcodebench'} +# Per-dataset sandbox configs. +# bigcodebench's upstream image has ENTRYPOINT ["python3", "-m", "bigcodebench.evaluate"], +# which exits immediately and breaks ms_enclave exec. We built a derivative image +# `bigcodebench-sandbox:latest` that keeps the same Python environment but drops the +# entrypoint and runs `tail -f /dev/null` so the container stays alive. +SANDBOX_CONFIGS = { + 'bigcodebench': { + 'image': 'bigcodebench-sandbox:latest', + 'working_dir': '/tmp', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }, + 'humaneval': { + 'image': 'python:3.11-slim', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }, +} + +# ============================================================ +# User-tunable parameters +# ============================================================ + +# Fixed seed for reproducibility. With temperature > 0 the model still samples +# randomly, so running N times with the same seed still yields variance. +SEED = 42 + +# Benchmarks to run multiple times with temperature=1.0. +# Format: {benchmark_name: num_runs} +# Number of runs chosen so total samples ≈ 400-500 per benchmark. +MULTI_RUN_CONFIG = { + # ~30 samples each -> 12 runs = ~360 samples + 'aime24': 12, + 'aime25': 12, + 'aime26': 12, + 'hmmt26': 12, + # ~100 samples -> 5 runs = ~500 samples + 'live_code_bench': 5, + # ~120 samples -> 4 runs = ~480 samples + 'imo_answerbench': 4, + # ~164 samples -> 3 runs = ~492 samples + 'humaneval': 3, + # ~198 samples -> 2 runs = ~396 samples + 'gpqa_diamond': 2, +} + +# Single-run benchmarks (temperature=0.0 greedy) +SINGLE_RUN_DATASETS = [ + 'bigcodebench', + 'bfcl_v3', # function calling: greedy decoding + 'competition_math', + 'gsm8k', + 'hle', + 'super_gpqa', + 'arc', + 'bbh', + 'cmmlu', + 'drop', + 'hellaswag', + 'mmlu', + 'mmlu_pro', + 'simple_qa', + 'trivia_qa', + 'winogrande', + 'openai_mrcr', # long-context, run once + 'longbench_v2', # long-context, run once ] -batch_size_list = [1, 2, 4, 8, 16, 32, 64,128] -for batch_size in batch_size_list: - work_dir = f'/data1/sora/benchmarks/temp/output_{batch_size}' - task_cfg = TaskConfig( - seed=42, - collect_perf= True, + +# Agent / tool benchmarks (temperature=0.0 greedy, run once) +AGENT_DATASETS = [ + 'tau2_bench', + 'general_fc', +] + +# Combine all datasets in order (shortest estimated time first) +# 1. Multi-run small benchmarks (temperature=1.0 for sampling diversity) +_multi_run_order = [ + # 'humaneval', # ~164 samples x 3 runs = 492 + # 'live_code_bench', # ~100 samples x 5 runs = 500 + # 'aime26', # ~30 samples x 12 runs = 360 + # 'aime24', # ~30 samples x 12 runs = 360 + # 'aime25', # ~30 samples x 12 runs = 360 + # 'gpqa_diamond', # ~198 samples x 2 runs = 396 + # 'imo_answerbench', # ~120 samples x 4 runs = 480 + # 'hmmt26', # ~30 samples x 12 runs = 360 +] + +# 2. Single-run benchmarks (temperature=0.0 greedy) +_single_run_order = [ + # 'arc', + # 'bfcl_v3', # function calling: greedy decoding + # 'winogrande', + # 'competition_math', + # 'gsm8k', + # 'hellaswag', + 'bigcodebench', + # 'drop', + # 'bbh', + # 'openai_mrcr', + # 'longbench_v2', + # 'mmlu', + # 'cmmlu', + # 'super_gpqa', + # 'simple_qa', + # 'mmlu_pro', + 'hle', + # 'trivia_qa', +] + +# 3. Agent / tool benchmarks (last) +_agent_order = [ + # 'tau2_bench', + # 'general_fc', +] + +DATASETS = _multi_run_order + _single_run_order + _agent_order + +# ENABLE_THINKING 已移至文件头部「必改参数」区 +BATCH_SIZE_LIST = [4] +# LIMIT is parsed from command line: --limit N +SHUFFLE = True + +# ============================================================ +# Fixed configuration +# ============================================================ + +CONFIG_PATH = str(Path(CONFIG) if CONFIG else SCRIPT_DIR / 'config' / 'dpv4-int8_nothinking.yaml') +if not Path(CONFIG_PATH).exists(): + raise FileNotFoundError(f'Config file not found: {CONFIG_PATH}') + +MODEL_PATH = '/data1/models/DeepSeek-V4-Flash-INT8' + +# Judge model for benchmarks that require LLM-as-judge (simple_qa, tau2_bench, etc.) +# Vectron endpoint with DeepSeek-V4-Pro +JUDGE_MODEL_ARGS = { + 'model_id': 'DeepSeek/DeepSeek-V4-Pro', + 'api_url': 'https://api.vectron.meta-stone.com/v1', + 'api_key': 'sk-dbd8a665f7634081b87ec409c7636500', + 'eval_type': 'openai_api', + 'generation_config': { + 'temperature': 0.0, + 'max_tokens': 10240, + }, +} + +TRUNCATION_CONFIG = { + 'longbench_v2': 32768*4, + 'openai_mrcr': 32768*4, +} + +MATH_DATASETS = { + 'aime24', 'aime25', 'aime26', 'hmmt26', + 'gsm8k', 'competition_math', 'imo_answerbench', +} + +MATH_PROMPT_TEMPLATE = ( + "{question}\n" + "Please reason step by step, and put your final answer within \\boxed{{}}." +) + +with open(CONFIG_PATH, 'r', encoding='utf-8') as f: + DATASET_CONFIGS = yaml.safe_load(f) + + +# ============================================================ +# Middle-truncation helpers +# ============================================================ + +_TOKENIZER = None + + +def get_tokenizer(): + global _TOKENIZER + if _TOKENIZER is None: + from transformers import AutoTokenizer + try: + # Try local path first + _TOKENIZER = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True) + except Exception: + # Fallback to HuggingFace model ID + _TOKENIZER = AutoTokenizer.from_pretrained('deepseek-ai/DeepSeek-V4-Flash', trust_remote_code=True) + return _TOKENIZER + + +def truncate_middle(text: str, max_tokens: int) -> str: + if max_tokens <= 0: + return text + tokenizer = get_tokenizer() + token_ids = tokenizer.encode(text, add_special_tokens=False) + if len(token_ids) <= max_tokens: + return text + keep_head = max_tokens // 2 + keep_tail = max_tokens - keep_head + truncated_ids = token_ids[:keep_head] + token_ids[-keep_tail:] + return tokenizer.decode(truncated_ids, skip_special_tokens=True) + + +def _patch_adapters_for_truncation(): + from evalscope.benchmarks.longbench_v2.longbench_v2_adapter import LongBenchV2Adapter + from evalscope.benchmarks.openai_mrcr.openai_mrcr_adapter import OpenAIMRCRAdapter + + # Patch LongBenchV2Adapter.format_prompt_template + _orig_longbench_format = LongBenchV2Adapter.format_prompt_template + def _patched_longbench_format(self, sample): + max_tok = TRUNCATION_CONFIG.get('longbench_v2') + if max_tok and sample.metadata and 'context' in sample.metadata: + sample.metadata['context'] = truncate_middle(sample.metadata['context'], max_tok) + return _orig_longbench_format(self, sample) + LongBenchV2Adapter.format_prompt_template = _patched_longbench_format + + # Patch OpenAIMRCRAdapter.record_to_sample with needle-aware truncation. + # MRCR is a long chat history with needles hidden at desired_msg_index. + # We keep the head, tail, and a window around the needle, and truncate + # each kept message if it is still too long. This preserves the retrieval + # task while fitting GPU memory. + _orig_mrcr_record = OpenAIMRCRAdapter.record_to_sample + def _patched_mrcr_record(self, record): + import json + max_total_tok = TRUNCATION_CONFIG.get('openai_mrcr') + per_msg_max_tok = 8192 + if max_total_tok and 'prompt' in record: + try: + prompt_data = json.loads(record['prompt']) + if not isinstance(prompt_data, list) or len(prompt_data) == 0: + return _orig_mrcr_record(self, record) + + tokenizer = get_tokenizer() + total_tok = sum( + len(tokenizer.encode(msg.get('content', '') if isinstance(msg, dict) else '', add_special_tokens=False)) + for msg in prompt_data + ) + if total_tok <= max_total_tok: + return _orig_mrcr_record(self, record) + + desired_idx = record.get('desired_msg_index', 0) + if not isinstance(desired_idx, int) or desired_idx < 0 or desired_idx >= len(prompt_data): + desired_idx = 0 + + n = len(prompt_data) + keep = set() + # Head and tail context + keep.update(range(min(2, n))) + keep.update(range(max(0, n - 2), n)) + # Window around the needle + window = 2 + keep.update(range(max(0, desired_idx - window), min(n, desired_idx + window + 1))) + keep = sorted(keep) + + new_prompt = [] + for idx in keep: + msg = prompt_data[idx] + if isinstance(msg, dict): + msg = dict(msg) + content = msg.get('content', '') + if len(tokenizer.encode(content, add_special_tokens=False)) > per_msg_max_tok: + msg['content'] = truncate_middle(content, per_msg_max_tok) + new_prompt.append(msg) + + record = dict(record) + record['prompt'] = json.dumps(new_prompt) + except (json.JSONDecodeError, TypeError): + pass + return _orig_mrcr_record(self, record) + OpenAIMRCRAdapter.record_to_sample = _patched_mrcr_record + + +_patch_adapters_for_truncation() + + +# ============================================================ +# Helpers +# ============================================================ + +def configure_thinking(generation_config: dict, enable: bool) -> dict: + extra_body = generation_config.get('extra_body', {}) + chat_template_kwargs = extra_body.get('chat_template_kwargs', {}) + if enable: + chat_template_kwargs['thinking'] = True + else: + chat_template_kwargs.pop('thinking', None) + if chat_template_kwargs: + extra_body['chat_template_kwargs'] = chat_template_kwargs + if extra_body: + generation_config['extra_body'] = extra_body + return generation_config + + +def build_agent_config(agent_cfg: dict) -> NativeAgentConfig: + agent_cfg = deepcopy(agent_cfg or {}) + known_fields = {'mode', 'strategy', 'tools', 'max_steps', 'mcp_servers', 'environment', 'environment_extra'} + kwargs = agent_cfg.pop('kwargs', {}) + for key in list(agent_cfg.keys()): + if key not in known_fields: + kwargs[key] = agent_cfg.pop(key) + if kwargs: + agent_cfg['kwargs'] = kwargs + return NativeAgentConfig(**agent_cfg) + + +def build_task_config(dataset_name: str, ds_cfg: dict, batch_size: int, enable_thinking: bool, seed: int, run_idx: int = 0) -> TaskConfig: + # Each run gets its own work_dir so use_cache does not reuse predictions + # across repeated samples. This is required for temperature=1.0 multi-run + # benchmarks to actually measure variance. + if run_idx > 0: + work_dir = Path(OUTPUT_DIR) / dataset_name / f'seed_{seed}_run_{run_idx}' + else: + work_dir = Path(OUTPUT_DIR) / dataset_name / f'seed_{seed}' + work_dir.mkdir(parents=True, exist_ok=True) + work_dir = str(work_dir) + + generation_config = configure_thinking(deepcopy(ds_cfg['generation_config']), enable_thinking) + dataset_args = deepcopy(ds_cfg.get('dataset_args', {})) + dataset_args.setdefault('shuffle', True) + + if dataset_name in MATH_DATASETS: + dataset_args['prompt_template'] = MATH_PROMPT_TEMPLATE + + dataset_args_dict = {dataset_name: dataset_args} + + agent_config = None + if 'agent_config' in ds_cfg: + agent_config = build_agent_config(ds_cfg['agent_config']) + + return TaskConfig( + model=MODEL, + api_url=API_URL, + eval_type='openai_api', + dataset_dir=DATASET_DIR, + judge_model_args=JUDGE_MODEL_ARGS, + seed=seed, + limit=LIMIT, + collect_perf=True, + no_timestamp=True, work_dir=work_dir, use_cache=work_dir, - no_timestamp=True, - model='DeepSeek-V4-Flash-Int8', - api_url='http://localhost:30000/v1', - # api_key='', - eval_type='openai_api', - datasets=datasets, - dataset_dir='/data1/sora/benchmarks/bash/datasets', - generation_config = { - 'temperature': 0.0, - 'stream': True, - "max_tokens": 1024 * 32, - }, - dataset_args={ - 'live_code_bench': { - 'subset_list': ['release_v6'] - } - }, - eval_batch_size = batch_size, - judge_model_args = { - "model_id": "deepseek-v4-pro", - "api_url": "https://api.deepseek.com/v1", - "api_key": "sk-9ed86ef546ca47e3afa7c3b014dea268", - "eval_type": "openai_api", - "generation_config": { - "temperature": 0.0, - "max_tokens": 1024 * 10, - }, - } + datasets=[dataset_name], + generation_config=generation_config, + dataset_args=dataset_args_dict, + agent_config=agent_config, + eval_batch_size=batch_size, + sandbox=SandboxTaskConfig( + enabled=True, + engine='docker', + default_config=SANDBOX_CONFIGS.get(dataset_name, { + 'image': 'python:3.11-slim', + 'tools_config': { + 'shell_executor': {}, + 'python_executor': {} + } + }) + ) if dataset_name in SANDBOX_DATASETS else None, ) - run_task(task_cfg) \ No newline at end of file + + +# ============================================================ +# Main loop +# ============================================================ + +def main(): + print(f"Config: {CONFIG_PATH}") + print(f"Model: {MODEL}") + print(f"API URL: {API_URL}") + print(f"Dataset Dir: {DATASET_DIR}") + print(f"Output Dir: {OUTPUT_DIR}") + print(f"Limit: {LIMIT if LIMIT is not None else 'ALL'}") + print(f"Total benchmarks: {len(DATASETS)}") + print("="*60) + + for batch_size in BATCH_SIZE_LIST: + # 1. Run multi-run benchmarks (temperature=1.0, same seed, variance from sampling) + for dataset_name in _multi_run_order: + if dataset_name not in DATASET_CONFIGS: + print(f"WARNING: {dataset_name} not in YAML config, skipping") + continue + ds_cfg = DATASET_CONFIGS[dataset_name] + num_runs = MULTI_RUN_CONFIG.get(dataset_name, 1) + for run_idx in range(num_runs): + print(f"\n{'='*60}") + print(f"Running: {dataset_name} (run {run_idx + 1}/{num_runs}, seed={SEED})") + print(f"{'='*60}") + task_cfg = build_task_config(dataset_name, ds_cfg, batch_size, ENABLE_THINKING, SEED, run_idx=run_idx) + try: + run_task(task_cfg) + except Exception as e: + print(f"ERROR in {dataset_name} (run {run_idx + 1}): {e}") + continue + + # 2. Run single-run benchmarks (temperature=0.0 greedy) + for dataset_name in _single_run_order: + if dataset_name not in DATASET_CONFIGS: + print(f"WARNING: {dataset_name} not in YAML config, skipping") + continue + ds_cfg = DATASET_CONFIGS[dataset_name] + print(f"\n{'='*60}") + print(f"Running: {dataset_name} (seed={SEED})") + print(f"{'='*60}") + task_cfg = build_task_config(dataset_name, ds_cfg, batch_size, ENABLE_THINKING, SEED) + try: + run_task(task_cfg) + except Exception as e: + print(f"ERROR in {dataset_name}: {e}") + continue + + # 3. Run agent benchmarks + for dataset_name in _agent_order: + if dataset_name not in DATASET_CONFIGS: + print(f"WARNING: {dataset_name} not in YAML config, skipping") + continue + ds_cfg = DATASET_CONFIGS[dataset_name] + print(f"\n{'='*60}") + print(f"Running: {dataset_name} (seed={SEED})") + print(f"{'='*60}") + task_cfg = build_task_config(dataset_name, ds_cfg, batch_size, ENABLE_THINKING, SEED) + try: + run_task(task_cfg) + except Exception as e: + print(f"ERROR in {dataset_name}: {e}") + continue + + print("\nAll benchmarks done!") + + +if __name__ == '__main__': + main() diff --git a/evalscope/evalscope/benchmarks/bigcodebench/bigcodebench_adapter.py b/evalscope/evalscope/benchmarks/bigcodebench/bigcodebench_adapter.py index 495173d..33bfa5d 100644 --- a/evalscope/evalscope/benchmarks/bigcodebench/bigcodebench_adapter.py +++ b/evalscope/evalscope/benchmarks/bigcodebench/bigcodebench_adapter.py @@ -179,8 +179,11 @@ class BigCodeBenchAdapter(DefaultDataAdapter): if self._calibrate: solution = problem['code_prompt'] + '\n pass\n' + solution - # Construct full test program: solution + unittest test class - check_program = solution + '\n' + problem['test'] + # Construct full test program: solution + unittest test class + runner. + # The `test` field only defines TestCase subclasses; without a runner the + # script exits 0 regardless of whether tests pass. Append unittest.main() + # so failures produce a non-zero exit code and are scored correctly. + check_program = solution + '\n' + problem['test'] + '\n\nif __name__ == "__main__":\n unittest.main()\n' # Execute in sandbox res = self.execute_code_in_sandbox(code=check_program, timeout=self.review_timeout, language='python') diff --git a/evalscope/evalscope/benchmarks/tau_bench/tau2_bench/generation.py b/evalscope/evalscope/benchmarks/tau_bench/tau2_bench/generation.py index 7810dcd..eb5d591 100644 --- a/evalscope/evalscope/benchmarks/tau_bench/tau2_bench/generation.py +++ b/evalscope/evalscope/benchmarks/tau_bench/tau2_bench/generation.py @@ -98,6 +98,12 @@ def patched_generate( global MODEL_DICT oa_model = MODEL_DICT.get(model) + if oa_model is None: + # Fallback: try 'user' for user simulator models, 'agent' for agent models + if 'user' in MODEL_DICT and MODEL_DICT['user'] is not None: + oa_model = MODEL_DICT['user'] + elif 'agent' in MODEL_DICT and MODEL_DICT['agent'] is not None: + oa_model = MODEL_DICT['agent'] assert oa_model is not None, f'Model {model} not found in MODEL_DICT' oa_messages = to_litellm_messages(messages)