sora/evalstone
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update benchmark suites and evaluation docs

Browse Source
This commit is contained in:
sora 2026-08-18 03:19:01 +00:00
parent 922ee3af43
commit 4d54a8c7ea
3 changed files with 7 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -237,7 +237,7 @@ docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3.
| `--folder-name` | `model_name` 或 `model_name_THINKING` | 顶层输出文件夹名 | | `--folder-name` | `model_name` 或 `model_name_THINKING` | 顶层输出文件夹名 |
| `--config` | `config/dpv4-int8_nothinking.yaml` | YAML 配置文件路径 | | `--config` | `config/dpv4-int8_nothinking.yaml` | YAML 配置文件路径 |
| `--tokenizer-path` | `/data1/models/DeepSeek-V4-Flash-INT8` | 本地 tokenizer 路径，用于长文本 middle-truncation | | `--tokenizer-path` | `/data1/models/DeepSeek-V4-Flash-INT8` | 本地 tokenizer 路径，用于长文本 middle-truncation |
| `--suite` | `full` | 预置套件：`full` / `lite` / `mid` / `group1` / `group2` / `group3` / `official` | | `--suite` | `full` | 预置套件：`full` / `lite` / `mid` / `full / lite / mid / k3 / tencent_hunyuan / group1 / group2 / group3 / official` |
| `--datasets` | `None` | 逗号分隔自定义 benchmark 列表，覆盖 `--suite` | | `--datasets` | `None` | 逗号分隔自定义 benchmark 列表，覆盖 `--suite` |
| `--exclude` | `None` | 从当前 suite 中排除的 benchmark | | `--exclude` | `None` | 从当前 suite 中排除的 benchmark |
| `--limit` | `None` | 每数据集最多跑几条；`none` / `all` 表示全量 | | `--limit` | `None` | 每数据集最多跑几条；`none` / `all` 表示全量 |

3
bash/load_swe_images.sh
View File

@ -3,13 +3,14 @@
set -e set -e
INPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}" INPUT_DIR="${1:-/data1/sora/temp/evalstone/docker/swe_images}"
if [ ! -d "$INPUT_DIR" ]; then if [ ! -d "$INPUT_DIR" ]; then
echo "错误：目录不存在 $INPUT_DIR" echo "错误：目录不存在 $INPUT_DIR"
exit 1 exit 1
fi fi
shopt -s nullglob
TARS=("$INPUT_DIR"/swebench_batch_*.tar.gz) TARS=("$INPUT_DIR"/swebench_batch_*.tar.gz)
TOTAL=${#TARS[@]} TOTAL=${#TARS[@]}

4
bash/run.py
View File

@ -42,6 +42,7 @@ from pathlib import Path
os.environ.setdefault('LITELLM_LOCAL_MODEL_COST_MAP', 'True') os.environ.setdefault('LITELLM_LOCAL_MODEL_COST_MAP', 'True')
import yaml import yaml
sys.path.insert(0, str(Path(__file__).parent.parent / "evalscope"))
from evalscope import run_task, TaskConfig from evalscope import run_task, TaskConfig
from evalscope.api.agent import NativeAgentConfig from evalscope.api.agent import NativeAgentConfig
@ -118,6 +119,7 @@ ALL_SINGLE_RUN = [
'openai_mrcr', 'longbench_v2', 'openai_mrcr', 'longbench_v2',
] ]
ALL_AGENT = ['tau2_bench', 'general_fc'] ALL_AGENT = ['tau2_bench', 'general_fc']
K3_SINGLE = ["gpqa_diamond", "hle", "terminal_bench_v2", "browsecomp", "mcp_atlas", "officeqa", "deepsearchqa", "jobbench", "automation_bench"]
# 分组基于 CSV 单次时间 + multi-run 后的 wall time 平衡： # 分组基于 CSV 单次时间 + multi-run 后的 wall time 平衡：
# Group1: ~61h | Group2: ~62h | Group3: ~55h # Group1: ~61h | Group2: ~62h | Group3: ~55h
@ -132,6 +134,8 @@ SUITES = {
'single': ['gsm8k', 'arc', 'longbench_v2'], 'single': ['gsm8k', 'arc', 'longbench_v2'],
'agent': ['general_fc'], 'agent': ['general_fc'],
}, },
"k3": {"multi": [], "single": K3_SINGLE, "agent": []},
"tencent_hunyuan": {"multi": [], "single": [], "agent": []},
'mid': { 'mid': {
'multi': ['aime24', 'humaneval'], 'multi': ['aime24', 'humaneval'],
'single': [ 'single': [