From 4d54a8c7eabb67fb04d948004660d3dc6b5a2256 Mon Sep 17 00:00:00 2001
From: sora <2075279110@qq.com>
Date: Tue, 18 Aug 2026 03:19:01 +0000
Subject: [PATCH] update benchmark suites and evaluation docs

---
 README.md               | 2 +-
 bash/load_swe_images.sh | 3 ++-
 bash/run.py             | 4 ++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index d4511fd..c48bd8e 100644
--- a/README.md
+++ b/README.md
@@ -237,7 +237,7 @@ docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3.
 | `--folder-name` | `model_name` 或 `model_name_THINKING` | 顶层输出文件夹名 |
 | `--config` | `config/dpv4-int8_nothinking.yaml` | YAML 配置文件路径 |
 | `--tokenizer-path` | `/data1/models/DeepSeek-V4-Flash-INT8` | 本地 tokenizer 路径，用于长文本 middle-truncation |
-| `--suite` | `full` | 预置套件：`full` / `lite` / `mid` / `group1` / `group2` / `group3` / `official` |
+| `--suite` | `full` | 预置套件：`full` / `lite` / `mid` / `full / lite / mid / k3 / tencent_hunyuan / group1 / group2 / group3 / official` |
 | `--datasets` | `None` | 逗号分隔自定义 benchmark 列表，覆盖 `--suite` |
 | `--exclude` | `None` | 从当前 suite 中排除的 benchmark |
 | `--limit` | `None` | 每数据集最多跑几条；`none` / `all` 表示全量 |
diff --git a/bash/load_swe_images.sh b/bash/load_swe_images.sh
index 6c63844..adc19c9 100755
--- a/bash/load_swe_images.sh
+++ b/bash/load_swe_images.sh
@@ -3,13 +3,14 @@
 
 set -e
 
-INPUT_DIR="${1:-/data1/sora/evalscope/docker/swe_images}"
+INPUT_DIR="${1:-/data1/sora/temp/evalstone/docker/swe_images}"
 
 if [ ! -d "$INPUT_DIR" ]; then
     echo "错误：目录不存在 $INPUT_DIR"
     exit 1
 fi
 
+shopt -s nullglob
 TARS=("$INPUT_DIR"/swebench_batch_*.tar.gz)
 TOTAL=${#TARS[@]}
 
diff --git a/bash/run.py b/bash/run.py
index d66a1f9..e99dec5 100644
--- a/bash/run.py
+++ b/bash/run.py
@@ -42,6 +42,7 @@ from pathlib import Path
 os.environ.setdefault('LITELLM_LOCAL_MODEL_COST_MAP', 'True')
 
 import yaml
+sys.path.insert(0, str(Path(__file__).parent.parent / "evalscope"))
 
 from evalscope import run_task, TaskConfig
 from evalscope.api.agent import NativeAgentConfig
@@ -118,6 +119,7 @@ ALL_SINGLE_RUN = [
     'openai_mrcr', 'longbench_v2',
 ]
 ALL_AGENT = ['tau2_bench', 'general_fc']
+K3_SINGLE = ["gpqa_diamond", "hle", "terminal_bench_v2", "browsecomp", "mcp_atlas", "officeqa", "deepsearchqa", "jobbench", "automation_bench"]
 
 # 分组基于 CSV 单次时间 + multi-run 后的 wall time 平衡：
 #   Group1: ~61h | Group2: ~62h | Group3: ~55h
@@ -132,6 +134,8 @@ SUITES = {
         'single': ['gsm8k', 'arc', 'longbench_v2'],
         'agent': ['general_fc'],
     },
+    "k3": {"multi": [], "single": K3_SINGLE, "agent": []},
+    "tencent_hunyuan": {"multi": [], "single": [], "agent": []},
     'mid': {
         'multi': ['aime24', 'humaneval'],
         'single': [