feat: refactor GLM52_API_TEST1.sh to generic API runner and add API test README
This commit is contained in:
sora
parent
11d921071b
commit
025763be2e
159
bash/case/GLM52_API_TEST1.sh
Executable file
159
bash/case/GLM52_API_TEST1.sh
Executable file
@ -0,0 +1,159 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# 通用 API 模型评测脚本（原 GLM52_API_TEST1.sh 升级版）
|
||||
#
|
||||
# 用法:
|
||||
# # 1. 环境变量方式（推荐，避免命令行泄露 key）
|
||||
# export EVAL_API_KEY="sk-xxxx"
|
||||
# export EVAL_API_URL="https://api.example.com/v1"
|
||||
# export EVAL_MODEL="glm-5.2"
|
||||
# export EVAL_DATASETS="gsm8k,aime24,arc"
|
||||
# bash bash/case/GLM52_API_TEST1.sh
|
||||
#
|
||||
# # 2. 命令行方式
|
||||
# bash bash/case/GLM52_API_TEST1.sh \
|
||||
# --api-key sk-xxxx \
|
||||
# --api-url https://api.example.com/v1 \
|
||||
# --model glm-5.2 \
|
||||
# --datasets gsm8k,aime24,arc
|
||||
#
|
||||
# # 3. 使用内置 mode（quick / lite / mid / full / official / custom）
|
||||
# bash bash/case/GLM52_API_TEST1.sh --mode quick
|
||||
#
|
||||
# 修改 datasets：改 EVAL_DATASETS 环境变量或 --datasets 参数即可。
|
||||
# ============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 默认配置（可通过环境变量或命令行覆盖）
|
||||
# --------------------------------------------------
|
||||
API_KEY="${EVAL_API_KEY:-}"
|
||||
API_URL="${EVAL_API_URL:-https://api.vectron.meta-stone.com/v1}"
|
||||
MODEL="${EVAL_MODEL:-DeepSeek/DeepSeek-V4-Flash}"
|
||||
DATASETS="${EVAL_DATASETS:-gsm8k,aime24,arc}"
|
||||
FOLDER_NAME="${EVAL_FOLDER_NAME:-API-Test}"
|
||||
CONFIG="${EVAL_CONFIG:-config/dpv4-int8_nothinking.yaml}"
|
||||
BATCH_SIZE="${EVAL_BATCH_SIZE:-4}"
|
||||
LIMIT="${EVAL_LIMIT:-none}"
|
||||
SEED="${EVAL_SEED:-42}"
|
||||
THINKING="${EVAL_THINKING:-false}"
|
||||
MODE="${EVAL_MODE:-custom}"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 解析命令行参数
|
||||
# --------------------------------------------------
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--api-key) API_KEY="$2"; shift 2 ;;
|
||||
--api-url) API_URL="$2"; shift 2 ;;
|
||||
--model) MODEL="$2"; shift 2 ;;
|
||||
--datasets) DATASETS="$2"; shift 2 ;;
|
||||
--folder-name) FOLDER_NAME="$2"; shift 2 ;;
|
||||
--config) CONFIG="$2"; shift 2 ;;
|
||||
--batch-size) BATCH_SIZE="$2"; shift 2 ;;
|
||||
--limit) LIMIT="$2"; shift 2 ;;
|
||||
--seed) SEED="$2"; shift 2 ;;
|
||||
--thinking) THINKING="true"; shift ;;
|
||||
--no-thinking) THINKING="false"; shift ;;
|
||||
--mode) MODE="$2"; shift 2 ;;
|
||||
-h|--help)
|
||||
grep '^# ' "$0" | sed 's/^# //'
|
||||
exit 0
|
||||
;;
|
||||
*) echo "未知参数: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
# --------------------------------------------------
|
||||
# 模式预设：按需改这里即可扩展常用组合
|
||||
# --------------------------------------------------
|
||||
case "$MODE" in
|
||||
quick)
|
||||
DATASETS="gsm8k,aime24,arc"
|
||||
LIMIT="20"
|
||||
;;
|
||||
lite)
|
||||
DATASETS="gsm8k,aime24,humaneval,arc"
|
||||
LIMIT="none"
|
||||
;;
|
||||
mid)
|
||||
DATASETS="gsm8k,aime24,aime25,bbh,humaneval,arc,simple_qa"
|
||||
LIMIT="none"
|
||||
;;
|
||||
full)
|
||||
DATASETS="bigcodebench,humaneval,live_code_bench,aime24,aime25,aime26,hmmt26,imo_answerbench,gsm8k,competition_math,bbh,drop,gpqa_diamond,mmlu_pro,simple_qa,mmlu,cmmlu,arc,hellaswag,trivia_qa,winogrande,longbench_v2,openai_mrcr,general_fc,bfcl_v3"
|
||||
LIMIT="none"
|
||||
;;
|
||||
official)
|
||||
# 与 DP4-Flash 官方套件对齐
|
||||
DATASETS="bigcodebench,humaneval,live_code_bench,aime24,aime25,aime26,hmmt26,imo_answerbench,gsm8k,competition_math,bbh,drop,gpqa_diamond,mmlu_pro,simple_qa,mmlu,cmmlu,arc,hellaswag,trivia_qa,winogrande,longbench_v2,openai_mrcr,tau2_bench,general_fc,bfcl_v3"
|
||||
LIMIT="none"
|
||||
;;
|
||||
custom)
|
||||
# 使用 DATASETS 环境变量或命令行传入的值
|
||||
;;
|
||||
*)
|
||||
echo "未知模式: $MODE"
|
||||
echo "可用模式: quick | lite | mid | full | official | custom"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# --------------------------------------------------
|
||||
# API key 校验
|
||||
# --------------------------------------------------
|
||||
if [[ -z "$API_KEY" ]]; then
|
||||
echo "ERROR: 请设置 EVAL_API_KEY 环境变量或传入 --api-key"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export EVALSCOPE_API_KEY="$API_KEY"
|
||||
export OPENAI_API_KEY="$API_KEY"
|
||||
|
||||
HEALTH=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: Bearer ${API_KEY}" \
|
||||
"${API_URL}/models")
|
||||
if [[ "$HEALTH" != "200" ]]; then
|
||||
echo "ERROR: API key 校验失败，${API_URL}/models 返回 HTTP $HEALTH"
|
||||
exit 1
|
||||
fi
|
||||
echo "API key 校验通过 (${API_URL})"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 组装 run.py 参数
|
||||
# --------------------------------------------------
|
||||
ARGS=(
|
||||
--model "$MODEL"
|
||||
--api-url "$API_URL"
|
||||
--dataset-dir "$ROOT_DIR"
|
||||
--output-dir "$ROOT_DIR/output"
|
||||
--folder-name "$FOLDER_NAME"
|
||||
--config "$CONFIG"
|
||||
--batch-size "$BATCH_SIZE"
|
||||
--seed "$SEED"
|
||||
--limit "$LIMIT"
|
||||
--datasets "$DATASETS"
|
||||
)
|
||||
|
||||
if [[ "$THINKING" == "true" ]]; then
|
||||
ARGS+=(--thinking)
|
||||
fi
|
||||
|
||||
echo "============================================================"
|
||||
echo "API 评测启动"
|
||||
echo "Mode: $MODE"
|
||||
echo "Model: $MODEL"
|
||||
echo "API URL: $API_URL"
|
||||
echo "Datasets: $DATASETS"
|
||||
echo "Folder: $FOLDER_NAME"
|
||||
echo "Config: $CONFIG"
|
||||
echo "Batch size: $BATCH_SIZE"
|
||||
echo "Limit: $LIMIT"
|
||||
echo "Thinking: $THINKING"
|
||||
echo "============================================================"
|
||||
|
||||
python bash/run.py "${ARGS[@]}"
|
||||
161
bash/case/README_API_TEST.md
Normal file
161
bash/case/README_API_TEST.md
Normal file
@ -0,0 +1,161 @@
|
||||
# EvalScope API 测试指南
|
||||
|
||||
本目录提供一键运行 API 模型评测的脚本。
|
||||
|
||||
## 1. 准备 API key
|
||||
|
||||
为了不把 key 写进代码，推荐用环境变量：
|
||||
|
||||
```bash
|
||||
export EVAL_API_KEY="sk-xxxx"
|
||||
```
|
||||
|
||||
如果使用智谱 / Vectron / 自定义 OpenAI-compatible 服务，按需改 `EVAL_API_URL`：
|
||||
|
||||
```bash
|
||||
export EVAL_API_URL="https://api.vectron.meta-stone.com/v1"
|
||||
export EVAL_MODEL="DeepSeek/DeepSeek-V4-Flash"
|
||||
```
|
||||
|
||||
## 2. 快速开始
|
||||
|
||||
### 2.1 测指定 datasets（最常用）
|
||||
|
||||
```bash
|
||||
cd /data1/sora/evalscope
|
||||
|
||||
export EVAL_API_KEY="sk-xxxx"
|
||||
export EVAL_DATASETS="gsm8k,aime24,arc"
|
||||
|
||||
bash bash/case/GLM52_API_TEST1.sh
|
||||
```
|
||||
|
||||
想换 benchmark，改 `EVAL_DATASETS` 即可。常用数据集：
|
||||
|
||||
```text
|
||||
gsm8k, aime24, aime25, aime26, hmmt26, imo_answerbench, competition_math
|
||||
bbh, drop
|
||||
bigcodebench, humaneval, live_code_bench
|
||||
gpqa_diamond, mmlu_pro, simple_qa, mmlu, cmmlu, arc, hellaswag, trivia_qa, winogrande
|
||||
longbench_v2, openai_mrcr
|
||||
tau2_bench, general_fc, bfcl_v3
|
||||
```
|
||||
|
||||
### 2.2 使用内置模式
|
||||
|
||||
```bash
|
||||
bash bash/case/GLM52_API_TEST1.sh --mode quick # 快速冒烟：gsm8k,aime24,arc limit=20
|
||||
bash bash/case/GLM52_API_TEST1.sh --mode lite # lite 套件
|
||||
bash bash/case/GLM52_API_TEST1.sh --mode mid # 中等套件
|
||||
bash bash/case/GLM52_API_TEST1.sh --mode full # 全量（不含 tau2_bench，因为耗时）
|
||||
bash bash/case/GLM52_API_TEST1.sh --mode official # 与 DP4-Flash 官方发布对齐
|
||||
```
|
||||
|
||||
### 2.3 测 GLM5.2
|
||||
|
||||
```bash
|
||||
export EVAL_API_KEY="sk-xxxx"
|
||||
export EVAL_API_URL="https://api.example.com/v1" # 替换为 GLM5.2 的实际 endpoint
|
||||
export EVAL_MODEL="glm-5.2"
|
||||
export EVAL_DATASETS="gsm8k,aime24,arc"
|
||||
export EVAL_FOLDER_NAME="GLM52-API-Test"
|
||||
|
||||
bash bash/case/GLM52_API_TEST1.sh
|
||||
```
|
||||
|
||||
## 3. 常用命令行参数
|
||||
|
||||
| 参数 | 说明 |
|
||||
|---|---|
|
||||
| `--api-key` | API key（也可用 `EVAL_API_KEY`） |
|
||||
| `--api-url` | OpenAI-compatible API 地址 |
|
||||
| `--model` | 模型名，如 `glm-5.2`、`DeepSeek/DeepSeek-V4-Flash` |
|
||||
| `--datasets` | 逗号分隔的 benchmark 列表 |
|
||||
| `--mode` | `quick / lite / mid / full / official / custom` |
|
||||
| `--folder-name` | 输出目录名，默认 `API-Test` |
|
||||
| `--config` | 评测配置 YAML，默认 `config/dpv4-int8_nothinking.yaml` |
|
||||
| `--batch-size` | 并发数，默认 4 |
|
||||
| `--limit` | 每个 benchmark 最多测多少条，`none` 表示全量 |
|
||||
| `--thinking` | 启用 thinking 模式 |
|
||||
| `--no-thinking` | 关闭 thinking 模式 |
|
||||
|
||||
## 4. 查看进度
|
||||
|
||||
评测日志在 `logs/` 目录，最新日志：
|
||||
|
||||
```bash
|
||||
ls -t logs/live_code_bench_thinking_*.log | head -1
|
||||
```
|
||||
|
||||
实时看进度：
|
||||
|
||||
```bash
|
||||
tail -f $(ls -t logs/live_code_bench_thinking_*.log | head -1)
|
||||
```
|
||||
|
||||
## 5. 结果与成本
|
||||
|
||||
### 5.1 结果位置
|
||||
|
||||
```text
|
||||
output/<FOLDER_NAME>/<benchmark>/seed_42/reports/<benchmark>.json
|
||||
```
|
||||
|
||||
### 5.2 计算成本
|
||||
|
||||
运行完成后，用成本脚本按 token 量算钱：
|
||||
|
||||
```bash
|
||||
# GLM5.2：输入 8 元/M，输出 28 元/M，折扣 0.65
|
||||
bash bash/case/calc_glm52_cost.sh
|
||||
|
||||
# 输出到 results/P800_benchmark_cost_GLM52.csv
|
||||
```
|
||||
|
||||
如果是其他模型，直接调工具：
|
||||
|
||||
```bash
|
||||
python3 tools/calculate_cost.py \
|
||||
--input "P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv" \
|
||||
--input-price 2 \
|
||||
--output-price 8 \
|
||||
--discount 1.0 \
|
||||
--model-name MyModel \
|
||||
--output results/cost_mymodel.csv
|
||||
```
|
||||
|
||||
### 5.3 把成本写回 Excel
|
||||
|
||||
```bash
|
||||
python3 tools/fill_excel_cost.py \
|
||||
--input "/data1/sora/P800模型能力评测结果_统一格式_filled.xlsx" \
|
||||
--output "/data1/sora/P800模型能力评测结果_统一格式_with_cost.xlsx" \
|
||||
--input-price 8 \
|
||||
--output-price 28 \
|
||||
--discount 0.65 \
|
||||
--model-name GLM-5.2
|
||||
```
|
||||
|
||||
## 6. 常见问题
|
||||
|
||||
### 6.1 simple_qa 分数为 0
|
||||
|
||||
通常是 judge API 被限流（HTTP 429）。SimpleQA 需要调用外部 judge 模型打分。解决方案：
|
||||
|
||||
- 换成本地模型当 judge：
|
||||
```bash
|
||||
bash bash/case/GLM52_API_TEST1.sh \
|
||||
--datasets simple_qa \
|
||||
--judge-model /data1/models/DeepSeek-V4-Flash-INT8 \
|
||||
--judge-api-url http://localhost:30000/v1 \
|
||||
--judge-api-key EMPTY
|
||||
```
|
||||
- 或降低 `--batch-size` / `--parallel-runs` 减少 judge 并发。
|
||||
|
||||
### 6.2 评测非常慢
|
||||
|
||||
代码类 benchmark（`live_code_bench`、`bigcodebench`、`humaneval`）需要实际执行生成的代码并跑测试用例，耗时比纯文本生成高很多。可以先 `--limit 10` 测小样本。
|
||||
|
||||
### 6.3 只想重跑失败/漏掉的 benchmark
|
||||
|
||||
直接指定 datasets 即可，EvalScope 会自动跳过已完成的（通过 `use_cache` 恢复）。
|
||||
Loading…
x
Reference in New Issue
Block a user