122 lines
3.8 KiB
Bash
Executable File
122 lines
3.8 KiB
Bash
Executable File
#!/bin/bash
|
||
# ============================================================
|
||
# 通用 API 评测脚本
|
||
# 用法:
|
||
# # 方式 1：通过环境变量配置
|
||
# export EVAL_API_KEY="sk-xxxx"
|
||
# export EVAL_API_URL="https://api.example.com/v1"
|
||
# export EVAL_MODEL="gpt-4o"
|
||
# export EVAL_DATASETS="gsm8k,aime24,arc"
|
||
# bash bash/case/api_test_runner.sh
|
||
#
|
||
# # 方式 2：命令行参数覆盖
|
||
# bash bash/case/api_test_runner.sh \
|
||
# --api-key sk-xxxx \
|
||
# --api-url https://api.example.com/v1 \
|
||
# --model gpt-4o \
|
||
# --datasets gsm8k,aime24,arc
|
||
#
|
||
# 修改 datasets 只需改 EVAL_DATASETS 或 --datasets。
|
||
# ============================================================
|
||
|
||
set -euo pipefail
|
||
|
||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||
cd "$ROOT_DIR"
|
||
|
||
# --------------------------------------------------
|
||
# 默认值（可通过环境变量或命令行覆盖）
|
||
# --------------------------------------------------
|
||
API_KEY="${EVAL_API_KEY:-}"
|
||
API_URL="${EVAL_API_URL:-https://api.vectron.meta-stone.com/v1}"
|
||
MODEL="${EVAL_MODEL:-DeepSeek/DeepSeek-V4-Flash}"
|
||
DATASETS="${EVAL_DATASETS:-gsm8k,aime24,arc}"
|
||
FOLDER_NAME="${EVAL_FOLDER_NAME:-API-Test}"
|
||
CONFIG="${EVAL_CONFIG:-config/dpv4-int8_nothinking.yaml}"
|
||
BATCH_SIZE="${EVAL_BATCH_SIZE:-4}"
|
||
LIMIT="${EVAL_LIMIT:-none}"
|
||
SEED="${EVAL_SEED:-42}"
|
||
THINKING="${EVAL_THINKING:-false}"
|
||
DATASET_DIR="${EVAL_DATASET_DIR:-$ROOT_DIR}"
|
||
OUTPUT_DIR="${EVAL_OUTPUT_DIR:-$ROOT_DIR/output}"
|
||
|
||
# --------------------------------------------------
|
||
# 解析命令行参数
|
||
# --------------------------------------------------
|
||
while [[ $# -gt 0 ]]; do
|
||
case "$1" in
|
||
--api-key) API_KEY="$2"; shift 2 ;;
|
||
--api-url) API_URL="$2"; shift 2 ;;
|
||
--model) MODEL="$2"; shift 2 ;;
|
||
--datasets) DATASETS="$2"; shift 2 ;;
|
||
--folder-name) FOLDER_NAME="$2"; shift 2 ;;
|
||
--config) CONFIG="$2"; shift 2 ;;
|
||
--batch-size) BATCH_SIZE="$2"; shift 2 ;;
|
||
--limit) LIMIT="$2"; shift 2 ;;
|
||
--seed) SEED="$2"; shift 2 ;;
|
||
--thinking) THINKING="true"; shift ;;
|
||
--no-thinking) THINKING="false"; shift ;;
|
||
--dataset-dir) DATASET_DIR="$2"; shift 2 ;;
|
||
--output-dir) OUTPUT_DIR="$2"; shift 2 ;;
|
||
-h|--help)
|
||
grep '^# ' "$0" | sed 's/^# //'
|
||
exit 0
|
||
;;
|
||
*) echo "未知参数: $1"; exit 1 ;;
|
||
esac
|
||
done
|
||
|
||
if [[ -z "$API_KEY" ]]; then
|
||
echo "ERROR: 请设置 EVAL_API_KEY 环境变量或传入 --api-key"
|
||
exit 1
|
||
fi
|
||
|
||
export EVALSCOPE_API_KEY="$API_KEY"
|
||
export OPENAI_API_KEY="$API_KEY"
|
||
|
||
# --------------------------------------------------
|
||
# API key 连通性校验
|
||
# --------------------------------------------------
|
||
HEALTH=$(curl -s -o /dev/null -w "%{http_code}" \
|
||
-H "Authorization: Bearer ${API_KEY}" \
|
||
"${API_URL}/models")
|
||
if [[ "$HEALTH" != "200" ]]; then
|
||
echo "ERROR: API key 校验失败，${API_URL}/models 返回 HTTP $HEALTH"
|
||
exit 1
|
||
fi
|
||
echo "API key 校验通过 (${API_URL})"
|
||
|
||
# --------------------------------------------------
|
||
# 组装 run.py 参数
|
||
# --------------------------------------------------
|
||
ARGS=(
|
||
--model "$MODEL"
|
||
--api-url "$API_URL"
|
||
--dataset-dir "$DATASET_DIR"
|
||
--output-dir "$OUTPUT_DIR"
|
||
--folder-name "$FOLDER_NAME"
|
||
--config "$CONFIG"
|
||
--batch-size "$BATCH_SIZE"
|
||
--seed "$SEED"
|
||
--limit "$LIMIT"
|
||
--datasets "$DATASETS"
|
||
)
|
||
|
||
if [[ "$THINKING" == "true" ]]; then
|
||
ARGS+=(--thinking)
|
||
fi
|
||
|
||
echo "============================================================"
|
||
echo "API 评测启动"
|
||
echo "Model: $MODEL"
|
||
echo "API URL: $API_URL"
|
||
echo "Datasets: $DATASETS"
|
||
echo "Folder: $FOLDER_NAME"
|
||
echo "Config: $CONFIG"
|
||
echo "Batch size: $BATCH_SIZE"
|
||
echo "Limit: $LIMIT"
|
||
echo "Thinking: $THINKING"
|
||
echo "============================================================"
|
||
|
||
python bash/run.py "${ARGS[@]}"
|