#!/bin/bash
# ============================================================
# 通用 API 评测脚本
# 用法:
#   # 方式 1：通过环境变量配置
#   export EVAL_API_KEY="sk-xxxx"
#   export EVAL_API_URL="https://api.example.com/v1"
#   export EVAL_MODEL="gpt-4o"
#   export EVAL_DATASETS="gsm8k,aime24,arc"
#   bash bash/case/api_test_runner.sh
#
#   # 方式 2：命令行参数覆盖
#   bash bash/case/api_test_runner.sh \
#       --api-key sk-xxxx \
#       --api-url https://api.example.com/v1 \
#       --model gpt-4o \
#       --datasets gsm8k,aime24,arc
#
# 修改 datasets 只需改 EVAL_DATASETS 或 --datasets。
# ============================================================

set -euo pipefail

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$ROOT_DIR"

# --------------------------------------------------
# 默认值（可通过环境变量或命令行覆盖）
# --------------------------------------------------
API_KEY="${EVAL_API_KEY:-}"
API_URL="${EVAL_API_URL:-https://api.vectron.meta-stone.com/v1}"
MODEL="${EVAL_MODEL:-DeepSeek/DeepSeek-V4-Flash}"
DATASETS="${EVAL_DATASETS:-gsm8k,aime24,arc}"
FOLDER_NAME="${EVAL_FOLDER_NAME:-API-Test}"
CONFIG="${EVAL_CONFIG:-config/dpv4-int8_nothinking.yaml}"
BATCH_SIZE="${EVAL_BATCH_SIZE:-4}"
LIMIT="${EVAL_LIMIT:-none}"
SEED="${EVAL_SEED:-42}"
THINKING="${EVAL_THINKING:-false}"
DATASET_DIR="${EVAL_DATASET_DIR:-$ROOT_DIR}"
OUTPUT_DIR="${EVAL_OUTPUT_DIR:-$ROOT_DIR/output}"

# --------------------------------------------------
# 解析命令行参数
# --------------------------------------------------
while [[ $# -gt 0 ]]; do
    case "$1" in
        --api-key) API_KEY="$2"; shift 2 ;;
        --api-url) API_URL="$2"; shift 2 ;;
        --model) MODEL="$2"; shift 2 ;;
        --datasets) DATASETS="$2"; shift 2 ;;
        --folder-name) FOLDER_NAME="$2"; shift 2 ;;
        --config) CONFIG="$2"; shift 2 ;;
        --batch-size) BATCH_SIZE="$2"; shift 2 ;;
        --limit) LIMIT="$2"; shift 2 ;;
        --seed) SEED="$2"; shift 2 ;;
        --thinking) THINKING="true"; shift ;;
        --no-thinking) THINKING="false"; shift ;;
        --dataset-dir) DATASET_DIR="$2"; shift 2 ;;
        --output-dir) OUTPUT_DIR="$2"; shift 2 ;;
        -h|--help)
            grep '^# ' "$0" | sed 's/^# //'
            exit 0
            ;;
        *) echo "未知参数: $1"; exit 1 ;;
    esac
done

if [[ -z "$API_KEY" ]]; then
    echo "ERROR: 请设置 EVAL_API_KEY 环境变量或传入 --api-key"
    exit 1
fi

export EVALSCOPE_API_KEY="$API_KEY"
export OPENAI_API_KEY="$API_KEY"

# --------------------------------------------------
# API key 连通性校验
# --------------------------------------------------
HEALTH=$(curl -s -o /dev/null -w "%{http_code}" \
    -H "Authorization: Bearer ${API_KEY}" \
    "${API_URL}/models")
if [[ "$HEALTH" != "200" ]]; then
    echo "ERROR: API key 校验失败，${API_URL}/models 返回 HTTP $HEALTH"
    exit 1
fi
echo "API key 校验通过 (${API_URL})"

# --------------------------------------------------
# 组装 run.py 参数
# --------------------------------------------------
ARGS=(
    --model "$MODEL"
    --api-url "$API_URL"
    --dataset-dir "$DATASET_DIR"
    --output-dir "$OUTPUT_DIR"
    --folder-name "$FOLDER_NAME"
    --config "$CONFIG"
    --batch-size "$BATCH_SIZE"
    --seed "$SEED"
    --limit "$LIMIT"
    --datasets "$DATASETS"
)

if [[ "$THINKING" == "true" ]]; then
    ARGS+=(--thinking)
fi

echo "============================================================"
echo "API 评测启动"
echo "Model:        $MODEL"
echo "API URL:      $API_URL"
echo "Datasets:     $DATASETS"
echo "Folder:       $FOLDER_NAME"
echo "Config:       $CONFIG"
echo "Batch size:   $BATCH_SIZE"
echo "Limit:        $LIMIT"
echo "Thinking:     $THINKING"
echo "============================================================"

python bash/run.py "${ARGS[@]}"