diff --git a/bash/case/api_test_config.env.example b/bash/case/api_test_config.env.example new file mode 100644 index 0000000..dcb76bc --- /dev/null +++ b/bash/case/api_test_config.env.example @@ -0,0 +1,24 @@ +# 复制为 api_test_config.env 后填入真实 key,再 source api_test_config.env + +# API 认证 +export EVAL_API_KEY="sk-xxxx" +export EVAL_API_URL="https://api.vectron.meta-stone.com/v1" + +# 模型名(以 API 服务商的 model id 为准) +export EVAL_MODEL="DeepSeek/DeepSeek-V4-Flash" + +# 评测数据集,逗号分隔,想测什么改这里 +# 可用参考:gsm8k,aime24,aime25,aime26,hmmt26,imo_answerbench,competition_math,bbh,drop +# gpqa_diamond,mmlu_pro,simple_qa,mmlu,cmmlu,arc,hellaswag,trivia_qa,winogrande +# longbench_v2,openai_mrcr,general_fc,bfcl_v3,bigcodebench,humaneval,live_code_bench +export EVAL_DATASETS="gsm8k,aime24,arc" + +# 输出目录标识 +export EVAL_FOLDER_NAME="API-Test" + +# 评测配置 +export EVAL_CONFIG="config/dpv4-int8_nothinking.yaml" +export EVAL_BATCH_SIZE=4 +export EVAL_LIMIT="none" +export EVAL_SEED=42 +export EVAL_THINKING="false" diff --git a/bash/case/api_test_runner.sh b/bash/case/api_test_runner.sh new file mode 100755 index 0000000..467cc96 --- /dev/null +++ b/bash/case/api_test_runner.sh @@ -0,0 +1,121 @@ +#!/bin/bash +# ============================================================ +# 通用 API 评测脚本 +# 用法: +# # 方式 1:通过环境变量配置 +# export EVAL_API_KEY="sk-xxxx" +# export EVAL_API_URL="https://api.example.com/v1" +# export EVAL_MODEL="gpt-4o" +# export EVAL_DATASETS="gsm8k,aime24,arc" +# bash bash/case/api_test_runner.sh +# +# # 方式 2:命令行参数覆盖 +# bash bash/case/api_test_runner.sh \ +# --api-key sk-xxxx \ +# --api-url https://api.example.com/v1 \ +# --model gpt-4o \ +# --datasets gsm8k,aime24,arc +# +# 修改 datasets 只需改 EVAL_DATASETS 或 --datasets。 +# ============================================================ + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +# -------------------------------------------------- +# 默认值(可通过环境变量或命令行覆盖) +# -------------------------------------------------- +API_KEY="${EVAL_API_KEY:-}" +API_URL="${EVAL_API_URL:-https://api.vectron.meta-stone.com/v1}" +MODEL="${EVAL_MODEL:-DeepSeek/DeepSeek-V4-Flash}" +DATASETS="${EVAL_DATASETS:-gsm8k,aime24,arc}" +FOLDER_NAME="${EVAL_FOLDER_NAME:-API-Test}" +CONFIG="${EVAL_CONFIG:-config/dpv4-int8_nothinking.yaml}" +BATCH_SIZE="${EVAL_BATCH_SIZE:-4}" +LIMIT="${EVAL_LIMIT:-none}" +SEED="${EVAL_SEED:-42}" +THINKING="${EVAL_THINKING:-false}" +DATASET_DIR="${EVAL_DATASET_DIR:-$ROOT_DIR}" +OUTPUT_DIR="${EVAL_OUTPUT_DIR:-$ROOT_DIR/output}" + +# -------------------------------------------------- +# 解析命令行参数 +# -------------------------------------------------- +while [[ $# -gt 0 ]]; do + case "$1" in + --api-key) API_KEY="$2"; shift 2 ;; + --api-url) API_URL="$2"; shift 2 ;; + --model) MODEL="$2"; shift 2 ;; + --datasets) DATASETS="$2"; shift 2 ;; + --folder-name) FOLDER_NAME="$2"; shift 2 ;; + --config) CONFIG="$2"; shift 2 ;; + --batch-size) BATCH_SIZE="$2"; shift 2 ;; + --limit) LIMIT="$2"; shift 2 ;; + --seed) SEED="$2"; shift 2 ;; + --thinking) THINKING="true"; shift ;; + --no-thinking) THINKING="false"; shift ;; + --dataset-dir) DATASET_DIR="$2"; shift 2 ;; + --output-dir) OUTPUT_DIR="$2"; shift 2 ;; + -h|--help) + grep '^# ' "$0" | sed 's/^# //' + exit 0 + ;; + *) echo "未知参数: $1"; exit 1 ;; + esac +done + +if [[ -z "$API_KEY" ]]; then + echo "ERROR: 请设置 EVAL_API_KEY 环境变量或传入 --api-key" + exit 1 +fi + +export EVALSCOPE_API_KEY="$API_KEY" +export OPENAI_API_KEY="$API_KEY" + +# -------------------------------------------------- +# API key 连通性校验 +# -------------------------------------------------- +HEALTH=$(curl -s -o /dev/null -w "%{http_code}" \ + -H "Authorization: Bearer ${API_KEY}" \ + "${API_URL}/models") +if [[ "$HEALTH" != "200" ]]; then + echo "ERROR: API key 校验失败,${API_URL}/models 返回 HTTP $HEALTH" + exit 1 +fi +echo "API key 校验通过 (${API_URL})" + +# -------------------------------------------------- +# 组装 run.py 参数 +# -------------------------------------------------- +ARGS=( + --model "$MODEL" + --api-url "$API_URL" + --dataset-dir "$DATASET_DIR" + --output-dir "$OUTPUT_DIR" + --folder-name "$FOLDER_NAME" + --config "$CONFIG" + --batch-size "$BATCH_SIZE" + --seed "$SEED" + --limit "$LIMIT" + --datasets "$DATASETS" +) + +if [[ "$THINKING" == "true" ]]; then + ARGS+=(--thinking) +fi + +echo "============================================================" +echo "API 评测启动" +echo "Model: $MODEL" +echo "API URL: $API_URL" +echo "Datasets: $DATASETS" +echo "Folder: $FOLDER_NAME" +echo "Config: $CONFIG" +echo "Batch size: $BATCH_SIZE" +echo "Limit: $LIMIT" +echo "Thinking: $THINKING" +echo "============================================================" + +python bash/run.py "${ARGS[@]}" diff --git a/bash/case/calc_glm52_cost.sh b/bash/case/calc_glm52_cost.sh new file mode 100755 index 0000000..b205a50 --- /dev/null +++ b/bash/case/calc_glm52_cost.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# ============================================================ +# GLM5.2 API 成本计算脚本 +# 用法: bash bash/case/calc_glm52_cost.sh [结果CSV/Excel] +# 默认读取最新的 DS4-Flash-INT8 NO-Thinking 2.0 FULL 结果 +# ============================================================ + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +cd "$ROOT_DIR" + +INPUT="${1:-$ROOT_DIR/P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv}" +OUTPUT="${2:-$ROOT_DIR/results/P800_benchmark_cost_GLM52.csv}" + +# GLM5.2 报价(示例):输入 8 元/百万 tokens,输出 28 元/百万 tokens,折扣 0.65 +INPUT_PRICE=8 +OUTPUT_PRICE=28 +DISCOUNT=0.65 +MODEL_NAME="GLM-5.2" + +python3 tools/calculate_cost.py \ + --input "$INPUT" \ + --input-price "$INPUT_PRICE" \ + --output-price "$OUTPUT_PRICE" \ + --discount "$DISCOUNT" \ + --model-name "$MODEL_NAME" \ + --output "$OUTPUT" diff --git a/tools/calculate_cost.py b/tools/calculate_cost.py new file mode 100644 index 0000000..e34ebcc --- /dev/null +++ b/tools/calculate_cost.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +""" +根据 benchmark 结果计算 API 调用成本。 + +支持输入: + - CSV(如 P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv) + - Excel(如 P800模型能力评测结果_统一格式_filled.xlsx) + +计价公式: + input_cost = n_samples * input_tokens_mean / 1_000_000 * input_price * discount + output_cost = n_samples * output_tokens_mean / 1_000_000 * output_price * discount + +价格单位:元 / 百万 tokens(按国内 API 常见报价)。 +""" + +import argparse +import sys +from pathlib import Path + +import pandas as pd + + +def load_input(path: str, sheet: str = None) -> pd.DataFrame: + p = Path(path) + if not p.exists(): + raise FileNotFoundError(path) + + if p.suffix.lower() in ('.xlsx', '.xls'): + if sheet: + return pd.read_excel(path, sheet_name=sheet) + xl = pd.ExcelFile(path) + print(f"可用 sheet: {xl.sheet_names}") + return pd.read_excel(path, sheet_name=xl.sheet_names[0]) + else: + return pd.read_csv(path) + + +def compute_cost( + df: pd.DataFrame, + input_price: float, + output_price: float, + discount: float, + model_name: str, +) -> pd.DataFrame: + df = df.copy() + + # 兼容中英文列名 + col_n_samples = next((c for c in df.columns if '样本' in c or c.lower() == 'n_samples'), None) + col_input = next((c for c in df.columns if '输入tokens' in c or c.lower() == 'input_tokens_mean'), None) + col_output = next((c for c in df.columns if '输出tokens' in c or c.lower() == 'output_tokens_mean'), None) + col_total = next((c for c in df.columns if '累计总tokens' in c or c.lower() == 'total_tokens'), None) + col_score = next((c for c in df.columns if '得分' in c or c.lower() == 'score'), None) + col_bench = next((c for c in df.columns if 'benchmark' in c.lower()), None) + + if col_n_samples is None or col_input is None or col_output is None: + raise ValueError("输入文件缺少必要列:总样本数 / 输入tokens_mean / 输出tokens_mean") + + price_in = input_price * discount / 1_000_000.0 + price_out = output_price * discount / 1_000_000.0 + + # 数值化,空值填 0 + n = pd.to_numeric(df[col_n_samples], errors='coerce').fillna(0) + in_tok = pd.to_numeric(df[col_input], errors='coerce').fillna(0) + out_tok = pd.to_numeric(df[col_output], errors='coerce').fillna(0) + + df['input_tokens_total'] = (n * in_tok).astype('int64') + df['output_tokens_total'] = (n * out_tok).astype('int64') + df['input_cost_yuan'] = n * in_tok * price_in + df['output_cost_yuan'] = n * out_tok * price_out + df['total_cost_yuan'] = df['input_cost_yuan'] + df['output_cost_yuan'] + + # 元/千 tokens 便于查看 + df['input_price_per_1k_tokens'] = price_in * 1000 + df['output_price_per_1k_tokens'] = price_out * 1000 + df['model_pricing'] = f"{model_name} (in={input_price}*{-discount}, out={output_price}*{discount})" + + return df + + +def save_output(df: pd.DataFrame, path: str): + p = Path(path) + p.parent.mkdir(parents=True, exist_ok=True) + if p.suffix.lower() in ('.xlsx', '.xls'): + with pd.ExcelWriter(path, engine='openpyxl') as writer: + df.to_excel(writer, sheet_name='cost', index=False) + else: + df.to_csv(path, index=False, encoding='utf-8-sig') + print(f"结果已保存: {path}") + + +def main(): + parser = argparse.ArgumentParser(description="根据 benchmark 结果计算 API 成本") + parser.add_argument('--input', '-i', required=True, help='输入 CSV/Excel 路径') + parser.add_argument('--sheet', '-s', default=None, help='Excel sheet 名(默认第一个)') + parser.add_argument('--input-price', type=float, default=8.0, help='输入单价(元/百万 tokens)') + parser.add_argument('--output-price', type=float, default=28.0, help='输出单价(元/百万 tokens)') + parser.add_argument('--discount', '-d', type=float, default=0.65, help='折扣率,如 0.65') + parser.add_argument('--model-name', '-m', default='GLM-5.2', help='模型名称(仅用于标注)') + parser.add_argument('--output', '-o', required=True, help='输出 CSV/Excel 路径') + args = parser.parse_args() + + df = load_input(args.input, args.sheet) + df_cost = compute_cost( + df, + input_price=args.input_price, + output_price=args.output_price, + discount=args.discount, + model_name=args.model_name, + ) + + # 汇总行(如果原表没有 totals,追加一个) + numeric_cols = ['input_cost_yuan', 'output_cost_yuan', 'total_cost_yuan'] + total_row = {c: df_cost[c].sum() for c in numeric_cols} + total_row['Benchmark'] = 'TOTAL' + total_row['model_pricing'] = args.model_name + + # 插入或更新 TOTAL 行 + is_total = df_cost['Benchmark'].astype(str).str.lower().isin(['total', '总计', '合计']) + if is_total.any(): + for c in numeric_cols: + df_cost.loc[is_total, c] = total_row[c] + else: + total_df = pd.DataFrame([total_row]) + df_cost = pd.concat([df_cost, total_df], ignore_index=True) + + save_output(df_cost, args.output) + + # 终端摘要 + summary = df_cost[['Benchmark', '总样本数', '输入tokens_mean', '输出tokens_mean', + 'input_cost_yuan', 'output_cost_yuan', 'total_cost_yuan']].copy() + print('\n=== 成本摘要(元)===') + print(summary.to_string(index=False)) + print(f"\n总成本: {total_row['total_cost_yuan']:.4f} 元") + + +if __name__ == '__main__': + main() diff --git a/tools/fill_excel_cost.py b/tools/fill_excel_cost.py new file mode 100644 index 0000000..59d5eff --- /dev/null +++ b/tools/fill_excel_cost.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +为 P800模型能力评测结果 Excel 的每个 sheet 追加 API 成本列。 +只处理包含 Benchmark + 总样本数 + 输入/输出 tokens_mean 的 sheet。 +""" + +import argparse +from pathlib import Path + +import pandas as pd + + +def try_compute_cost(df: pd.DataFrame, input_price: float, output_price: float, + discount: float, model_name: str): + col_n = next((c for c in df.columns if '样本' in c and '数' in c), None) + col_in = next((c for c in df.columns if '输入tokens' in c), None) + col_out = next((c for c in df.columns if '输出tokens' in c), None) + col_bench = next((c for c in df.columns if 'benchmark' in c.lower()), None) + + if col_n is None or col_in is None or col_out is None or col_bench is None: + return None + + price_in = input_price * discount / 1_000_000.0 + price_out = output_price * discount / 1_000_000.0 + + n = pd.to_numeric(df[col_n], errors='coerce').fillna(0) + in_tok = pd.to_numeric(df[col_in], errors='coerce').fillna(0) + out_tok = pd.to_numeric(df[col_out], errors='coerce').fillna(0) + + result = df.copy() + result['input_cost_元'] = n * in_tok * price_in + result['output_cost_元'] = n * out_tok * price_out + result['total_cost_元'] = result['input_cost_元'] + result['output_cost_元'] + result['price_model'] = model_name + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--input', '-i', required=True) + parser.add_argument('--output', '-o', required=True) + parser.add_argument('--input-price', type=float, default=8.0) + parser.add_argument('--output-price', type=float, default=28.0) + parser.add_argument('--discount', '-d', type=float, default=0.65) + parser.add_argument('--model-name', '-m', default='GLM-5.2') + args = parser.parse_args() + + xls = pd.ExcelFile(args.input) + out_path = Path(args.output) + out_path.parent.mkdir(parents=True, exist_ok=True) + + with pd.ExcelWriter(out_path, engine='openpyxl') as writer: + for sheet in xls.sheet_names: + df = pd.read_excel(args.input, sheet_name=sheet) + new_df = try_compute_cost( + df, args.input_price, args.output_price, + args.discount, args.model_name, + ) + if new_df is not None: + # 汇总 + total_mask = new_df.iloc[:, 0].astype(str).str.lower().isin(['total', '总计', '合计']) + if not total_mask.any(): + total_row = {'price_model': args.model_name} + for c in new_df.columns: + if 'cost_元' in c: + total_row[c] = new_df[c].sum() + total_df = pd.DataFrame([total_row]) + new_df = pd.concat([new_df, total_df], ignore_index=True) + sheet_name = f"{sheet}_cost" if len(sheet) <= 25 else sheet[:25] + new_df.to_excel(writer, sheet_name=sheet_name, index=False) + else: + # 原样复制 + df.to_excel(writer, sheet_name=sheet, index=False) + + print(f"已保存: {out_path}") + + +if __name__ == '__main__': + main()