feat: add benchmark cost calculator and generic API test runner
This commit is contained in:
sora
parent
b1eb4e116a
commit
11d921071b
24
bash/case/api_test_config.env.example
Normal file
24
bash/case/api_test_config.env.example
Normal file
@ -0,0 +1,24 @@
|
||||
# 复制为 api_test_config.env 后填入真实 key，再 source api_test_config.env
|
||||
|
||||
# API 认证
|
||||
export EVAL_API_KEY="sk-xxxx"
|
||||
export EVAL_API_URL="https://api.vectron.meta-stone.com/v1"
|
||||
|
||||
# 模型名（以 API 服务商的 model id 为准）
|
||||
export EVAL_MODEL="DeepSeek/DeepSeek-V4-Flash"
|
||||
|
||||
# 评测数据集，逗号分隔，想测什么改这里
|
||||
# 可用参考：gsm8k,aime24,aime25,aime26,hmmt26,imo_answerbench,competition_math,bbh,drop
|
||||
# gpqa_diamond,mmlu_pro,simple_qa,mmlu,cmmlu,arc,hellaswag,trivia_qa,winogrande
|
||||
# longbench_v2,openai_mrcr,general_fc,bfcl_v3,bigcodebench,humaneval,live_code_bench
|
||||
export EVAL_DATASETS="gsm8k,aime24,arc"
|
||||
|
||||
# 输出目录标识
|
||||
export EVAL_FOLDER_NAME="API-Test"
|
||||
|
||||
# 评测配置
|
||||
export EVAL_CONFIG="config/dpv4-int8_nothinking.yaml"
|
||||
export EVAL_BATCH_SIZE=4
|
||||
export EVAL_LIMIT="none"
|
||||
export EVAL_SEED=42
|
||||
export EVAL_THINKING="false"
|
||||
121
bash/case/api_test_runner.sh
Executable file
121
bash/case/api_test_runner.sh
Executable file
@ -0,0 +1,121 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# 通用 API 评测脚本
|
||||
# 用法:
|
||||
# # 方式 1：通过环境变量配置
|
||||
# export EVAL_API_KEY="sk-xxxx"
|
||||
# export EVAL_API_URL="https://api.example.com/v1"
|
||||
# export EVAL_MODEL="gpt-4o"
|
||||
# export EVAL_DATASETS="gsm8k,aime24,arc"
|
||||
# bash bash/case/api_test_runner.sh
|
||||
#
|
||||
# # 方式 2：命令行参数覆盖
|
||||
# bash bash/case/api_test_runner.sh \
|
||||
# --api-key sk-xxxx \
|
||||
# --api-url https://api.example.com/v1 \
|
||||
# --model gpt-4o \
|
||||
# --datasets gsm8k,aime24,arc
|
||||
#
|
||||
# 修改 datasets 只需改 EVAL_DATASETS 或 --datasets。
|
||||
# ============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 默认值（可通过环境变量或命令行覆盖）
|
||||
# --------------------------------------------------
|
||||
API_KEY="${EVAL_API_KEY:-}"
|
||||
API_URL="${EVAL_API_URL:-https://api.vectron.meta-stone.com/v1}"
|
||||
MODEL="${EVAL_MODEL:-DeepSeek/DeepSeek-V4-Flash}"
|
||||
DATASETS="${EVAL_DATASETS:-gsm8k,aime24,arc}"
|
||||
FOLDER_NAME="${EVAL_FOLDER_NAME:-API-Test}"
|
||||
CONFIG="${EVAL_CONFIG:-config/dpv4-int8_nothinking.yaml}"
|
||||
BATCH_SIZE="${EVAL_BATCH_SIZE:-4}"
|
||||
LIMIT="${EVAL_LIMIT:-none}"
|
||||
SEED="${EVAL_SEED:-42}"
|
||||
THINKING="${EVAL_THINKING:-false}"
|
||||
DATASET_DIR="${EVAL_DATASET_DIR:-$ROOT_DIR}"
|
||||
OUTPUT_DIR="${EVAL_OUTPUT_DIR:-$ROOT_DIR/output}"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 解析命令行参数
|
||||
# --------------------------------------------------
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--api-key) API_KEY="$2"; shift 2 ;;
|
||||
--api-url) API_URL="$2"; shift 2 ;;
|
||||
--model) MODEL="$2"; shift 2 ;;
|
||||
--datasets) DATASETS="$2"; shift 2 ;;
|
||||
--folder-name) FOLDER_NAME="$2"; shift 2 ;;
|
||||
--config) CONFIG="$2"; shift 2 ;;
|
||||
--batch-size) BATCH_SIZE="$2"; shift 2 ;;
|
||||
--limit) LIMIT="$2"; shift 2 ;;
|
||||
--seed) SEED="$2"; shift 2 ;;
|
||||
--thinking) THINKING="true"; shift ;;
|
||||
--no-thinking) THINKING="false"; shift ;;
|
||||
--dataset-dir) DATASET_DIR="$2"; shift 2 ;;
|
||||
--output-dir) OUTPUT_DIR="$2"; shift 2 ;;
|
||||
-h|--help)
|
||||
grep '^# ' "$0" | sed 's/^# //'
|
||||
exit 0
|
||||
;;
|
||||
*) echo "未知参数: $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ -z "$API_KEY" ]]; then
|
||||
echo "ERROR: 请设置 EVAL_API_KEY 环境变量或传入 --api-key"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
export EVALSCOPE_API_KEY="$API_KEY"
|
||||
export OPENAI_API_KEY="$API_KEY"
|
||||
|
||||
# --------------------------------------------------
|
||||
# API key 连通性校验
|
||||
# --------------------------------------------------
|
||||
HEALTH=$(curl -s -o /dev/null -w "%{http_code}" \
|
||||
-H "Authorization: Bearer ${API_KEY}" \
|
||||
"${API_URL}/models")
|
||||
if [[ "$HEALTH" != "200" ]]; then
|
||||
echo "ERROR: API key 校验失败，${API_URL}/models 返回 HTTP $HEALTH"
|
||||
exit 1
|
||||
fi
|
||||
echo "API key 校验通过 (${API_URL})"
|
||||
|
||||
# --------------------------------------------------
|
||||
# 组装 run.py 参数
|
||||
# --------------------------------------------------
|
||||
ARGS=(
|
||||
--model "$MODEL"
|
||||
--api-url "$API_URL"
|
||||
--dataset-dir "$DATASET_DIR"
|
||||
--output-dir "$OUTPUT_DIR"
|
||||
--folder-name "$FOLDER_NAME"
|
||||
--config "$CONFIG"
|
||||
--batch-size "$BATCH_SIZE"
|
||||
--seed "$SEED"
|
||||
--limit "$LIMIT"
|
||||
--datasets "$DATASETS"
|
||||
)
|
||||
|
||||
if [[ "$THINKING" == "true" ]]; then
|
||||
ARGS+=(--thinking)
|
||||
fi
|
||||
|
||||
echo "============================================================"
|
||||
echo "API 评测启动"
|
||||
echo "Model: $MODEL"
|
||||
echo "API URL: $API_URL"
|
||||
echo "Datasets: $DATASETS"
|
||||
echo "Folder: $FOLDER_NAME"
|
||||
echo "Config: $CONFIG"
|
||||
echo "Batch size: $BATCH_SIZE"
|
||||
echo "Limit: $LIMIT"
|
||||
echo "Thinking: $THINKING"
|
||||
echo "============================================================"
|
||||
|
||||
python bash/run.py "${ARGS[@]}"
|
||||
28
bash/case/calc_glm52_cost.sh
Executable file
28
bash/case/calc_glm52_cost.sh
Executable file
@ -0,0 +1,28 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# GLM5.2 API 成本计算脚本
|
||||
# 用法: bash bash/case/calc_glm52_cost.sh [结果CSV/Excel]
|
||||
# 默认读取最新的 DS4-Flash-INT8 NO-Thinking 2.0 FULL 结果
|
||||
# ============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
cd "$ROOT_DIR"
|
||||
|
||||
INPUT="${1:-$ROOT_DIR/P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv}"
|
||||
OUTPUT="${2:-$ROOT_DIR/results/P800_benchmark_cost_GLM52.csv}"
|
||||
|
||||
# GLM5.2 报价（示例）：输入 8 元/百万 tokens，输出 28 元/百万 tokens，折扣 0.65
|
||||
INPUT_PRICE=8
|
||||
OUTPUT_PRICE=28
|
||||
DISCOUNT=0.65
|
||||
MODEL_NAME="GLM-5.2"
|
||||
|
||||
python3 tools/calculate_cost.py \
|
||||
--input "$INPUT" \
|
||||
--input-price "$INPUT_PRICE" \
|
||||
--output-price "$OUTPUT_PRICE" \
|
||||
--discount "$DISCOUNT" \
|
||||
--model-name "$MODEL_NAME" \
|
||||
--output "$OUTPUT"
|
||||
137
tools/calculate_cost.py
Normal file
137
tools/calculate_cost.py
Normal file
@ -0,0 +1,137 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
根据 benchmark 结果计算 API 调用成本。
|
||||
|
||||
支持输入：
|
||||
- CSV（如 P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv）
|
||||
- Excel（如 P800模型能力评测结果_统一格式_filled.xlsx）
|
||||
|
||||
计价公式：
|
||||
input_cost = n_samples * input_tokens_mean / 1_000_000 * input_price * discount
|
||||
output_cost = n_samples * output_tokens_mean / 1_000_000 * output_price * discount
|
||||
|
||||
价格单位：元 / 百万 tokens（按国内 API 常见报价）。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def load_input(path: str, sheet: str = None) -> pd.DataFrame:
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
raise FileNotFoundError(path)
|
||||
|
||||
if p.suffix.lower() in ('.xlsx', '.xls'):
|
||||
if sheet:
|
||||
return pd.read_excel(path, sheet_name=sheet)
|
||||
xl = pd.ExcelFile(path)
|
||||
print(f"可用 sheet: {xl.sheet_names}")
|
||||
return pd.read_excel(path, sheet_name=xl.sheet_names[0])
|
||||
else:
|
||||
return pd.read_csv(path)
|
||||
|
||||
|
||||
def compute_cost(
|
||||
df: pd.DataFrame,
|
||||
input_price: float,
|
||||
output_price: float,
|
||||
discount: float,
|
||||
model_name: str,
|
||||
) -> pd.DataFrame:
|
||||
df = df.copy()
|
||||
|
||||
# 兼容中英文列名
|
||||
col_n_samples = next((c for c in df.columns if '样本' in c or c.lower() == 'n_samples'), None)
|
||||
col_input = next((c for c in df.columns if '输入tokens' in c or c.lower() == 'input_tokens_mean'), None)
|
||||
col_output = next((c for c in df.columns if '输出tokens' in c or c.lower() == 'output_tokens_mean'), None)
|
||||
col_total = next((c for c in df.columns if '累计总tokens' in c or c.lower() == 'total_tokens'), None)
|
||||
col_score = next((c for c in df.columns if '得分' in c or c.lower() == 'score'), None)
|
||||
col_bench = next((c for c in df.columns if 'benchmark' in c.lower()), None)
|
||||
|
||||
if col_n_samples is None or col_input is None or col_output is None:
|
||||
raise ValueError("输入文件缺少必要列：总样本数 / 输入tokens_mean / 输出tokens_mean")
|
||||
|
||||
price_in = input_price * discount / 1_000_000.0
|
||||
price_out = output_price * discount / 1_000_000.0
|
||||
|
||||
# 数值化，空值填 0
|
||||
n = pd.to_numeric(df[col_n_samples], errors='coerce').fillna(0)
|
||||
in_tok = pd.to_numeric(df[col_input], errors='coerce').fillna(0)
|
||||
out_tok = pd.to_numeric(df[col_output], errors='coerce').fillna(0)
|
||||
|
||||
df['input_tokens_total'] = (n * in_tok).astype('int64')
|
||||
df['output_tokens_total'] = (n * out_tok).astype('int64')
|
||||
df['input_cost_yuan'] = n * in_tok * price_in
|
||||
df['output_cost_yuan'] = n * out_tok * price_out
|
||||
df['total_cost_yuan'] = df['input_cost_yuan'] + df['output_cost_yuan']
|
||||
|
||||
# 元/千 tokens 便于查看
|
||||
df['input_price_per_1k_tokens'] = price_in * 1000
|
||||
df['output_price_per_1k_tokens'] = price_out * 1000
|
||||
df['model_pricing'] = f"{model_name} (in={input_price}*{-discount}, out={output_price}*{discount})"
|
||||
|
||||
return df
|
||||
|
||||
|
||||
def save_output(df: pd.DataFrame, path: str):
|
||||
p = Path(path)
|
||||
p.parent.mkdir(parents=True, exist_ok=True)
|
||||
if p.suffix.lower() in ('.xlsx', '.xls'):
|
||||
with pd.ExcelWriter(path, engine='openpyxl') as writer:
|
||||
df.to_excel(writer, sheet_name='cost', index=False)
|
||||
else:
|
||||
df.to_csv(path, index=False, encoding='utf-8-sig')
|
||||
print(f"结果已保存: {path}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="根据 benchmark 结果计算 API 成本")
|
||||
parser.add_argument('--input', '-i', required=True, help='输入 CSV/Excel 路径')
|
||||
parser.add_argument('--sheet', '-s', default=None, help='Excel sheet 名（默认第一个）')
|
||||
parser.add_argument('--input-price', type=float, default=8.0, help='输入单价（元/百万 tokens）')
|
||||
parser.add_argument('--output-price', type=float, default=28.0, help='输出单价（元/百万 tokens）')
|
||||
parser.add_argument('--discount', '-d', type=float, default=0.65, help='折扣率，如 0.65')
|
||||
parser.add_argument('--model-name', '-m', default='GLM-5.2', help='模型名称（仅用于标注）')
|
||||
parser.add_argument('--output', '-o', required=True, help='输出 CSV/Excel 路径')
|
||||
args = parser.parse_args()
|
||||
|
||||
df = load_input(args.input, args.sheet)
|
||||
df_cost = compute_cost(
|
||||
df,
|
||||
input_price=args.input_price,
|
||||
output_price=args.output_price,
|
||||
discount=args.discount,
|
||||
model_name=args.model_name,
|
||||
)
|
||||
|
||||
# 汇总行（如果原表没有 totals，追加一个）
|
||||
numeric_cols = ['input_cost_yuan', 'output_cost_yuan', 'total_cost_yuan']
|
||||
total_row = {c: df_cost[c].sum() for c in numeric_cols}
|
||||
total_row['Benchmark'] = 'TOTAL'
|
||||
total_row['model_pricing'] = args.model_name
|
||||
|
||||
# 插入或更新 TOTAL 行
|
||||
is_total = df_cost['Benchmark'].astype(str).str.lower().isin(['total', '总计', '合计'])
|
||||
if is_total.any():
|
||||
for c in numeric_cols:
|
||||
df_cost.loc[is_total, c] = total_row[c]
|
||||
else:
|
||||
total_df = pd.DataFrame([total_row])
|
||||
df_cost = pd.concat([df_cost, total_df], ignore_index=True)
|
||||
|
||||
save_output(df_cost, args.output)
|
||||
|
||||
# 终端摘要
|
||||
summary = df_cost[['Benchmark', '总样本数', '输入tokens_mean', '输出tokens_mean',
|
||||
'input_cost_yuan', 'output_cost_yuan', 'total_cost_yuan']].copy()
|
||||
print('\n=== 成本摘要（元）===')
|
||||
print(summary.to_string(index=False))
|
||||
print(f"\n总成本: {total_row['total_cost_yuan']:.4f} 元")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
79
tools/fill_excel_cost.py
Normal file
79
tools/fill_excel_cost.py
Normal file
@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
为 P800模型能力评测结果 Excel 的每个 sheet 追加 API 成本列。
|
||||
只处理包含 Benchmark + 总样本数 + 输入/输出 tokens_mean 的 sheet。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import pandas as pd
|
||||
|
||||
|
||||
def try_compute_cost(df: pd.DataFrame, input_price: float, output_price: float,
|
||||
discount: float, model_name: str):
|
||||
col_n = next((c for c in df.columns if '样本' in c and '数' in c), None)
|
||||
col_in = next((c for c in df.columns if '输入tokens' in c), None)
|
||||
col_out = next((c for c in df.columns if '输出tokens' in c), None)
|
||||
col_bench = next((c for c in df.columns if 'benchmark' in c.lower()), None)
|
||||
|
||||
if col_n is None or col_in is None or col_out is None or col_bench is None:
|
||||
return None
|
||||
|
||||
price_in = input_price * discount / 1_000_000.0
|
||||
price_out = output_price * discount / 1_000_000.0
|
||||
|
||||
n = pd.to_numeric(df[col_n], errors='coerce').fillna(0)
|
||||
in_tok = pd.to_numeric(df[col_in], errors='coerce').fillna(0)
|
||||
out_tok = pd.to_numeric(df[col_out], errors='coerce').fillna(0)
|
||||
|
||||
result = df.copy()
|
||||
result['input_cost_元'] = n * in_tok * price_in
|
||||
result['output_cost_元'] = n * out_tok * price_out
|
||||
result['total_cost_元'] = result['input_cost_元'] + result['output_cost_元']
|
||||
result['price_model'] = model_name
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--input', '-i', required=True)
|
||||
parser.add_argument('--output', '-o', required=True)
|
||||
parser.add_argument('--input-price', type=float, default=8.0)
|
||||
parser.add_argument('--output-price', type=float, default=28.0)
|
||||
parser.add_argument('--discount', '-d', type=float, default=0.65)
|
||||
parser.add_argument('--model-name', '-m', default='GLM-5.2')
|
||||
args = parser.parse_args()
|
||||
|
||||
xls = pd.ExcelFile(args.input)
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with pd.ExcelWriter(out_path, engine='openpyxl') as writer:
|
||||
for sheet in xls.sheet_names:
|
||||
df = pd.read_excel(args.input, sheet_name=sheet)
|
||||
new_df = try_compute_cost(
|
||||
df, args.input_price, args.output_price,
|
||||
args.discount, args.model_name,
|
||||
)
|
||||
if new_df is not None:
|
||||
# 汇总
|
||||
total_mask = new_df.iloc[:, 0].astype(str).str.lower().isin(['total', '总计', '合计'])
|
||||
if not total_mask.any():
|
||||
total_row = {'price_model': args.model_name}
|
||||
for c in new_df.columns:
|
||||
if 'cost_元' in c:
|
||||
total_row[c] = new_df[c].sum()
|
||||
total_df = pd.DataFrame([total_row])
|
||||
new_df = pd.concat([new_df, total_df], ignore_index=True)
|
||||
sheet_name = f"{sheet}_cost" if len(sheet) <= 25 else sheet[:25]
|
||||
new_df.to_excel(writer, sheet_name=sheet_name, index=False)
|
||||
else:
|
||||
# 原样复制
|
||||
df.to_excel(writer, sheet_name=sheet, index=False)
|
||||
|
||||
print(f"已保存: {out_path}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user