sora/evalstone
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add benchmark cost calculator and generic API test runner

Browse Source
This commit is contained in:
sora 2026-07-31 02:44:14 +00:00
parent b1eb4e116a
commit 11d921071b
5 changed files with 389 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
bash/case/api_test_config.env.example Normal file
View File

@ -0,0 +1,24 @@
# 复制为 api_test_config.env 后填入真实 key，再 source api_test_config.env
# API 认证
export EVAL_API_KEY="sk-xxxx"
export EVAL_API_URL="https://api.vectron.meta-stone.com/v1"
# 模型名（以 API 服务商的 model id 为准）
export EVAL_MODEL="DeepSeek/DeepSeek-V4-Flash"
# 评测数据集，逗号分隔，想测什么改这里
# 可用参考：gsm8k,aime24,aime25,aime26,hmmt26,imo_answerbench,competition_math,bbh,drop
# gpqa_diamond,mmlu_pro,simple_qa,mmlu,cmmlu,arc,hellaswag,trivia_qa,winogrande
# longbench_v2,openai_mrcr,general_fc,bfcl_v3,bigcodebench,humaneval,live_code_bench
export EVAL_DATASETS="gsm8k,aime24,arc"
# 输出目录标识
export EVAL_FOLDER_NAME="API-Test"
# 评测配置
export EVAL_CONFIG="config/dpv4-int8_nothinking.yaml"
export EVAL_BATCH_SIZE=4
export EVAL_LIMIT="none"
export EVAL_SEED=42
export EVAL_THINKING="false"

121
bash/case/api_test_runner.sh Executable file
View File

@ -0,0 +1,121 @@
#!/bin/bash
# ============================================================
# 通用 API 评测脚本
# 用法:
# # 方式 1：通过环境变量配置
# export EVAL_API_KEY="sk-xxxx"
# export EVAL_API_URL="https://api.example.com/v1"
# export EVAL_MODEL="gpt-4o"
# export EVAL_DATASETS="gsm8k,aime24,arc"
# bash bash/case/api_test_runner.sh
#
# # 方式 2：命令行参数覆盖
# bash bash/case/api_test_runner.sh \
# --api-key sk-xxxx \
# --api-url https://api.example.com/v1 \
# --model gpt-4o \
# --datasets gsm8k,aime24,arc
#
# 修改 datasets 只需改 EVAL_DATASETS 或 --datasets。
# ============================================================
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$ROOT_DIR"
# --------------------------------------------------
# 默认值（可通过环境变量或命令行覆盖）
# --------------------------------------------------
API_KEY="${EVAL_API_KEY:-}"
API_URL="${EVAL_API_URL:-https://api.vectron.meta-stone.com/v1}"
MODEL="${EVAL_MODEL:-DeepSeek/DeepSeek-V4-Flash}"
DATASETS="${EVAL_DATASETS:-gsm8k,aime24,arc}"
FOLDER_NAME="${EVAL_FOLDER_NAME:-API-Test}"
CONFIG="${EVAL_CONFIG:-config/dpv4-int8_nothinking.yaml}"
BATCH_SIZE="${EVAL_BATCH_SIZE:-4}"
LIMIT="${EVAL_LIMIT:-none}"
SEED="${EVAL_SEED:-42}"
THINKING="${EVAL_THINKING:-false}"
DATASET_DIR="${EVAL_DATASET_DIR:-$ROOT_DIR}"
OUTPUT_DIR="${EVAL_OUTPUT_DIR:-$ROOT_DIR/output}"
# --------------------------------------------------
# 解析命令行参数
# --------------------------------------------------
while [[ $# -gt 0 ]]; do
case "$1" in
--api-key) API_KEY="$2"; shift 2 ;;
--api-url) API_URL="$2"; shift 2 ;;
--model) MODEL="$2"; shift 2 ;;
--datasets) DATASETS="$2"; shift 2 ;;
--folder-name) FOLDER_NAME="$2"; shift 2 ;;
--config) CONFIG="$2"; shift 2 ;;
--batch-size) BATCH_SIZE="$2"; shift 2 ;;
--limit) LIMIT="$2"; shift 2 ;;
--seed) SEED="$2"; shift 2 ;;
--thinking) THINKING="true"; shift ;;
--no-thinking) THINKING="false"; shift ;;
--dataset-dir) DATASET_DIR="$2"; shift 2 ;;
--output-dir) OUTPUT_DIR="$2"; shift 2 ;;
-h|--help)
grep '^# ' "$0" | sed 's/^# //'
exit 0
;;
*) echo "未知参数: $1"; exit 1 ;;
esac
done
if [[ -z "$API_KEY" ]]; then
echo "ERROR: 请设置 EVAL_API_KEY 环境变量或传入 --api-key"
exit 1
fi
export EVALSCOPE_API_KEY="$API_KEY"
export OPENAI_API_KEY="$API_KEY"
# --------------------------------------------------
# API key 连通性校验
# --------------------------------------------------
HEALTH=$(curl -s -o /dev/null -w "%{http_code}" \
-H "Authorization: Bearer ${API_KEY}" \
"${API_URL}/models")
if [[ "$HEALTH" != "200" ]]; then
echo "ERROR: API key 校验失败，${API_URL}/models 返回 HTTP $HEALTH"
exit 1
fi
echo "API key 校验通过 (${API_URL})"
# --------------------------------------------------
# 组装 run.py 参数
# --------------------------------------------------
ARGS=(
--model "$MODEL"
--api-url "$API_URL"
--dataset-dir "$DATASET_DIR"
--output-dir "$OUTPUT_DIR"
--folder-name "$FOLDER_NAME"
--config "$CONFIG"
--batch-size "$BATCH_SIZE"
--seed "$SEED"
--limit "$LIMIT"
--datasets "$DATASETS"
)
if [[ "$THINKING" == "true" ]]; then
ARGS+=(--thinking)
fi
echo "============================================================"
echo "API 评测启动"
echo "Model: $MODEL"
echo "API URL: $API_URL"
echo "Datasets: $DATASETS"
echo "Folder: $FOLDER_NAME"
echo "Config: $CONFIG"
echo "Batch size: $BATCH_SIZE"
echo "Limit: $LIMIT"
echo "Thinking: $THINKING"
echo "============================================================"
python bash/run.py "${ARGS[@]}"

28
bash/case/calc_glm52_cost.sh Executable file
View File

@ -0,0 +1,28 @@
#!/bin/bash
# ============================================================
# GLM5.2 API 成本计算脚本
# 用法: bash bash/case/calc_glm52_cost.sh [结果CSV/Excel]
# 默认读取最新的 DS4-Flash-INT8 NO-Thinking 2.0 FULL 结果
# ============================================================
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
cd "$ROOT_DIR"
INPUT="${1:-$ROOT_DIR/P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv}"
OUTPUT="${2:-$ROOT_DIR/results/P800_benchmark_cost_GLM52.csv}"
# GLM5.2 报价（示例）：输入 8 元/百万 tokens，输出 28 元/百万 tokens，折扣 0.65
INPUT_PRICE=8
OUTPUT_PRICE=28
DISCOUNT=0.65
MODEL_NAME="GLM-5.2"
python3 tools/calculate_cost.py \
--input "$INPUT" \
--input-price "$INPUT_PRICE" \
--output-price "$OUTPUT_PRICE" \
--discount "$DISCOUNT" \
--model-name "$MODEL_NAME" \
--output "$OUTPUT"

137
tools/calculate_cost.py Normal file
View File

@ -0,0 +1,137 @@
#!/usr/bin/env python3
"""
根据 benchmark 结果计算 API 调用成本。
支持输入：
- CSV（如 P800模型能力评测结果 - DS4-Flash-INT8-NO-Thinking-2.0-FULL.csv）
- Excel（如 P800模型能力评测结果_统一格式_filled.xlsx）
计价公式：
input_cost = n_samples * input_tokens_mean / 1_000_000 * input_price * discount
output_cost = n_samples * output_tokens_mean / 1_000_000 * output_price * discount
价格单位：元 / 百万 tokens（按国内 API 常见报价）。
"""
import argparse
import sys
from pathlib import Path
import pandas as pd
def load_input(path: str, sheet: str = None) -> pd.DataFrame:
p = Path(path)
if not p.exists():
raise FileNotFoundError(path)
if p.suffix.lower() in ('.xlsx', '.xls'):
if sheet:
return pd.read_excel(path, sheet_name=sheet)
xl = pd.ExcelFile(path)
print(f"可用 sheet: {xl.sheet_names}")
return pd.read_excel(path, sheet_name=xl.sheet_names[0])
else:
return pd.read_csv(path)
def compute_cost(
df: pd.DataFrame,
input_price: float,
output_price: float,
discount: float,
model_name: str,
) -> pd.DataFrame:
df = df.copy()
# 兼容中英文列名
col_n_samples = next((c for c in df.columns if '样本' in c or c.lower() == 'n_samples'), None)
col_input = next((c for c in df.columns if '输入tokens' in c or c.lower() == 'input_tokens_mean'), None)
col_output = next((c for c in df.columns if '输出tokens' in c or c.lower() == 'output_tokens_mean'), None)
col_total = next((c for c in df.columns if '累计总tokens' in c or c.lower() == 'total_tokens'), None)
col_score = next((c for c in df.columns if '得分' in c or c.lower() == 'score'), None)
col_bench = next((c for c in df.columns if 'benchmark' in c.lower()), None)
if col_n_samples is None or col_input is None or col_output is None:
raise ValueError("输入文件缺少必要列：总样本数 / 输入tokens_mean / 输出tokens_mean")
price_in = input_price * discount / 1_000_000.0
price_out = output_price * discount / 1_000_000.0
# 数值化，空值填 0
n = pd.to_numeric(df[col_n_samples], errors='coerce').fillna(0)
in_tok = pd.to_numeric(df[col_input], errors='coerce').fillna(0)
out_tok = pd.to_numeric(df[col_output], errors='coerce').fillna(0)
df['input_tokens_total'] = (n * in_tok).astype('int64')
df['output_tokens_total'] = (n * out_tok).astype('int64')
df['input_cost_yuan'] = n * in_tok * price_in
df['output_cost_yuan'] = n * out_tok * price_out
df['total_cost_yuan'] = df['input_cost_yuan'] + df['output_cost_yuan']
# 元/千 tokens 便于查看
df['input_price_per_1k_tokens'] = price_in * 1000
df['output_price_per_1k_tokens'] = price_out * 1000
df['model_pricing'] = f"{model_name} (in={input_price}*{-discount}, out={output_price}*{discount})"
return df
def save_output(df: pd.DataFrame, path: str):
p = Path(path)
p.parent.mkdir(parents=True, exist_ok=True)
if p.suffix.lower() in ('.xlsx', '.xls'):
with pd.ExcelWriter(path, engine='openpyxl') as writer:
df.to_excel(writer, sheet_name='cost', index=False)
else:
df.to_csv(path, index=False, encoding='utf-8-sig')
print(f"结果已保存: {path}")
def main():
parser = argparse.ArgumentParser(description="根据 benchmark 结果计算 API 成本")
parser.add_argument('--input', '-i', required=True, help='输入 CSV/Excel 路径')
parser.add_argument('--sheet', '-s', default=None, help='Excel sheet 名（默认第一个）')
parser.add_argument('--input-price', type=float, default=8.0, help='输入单价（元/百万 tokens）')
parser.add_argument('--output-price', type=float, default=28.0, help='输出单价（元/百万 tokens）')
parser.add_argument('--discount', '-d', type=float, default=0.65, help='折扣率，如 0.65')
parser.add_argument('--model-name', '-m', default='GLM-5.2', help='模型名称（仅用于标注）')
parser.add_argument('--output', '-o', required=True, help='输出 CSV/Excel 路径')
args = parser.parse_args()
df = load_input(args.input, args.sheet)
df_cost = compute_cost(
df,
input_price=args.input_price,
output_price=args.output_price,
discount=args.discount,
model_name=args.model_name,
)
# 汇总行（如果原表没有 totals，追加一个）
numeric_cols = ['input_cost_yuan', 'output_cost_yuan', 'total_cost_yuan']
total_row = {c: df_cost[c].sum() for c in numeric_cols}
total_row['Benchmark'] = 'TOTAL'
total_row['model_pricing'] = args.model_name
# 插入或更新 TOTAL 行
is_total = df_cost['Benchmark'].astype(str).str.lower().isin(['total', '总计', '合计'])
if is_total.any():
for c in numeric_cols:
df_cost.loc[is_total, c] = total_row[c]
else:
total_df = pd.DataFrame([total_row])
df_cost = pd.concat([df_cost, total_df], ignore_index=True)
save_output(df_cost, args.output)
# 终端摘要
summary = df_cost[['Benchmark', '总样本数', '输入tokens_mean', '输出tokens_mean',
'input_cost_yuan', 'output_cost_yuan', 'total_cost_yuan']].copy()
print('\n=== 成本摘要（元）===')
print(summary.to_string(index=False))
print(f"\n总成本: {total_row['total_cost_yuan']:.4f} 元")
if __name__ == '__main__':
main()

79
tools/fill_excel_cost.py Normal file
View File

@ -0,0 +1,79 @@
#!/usr/bin/env python3
"""
为 P800模型能力评测结果 Excel 的每个 sheet 追加 API 成本列。
只处理包含 Benchmark + 总样本数 + 输入/输出 tokens_mean 的 sheet。
"""
import argparse
from pathlib import Path
import pandas as pd
def try_compute_cost(df: pd.DataFrame, input_price: float, output_price: float,
discount: float, model_name: str):
col_n = next((c for c in df.columns if '样本' in c and '数' in c), None)
col_in = next((c for c in df.columns if '输入tokens' in c), None)
col_out = next((c for c in df.columns if '输出tokens' in c), None)
col_bench = next((c for c in df.columns if 'benchmark' in c.lower()), None)
if col_n is None or col_in is None or col_out is None or col_bench is None:
return None
price_in = input_price * discount / 1_000_000.0
price_out = output_price * discount / 1_000_000.0
n = pd.to_numeric(df[col_n], errors='coerce').fillna(0)
in_tok = pd.to_numeric(df[col_in], errors='coerce').fillna(0)
out_tok = pd.to_numeric(df[col_out], errors='coerce').fillna(0)
result = df.copy()
result['input_cost_元'] = n * in_tok * price_in
result['output_cost_元'] = n * out_tok * price_out
result['total_cost_元'] = result['input_cost_元'] + result['output_cost_元']
result['price_model'] = model_name
return result
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--input', '-i', required=True)
parser.add_argument('--output', '-o', required=True)
parser.add_argument('--input-price', type=float, default=8.0)
parser.add_argument('--output-price', type=float, default=28.0)
parser.add_argument('--discount', '-d', type=float, default=0.65)
parser.add_argument('--model-name', '-m', default='GLM-5.2')
args = parser.parse_args()
xls = pd.ExcelFile(args.input)
out_path = Path(args.output)
out_path.parent.mkdir(parents=True, exist_ok=True)
with pd.ExcelWriter(out_path, engine='openpyxl') as writer:
for sheet in xls.sheet_names:
df = pd.read_excel(args.input, sheet_name=sheet)
new_df = try_compute_cost(
df, args.input_price, args.output_price,
args.discount, args.model_name,
)
if new_df is not None:
# 汇总
total_mask = new_df.iloc[:, 0].astype(str).str.lower().isin(['total', '总计', '合计'])
if not total_mask.any():
total_row = {'price_model': args.model_name}
for c in new_df.columns:
if 'cost_元' in c:
total_row[c] = new_df[c].sum()
total_df = pd.DataFrame([total_row])
new_df = pd.concat([new_df, total_df], ignore_index=True)
sheet_name = f"{sheet}_cost" if len(sheet) <= 25 else sheet[:25]
new_df.to_excel(writer, sheet_name=sheet_name, index=False)
else:
# 原样复制
df.to_excel(writer, sheet_name=sheet, index=False)
print(f"已保存: {out_path}")
if __name__ == '__main__':
main()