09b2add673
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1, 2026-09-07 pruning final): probe battery -> concurrent collection -> five scoring views (verify/attribution/variant/adversarial/robustness), bundled family aliases + 27 reference fingerprints (12 fp_fusion schema). - CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single source of arg definitions) + 'fingerprint list' for bundled references - imports rewritten package-relative; direct 'python3 run_fp_fusion.py' execution kept working via package bootstrap - offline analysis/collection scripts made path-independent (previously pinned to a /opt/evalscope path absent on this host) - shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars - --reference accepts short names resolved against bundled references/ - pyproject: +httpx dependency, package-data references/*.json - tests/test_fingerprint.py: 10 offline tests (battery definitions, assembly counts, normalization, signals, verdict ladder, CLI wiring) - README: fingerprint section + architecture entry Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
32 lines
1.7 KiB
Bash
32 lines
1.7 KiB
Bash
#!/bin/bash
|
||
# 任务2a 主循环：6 模型 × verify，串行执行
|
||
# 分批：batch1 = DS-Flash, DS-Pro, GLM-5.3（先出代表结果）
|
||
# 需在 evalharness 环境下运行。凭据走环境变量，不再硬编码：
|
||
# export FP_API_KEY=sk-xxxx # 或复用 OPENAI_API_KEY
|
||
cd "$(dirname "$0")"
|
||
|
||
run_one() {
|
||
local mode=$1 model=$2 ref=$3 name=$4
|
||
local key="${FP_API_KEY:-${OPENAI_API_KEY:-}}"
|
||
local api="${FP_API_URL:-https://api.vectron.meta-stone.com/v1}"
|
||
local eb="${FP_EXTRA_BODY:-{\"chat_template_kwargs\":{\"enable_thinking\":false}}}"
|
||
if [ -z "$key" ]; then echo "ERROR: 未设置 FP_API_KEY/OPENAI_API_KEY，跳过 $name" >&2; return 1; fi
|
||
mkdir -p /tmp/bfd/$name
|
||
echo "=== START $name $mode $(date +%H:%M) ==="
|
||
python3 run_fp_fusion.py --api-url "$api" --model "$model" \
|
||
--mode "$mode" --d-samples 25 --baseline-samples 5 --timeout 90 \
|
||
--d-concurrency 1 --text-concurrency 1 \
|
||
--api-key "$key" --extra-body "$eb" --reference "$ref" \
|
||
--report-path /tmp/bfd/$name/${mode}.json > /tmp/bfd/$name/${mode}.log 2>&1
|
||
echo "=== END $name $mode exit=$? $(date +%H:%M) ==="
|
||
}
|
||
|
||
R="$(pwd)/references"
|
||
|
||
run_one verify DeepSeek/DeepSeek-V4-Flash $R/deepseek_v4_flash_fusion_reference.json deepseek_v4_flash
|
||
run_one verify DeepSeek/DeepSeek-V4-Pro $R/deepseek_v4_pro_fusion_reference.json deepseek_v4_pro
|
||
run_one verify ZhipuAi/GLM-5.3 $R/glm53_fusion_reference.json glm_53
|
||
run_one verify DeepSeek/DeepSeek-V4-Flash-0731 $R/deepseek_v4_flash_0731_fusion_reference.json deepseek_v4_flash_0731
|
||
run_one verify ZhipuAi/GLM-5.2 $R/glm52_vectron_fusion_reference.json glm_52
|
||
run_one verify MoonshotAi/Kimi-K3 $R/kimi_k3_fusion_reference.json kimi_k3
|