09b2add673
New evalharness/fingerprint/ package (from evalstone fp_fusion v1.1, 2026-09-07 pruning final): probe battery -> concurrent collection -> five scoring views (verify/attribution/variant/adversarial/robustness), bundled family aliases + 27 reference fingerprints (12 fp_fusion schema). - CLI: 'evalharness fingerprint run ...' (REMAINDER passthrough, single source of arg definitions) + 'fingerprint list' for bundled references - imports rewritten package-relative; direct 'python3 run_fp_fusion.py' execution kept working via package bootstrap - offline analysis/collection scripts made path-independent (previously pinned to a /opt/evalscope path absent on this host) - shell scripts: hardcoded API key -> FP_API_KEY/OPENAI_API_KEY env vars - --reference accepts short names resolved against bundled references/ - pyproject: +httpx dependency, package-data references/*.json - tests/test_fingerprint.py: 10 offline tests (battery definitions, assembly counts, normalization, signals, verdict ladder, CLI wiring) - README: fingerprint section + architecture entry Verified on H20-1: tests 10/10, installed CLI OK, full-protocol run vs vectron GLM-5.3 reproduces baseline (score 0.9451, s_idn 0.846).
39 lines
1.3 KiB
TOML
39 lines
1.3 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=68"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "evalharness"
|
|
version = "0.1.0"
|
|
description = "A plugin-based LLM/agent evaluation harness (data layer first)"
|
|
requires-python = ">=3.10"
|
|
dependencies = [
|
|
"pydantic>=2",
|
|
"datasets", # HuggingFace-hosted datasets (light, conflict-free)
|
|
"pyarrow", # parquet sources (ModelScope/HF raw mirrors)
|
|
"sympy", # official PRM800K symbolic math grading
|
|
"pylatexenc",
|
|
"numpy", # official DROP aligner
|
|
"scipy",
|
|
"rich",
|
|
"xlsxwriter", # excel result workbook
|
|
"httpx", # model fingerprint probing (evalharness/fingerprint)
|
|
"tree_sitter>=0.21", # vendored BFCL official AST checker (python)
|
|
"tree-sitter-java>=0.21", # bfcl java categories
|
|
"tree-sitter-javascript>=0.21", # bfcl javascript categories
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
# none today: the official BFCL checker is vendored under
|
|
# evalharness/third_party/bfcl (Apache-2.0); heavy execution environments
|
|
# (humaneval/bigcodebench/swe) live in docker images, never in the venv
|
|
|
|
[project.scripts]
|
|
evalharness = "evalharness.cli:main"
|
|
|
|
[tool.setuptools.packages.find]
|
|
include = ["evalharness*"]
|
|
|
|
[tool.setuptools.package-data]
|
|
"*" = ["*.jsonl", "*.json", "*.csv", "*.tsv", "*.txt", "*.md", "references/*.json"]
|