27cf8b3c7e
- progress/: Rich per-sample terminal progress plugin (Run Plan panel,
in-flight/rate/ETA bar); shared console + log-through-live to avoid
interleaved writes, rollback() pairs begin_sample on the retry path,
begin moved inside the semaphore (in-flight = actually generating),
graceful degradation when rich is absent
- cli.py: --provider/--api-url/--model composition (openai-chat |
openai-pool), --disable-thinking/--perf/--textools as first-class
flags, per-bench phase lines and done/failed result lines
- __init__: top-level run()/arun() entries (event-loop safe for notebooks)
- third_party/bfcl: vendored official BFCL ast_checker + type mappings
(Apache-2.0, provenance in __init__.py); imports rerouted locally,
underscore_to_dot parameterized; verified bit-identical with the
bfcl-eval package on 100 real rows -- removes the heavy extra
(pinned numpy + cloud SDK wall) from the install path
- runner: progress/status hooks through generate+evaluate, checkpoint
key scheme fix (empty-store falsy bug), tiered retry backoff,
multi-segment pool {range} expansion fix, adapter-instance passthrough
- pyproject: tree_sitter family joins core deps; [bfcl] extra retired
- README: rewritten (zh) -- install/quickstart/flags reference/bench
table/reliability/extension/architecture/validation
Co-Authored-By: Claude <noreply@anthropic.com>
37 lines
1.2 KiB
TOML
37 lines
1.2 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=68"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "evalharness"
|
|
version = "0.1.0"
|
|
description = "A plugin-based LLM/agent evaluation harness (data layer first)"
|
|
requires-python = ">=3.10"
|
|
dependencies = [
|
|
"pydantic>=2",
|
|
"datasets", # HuggingFace-hosted datasets (light, conflict-free)
|
|
"pyarrow", # parquet sources (ModelScope/HF raw mirrors)
|
|
"sympy", # official PRM800K symbolic math grading
|
|
"pylatexenc",
|
|
"numpy", # official DROP aligner
|
|
"scipy",
|
|
"rich",
|
|
"tree_sitter>=0.21", # vendored BFCL official AST checker (python)
|
|
"tree-sitter-java>=0.21", # bfcl java categories
|
|
"tree-sitter-javascript>=0.21", # bfcl javascript categories
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
# none today: the official BFCL checker is vendored under
|
|
# evalharness/third_party/bfcl (Apache-2.0); heavy execution environments
|
|
# (humaneval/bigcodebench/swe) live in docker images, never in the venv
|
|
|
|
[project.scripts]
|
|
evalharness = "evalharness.cli:main"
|
|
|
|
[tool.setuptools.packages.find]
|
|
include = ["evalharness*"]
|
|
|
|
[tool.setuptools.package-data]
|
|
"*" = ["*.jsonl", "*.json", "*.csv", "*.tsv"]
|