27cf8b3c7e
- progress/: Rich per-sample terminal progress plugin (Run Plan panel,
in-flight/rate/ETA bar); shared console + log-through-live to avoid
interleaved writes, rollback() pairs begin_sample on the retry path,
begin moved inside the semaphore (in-flight = actually generating),
graceful degradation when rich is absent
- cli.py: --provider/--api-url/--model composition (openai-chat |
openai-pool), --disable-thinking/--perf/--textools as first-class
flags, per-bench phase lines and done/failed result lines
- __init__: top-level run()/arun() entries (event-loop safe for notebooks)
- third_party/bfcl: vendored official BFCL ast_checker + type mappings
(Apache-2.0, provenance in __init__.py); imports rerouted locally,
underscore_to_dot parameterized; verified bit-identical with the
bfcl-eval package on 100 real rows -- removes the heavy extra
(pinned numpy + cloud SDK wall) from the install path
- runner: progress/status hooks through generate+evaluate, checkpoint
key scheme fix (empty-store falsy bug), tiered retry backoff,
multi-segment pool {range} expansion fix, adapter-instance passthrough
- pyproject: tree_sitter family joins core deps; [bfcl] extra retired
- README: rewritten (zh) -- install/quickstart/flags reference/bench
table/reliability/extension/architecture/validation
Co-Authored-By: Claude <noreply@anthropic.com>
65 lines
1.8 KiB
Python
65 lines
1.8 KiB
Python
"""EvalHarness: a plugin-based LLM/agent evaluation harness (data layer first)."""
|
|
|
|
from .data import (
|
|
ChatMessage,
|
|
Dataset,
|
|
DatasetSpec,
|
|
FieldSpec,
|
|
Sample,
|
|
SandboxSpec,
|
|
ToolInfo,
|
|
get_dataset,
|
|
list_datasets,
|
|
register_dataset,
|
|
)
|
|
|
|
__version__ = '0.1.0'
|
|
|
|
|
|
async def arun(bench, model, **kwargs):
|
|
"""ASYNC entry: for callers already inside an event loop.
|
|
|
|
import evalharness
|
|
rep = await evalharness.arun('gsm8k', 'openai/http://...?m', limit=200)
|
|
"""
|
|
return await _run_dispatch(bench, model, **kwargs)
|
|
|
|
|
|
def run(bench, model, **kwargs):
|
|
"""SYNC entry (primary API, mirroring inspect_ai.run / lm_eval).
|
|
|
|
import evalharness
|
|
rep = evalharness.run('gsm8k', 'openai/http://...?m', limit=200)
|
|
|
|
Creates its own event loop; safe to call from scripts/notebooks.
|
|
"""
|
|
import asyncio
|
|
|
|
try: # already inside a loop (notebook)? run in a thread
|
|
asyncio.get_running_loop()
|
|
import concurrent.futures
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool:
|
|
return pool.submit(asyncio.run, _run_dispatch(bench, model, **kwargs)).result()
|
|
except RuntimeError:
|
|
return asyncio.run(_run_dispatch(bench, model, **kwargs))
|
|
|
|
|
|
async def _run_dispatch(bench, model, **kwargs):
|
|
from .data import Dataset as _Dataset, get_dataset as _gd
|
|
from .model import run_eval as _run_eval
|
|
|
|
if isinstance(bench, str):
|
|
ds = _gd(bench, subset=kwargs.pop('subset', None))
|
|
elif isinstance(bench, (_Dataset, list)):
|
|
ds = bench
|
|
else:
|
|
raise TypeError(f'bench must be str/Dataset/list, got {type(bench)}')
|
|
return await _run_eval(ds, model, **kwargs)
|
|
|
|
|
|
__all__ = [
|
|
'Dataset', 'DatasetSpec', 'FieldSpec', 'Sample', 'ChatMessage', 'SandboxSpec', 'ToolInfo',
|
|
'get_dataset', 'list_datasets', 'register_dataset', 'run', 'arun',
|
|
]
|