From 27cf8b3c7e449cd185fe6d9dfa6907f39795f1d8 Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 10 Sep 2026 05:46:45 +0000 Subject: [PATCH] Usability round: progress plugin, CLI provider flags, top-level run(), vendored BFCL checker - progress/: Rich per-sample terminal progress plugin (Run Plan panel, in-flight/rate/ETA bar); shared console + log-through-live to avoid interleaved writes, rollback() pairs begin_sample on the retry path, begin moved inside the semaphore (in-flight = actually generating), graceful degradation when rich is absent - cli.py: --provider/--api-url/--model composition (openai-chat | openai-pool), --disable-thinking/--perf/--textools as first-class flags, per-bench phase lines and done/failed result lines - __init__: top-level run()/arun() entries (event-loop safe for notebooks) - third_party/bfcl: vendored official BFCL ast_checker + type mappings (Apache-2.0, provenance in __init__.py); imports rerouted locally, underscore_to_dot parameterized; verified bit-identical with the bfcl-eval package on 100 real rows -- removes the heavy extra (pinned numpy + cloud SDK wall) from the install path - runner: progress/status hooks through generate+evaluate, checkpoint key scheme fix (empty-store falsy bug), tiered retry backoff, multi-segment pool {range} expansion fix, adapter-instance passthrough - pyproject: tree_sitter family joins core deps; [bfcl] extra retired - README: rewritten (zh) -- install/quickstart/flags reference/bench table/reliability/extension/architecture/validation Co-Authored-By: Claude --- README.md | 670 ++++++------------ evalharness/__init__.py | 45 +- evalharness/agent/envs/bfcl_mock.py | 11 +- evalharness/cli.py | 170 ++++- evalharness/data/datasets/drop.py | 4 +- evalharness/model/runner.py | 112 ++- evalharness/progress/__init__.py | 12 + evalharness/progress/rich_terminal.py | 122 ++++ evalharness/third_party/bfcl/__init__.py | 8 + evalharness/third_party/bfcl/ast_checker.py | 636 +++++++++++++++++ .../bfcl/type_convertor/__init__.py | 0 .../type_convertor/java_type_converter.py | 407 +++++++++++ .../bfcl/type_convertor/js_type_converter.py | 311 ++++++++ evalharness/third_party/bfcl/type_mappings.py | 89 +++ evalharness/viz/renderers/excel.py | 2 +- pyproject.toml | 9 +- 16 files changed, 2127 insertions(+), 481 deletions(-) create mode 100644 evalharness/progress/__init__.py create mode 100644 evalharness/progress/rich_terminal.py create mode 100644 evalharness/third_party/bfcl/__init__.py create mode 100644 evalharness/third_party/bfcl/ast_checker.py create mode 100644 evalharness/third_party/bfcl/type_convertor/__init__.py create mode 100644 evalharness/third_party/bfcl/type_convertor/java_type_converter.py create mode 100644 evalharness/third_party/bfcl/type_convertor/js_type_converter.py create mode 100644 evalharness/third_party/bfcl/type_mappings.py diff --git a/README.md b/README.md index 3c429a4..c8a24b2 100644 --- a/README.md +++ b/README.md @@ -1,503 +1,287 @@ -# EvalHarness — 插件化评测框架完全指南 +# EvalHarness -> 万物皆插件的 LLM/Agent 评测框架。28 个内置 benchmark，与 evalscope 同题对齐验证 -> （Qwen3-8B 23/28 达标；DeepSeek-V4-Flash 全量 20+/25 达标）。 +**插件式 LLM / Agent 评测框架。** 28 个主流 benchmark 开箱即用：官方口径的 prompt 模板与判分器、 +任意 OpenAI 兼容推理端点、断点续跑、沙箱化代码执行 —— 每一层都可以用单文件插件扩展。 ---- +``` +你只需提供： 一个 OpenAI 兼容端点（vLLM / SGLang / lmdeploy / ollama / 云 API） +框架完成： 拉取数据 → 渲染官方 prompt → 并发生成 → 官方口径判分 → 产出报告 +``` -# 〇、从零到跑完 28 个 bench（Quick Start） +## 核心特性 -## 0.1 安装 +- **28 个内置 benchmark** —— 数学、知识/选择题、问答、长上下文、代码执行、agent 工具调用， + 全部对接官方数据源注册。 +- **官方口径评测** —— prompt 模板、few-shot 范例渲染（含按科目域匹配选范例）、判分器全部复刻 + 官方实现：数学用 PRM800K sympy 等价、DROP 用匈牙利对齐、SimpleQA 用官方 A/B/C judge 协议、 + BFCL 用官方 AST 判定、代码题在 docker 沙箱执行。已在 Qwen3-8B 与 DeepSeek-V4-Flash 上 + 与 evalscope 逐题对齐[验证](#对齐验证)。 +- **任意推理栈** —— 单端点或端点池：轮询分发、自适应并发（AIMD）、坏端点冷却、自动 failover。 +- **可断点、抗抖动** —— 每条预测完成即落盘，中断重跑只补缺失样本；分钟级断网靠分层重试扛过 + 而不丢批次。 +- **换判分器不用重新生成** —— 原始预测是不可变 artifact：改 recipe、换 grader 直接对存量输出 + 重新判分，模型永不被重复调用。 +- **万物皆插件** —— 数据集、prompt 渲染器、抽取器、判分器、聚合器、recipe、模型适配器、沙箱、 + agent 环境，全部 `@register_*` 单文件注册，没有需要修改的中央清单。 +- **评测 agent 而非扮演 agent** —— 被测模型负责思考；框架只把它的 tool_calls 交给 `Environment` + 插件执行并回灌观察，全程记录轨迹。 + +## 安装 + +要求 Python ≥ 3.10。 ```bash -git clone EvalHarness +git clone https://git.meta-stone.net/sora/EvalHarness.git cd EvalHarness -pip install -e . # editable 安装：改源码立即生效 -# 可选重依赖（只有 BFCL 官方判定器需要）： -pip install '.[bfcl]' +pip install . ``` -安装后命令行直接可用（无需 sys.path hack）： +**一条命令装完即可跑全部 28 个 benchmark，没有任何可选依赖。** + +| 官方判分逻辑形态 | 涉及 benchmark | 位置 | +|---|---|---| +| 纯 Python 算法 | 数学、DROP、MCQ | 核心依赖（sympy/numpy/scipy） | +| BFCL 官方 AST 判定器 | bfcl_v3 | 已内置（`evalharness/third_party/bfcl/`，Apache-2.0） | +| 官方执行环境 | humaneval、bigcodebench、live_code_bench、swe_bench | Docker 镜像，判分时按需拉取 | + +另有本地引擎类（`tau2_bench`）使用官方 tau2 包（本地源码安装，无重依赖）。 + +离线验证安装（不需要模型、不联网）： ```bash -evalharness --help +evalharness eval run gsm8k --model mock:boxed --limit 8 +# acc 100% —— mock 适配器直接输出金标答案，证明 +# 数据 → prompt → 生成 → 判分 → 报告 全链路可用 ``` -## 0.2 看看有什么 +代码执行类 benchmark 需要宿主机有 Docker。 + +## 快速开始 + +### 跑一个 benchmark（CLI） ```bash -evalharness data list # 28 个数据集插件（零网络） -evalharness eval list # 28 个判分 recipe -``` - -## 0.3 拉数据（惰性，也可以跳过让跑批时自动拉） - -```bash -evalharness data fetch gsm8k mmlu arc --workers 8 # 常用 bench 预拉 -evalharness data fetch bbh --subset word_sorting # 单个子集 -evalharness data stats cmmlu # 条数/长度/答案分布 -evalharness data show gsm8k -n 2 # 看前 2 条样本长什么样 -evalharness data unload gsm8k # 删缓存 -``` - -## 0.4 跑一个 bench（三种方式） - -**方式 A：CLI 一条命令** -```bash -evalharness eval run gsm8k --model openai/http://localhost:8000/v1?qwen3-8b \ +evalharness eval run gsm8k \ + --api-url http://localhost:8000/v1 \ + --model qwen3-8b \ --limit 200 --resume ``` -**方式 B：Python 三行** +**完整参数表**（`evalharness eval run --help` 的整理版）： + +**模型接入** + +| 参数 | 作用 | +|---|---| +| `--api-url URL` | OpenAI 兼容端点；与 `--model` 搭配使用（不用手拼 spec 字符串） | +| `--model NAME` | 服务端模型名（配合 `--api-url`）；或直接给完整 spec：`openai/http://h:8000/v1?qwen3-8b` | +| `--provider {openai-chat,openai-pool}` | 协议/提供方，默认 `openai-chat`；端点池用 `openai-pool` | +| `--judge SPEC` | LLM-judge 模型 spec（hle / simple_qa / imo 等 judge 类 recipe 需要） | +| `--profile NAME` | 命名生成参数集（内置 `dp4-nothink`、`qwen3-es-parity`、`t1-short`，或任意 `@register_gen_profile` 名）；优先级：插件默认 < profile 默认 < profile 单 bench 覆盖 < 显式参数 | +| `--disable-thinking` | 发送 `enable_thinking=false`（Qwen3 类思考模型推荐；带 tools 的请求自动退回模板安全的软开关） | +| `--textools` | 工具以文本形式随 prompt 下发，而非原生 tool_calls | +| `--perf` | 采集流式 TTFT / ITL / 重试率，写入报告 `perf` 组 | + +**采样与选样** + +| 参数 | 作用 | +|---|---| +| `--limit N` | 只跑全局前 N 条 | +| `--limit-per-task N` | 每个 subset / 科目取前 N 条（多科目 bench 的语义；可与 `--limit` 组合取交集） | +| `--subset NAME` | 覆盖子集（如 mmlu 的 `anatomy`、bbh 的 `word_sorting`） | +| `--split NAME` | 覆盖 split | +| `--source PATH` | 覆盖数据源（指向本地目录/文件，离线可用） | +| `--cache-dir DIR` | 缓存根目录（默认 `$EVALHARNESS_CACHE` 或 `~/.cache/evalharness`） | + +**运行控制** + +| 参数 | 作用 | +|---|---| +| `--concurrency N` | 并发请求数（默认 32；长输出 bench 建议 8-16） | +| `--resume [PATH]` | 断点续跑（默认路径自动推导；可显式给路径） | +| `--env NAME` | agent 环境（如 `bfcl_mock`）→ 走多轮消息泵 | +| `--progress` / `--no-progress` | Rich 每样本进度条（默认开；重定向日志时建议 `--no-progress`，未装 rich 自动降级纯文本） | + +**输出** + +| 参数 | 作用 | +|---|---| +| `--out FILE` | 单 benchmark 时把 EvalReport JSON 存到此处 | +| `--out-dir DIR` | 多 benchmark 运行落盘：`reports/.json` + `viz/.txt` + `summary.md` | +| `--style {text,md,radar,errors}` | 结果渲染样式 | +| `--verbose` | 多 benchmark 时也打印每个 bench 的完整渲染 | + +端点池跨机器跨端口，自带 failover 与自适应并发： + +```bash +evalharness eval run mmlu \ + --provider openai-pool \ + --api-url 'http://gpu1:{8123..8130}/v1,http://gpu2:{8200..8203}/v1' \ + --model qwen3-8b --disable-thinking +``` + +### 用 Python 跑 + +```python +import evalharness + +rep = evalharness.run('gsm8k', 'openai/http://localhost:8000/v1?qwen3-8b', limit=200) +print(rep.metrics['acc']) +rep.save('gsm8k.report.json') + +# 已在事件循环里（notebook）？用 await 版： +rep = await evalharness.arun('mmlu', '...', subset='anatomy') +``` + +也可以分阶段自己驱动 —— 数据集是一等公民，判分永远不会重新调模型： + ```python from evalharness import get_dataset -from evalharness.model import run_eval -import asyncio +from evalharness.eval import evaluate -rep = asyncio.run(run_eval( - get_dataset('gsm8k'), - 'openai/http://localhost:8000/v1?qwen3-8b', # 单端点 - limit=200, -)) -print(rep.metrics) # {'acc': 0.95, ...} +ds = get_dataset('mmlu', subset='anatomy') # 惰性句柄；首次使用才物化 +preds = [json.loads(l)['raw'] for l in open('preds.jsonl')] +rep = evaluate(ds, preds) # recipe 按 bench 名自动解析 ``` -**方式 C：多端点池 + 生成参数 profile** -```python -rep = asyncio.run(run_eval( - get_dataset('mmlu'), - 'openai-pool/http://gpu1:{8123..8130}/v1,gpu2:{8200..8203}/v1?qwen3-8b!nothink', - gen_profile='qwen3-es-parity', # 命名参数集（温度/max_tokens per bench） - limit_per_task=10, # 每科目 10 条（evalscope --limit 语义） -)) -``` - -## 0.5 需要 judge 的 bench（hle / simple_qa / imo） +### 查看与渲染结果 ```bash -evalharness eval run hle --model openai/...?qwen3-8b \ - --judge openai/https://api.example.com/v1?deepseek-v4-flash \ - --limit-per-task 25 +evalharness viz show gsm8k.report.json # 控制台表格 +evalharness viz show a.json b.json --style md_compare # 多模型对照 +evalharness viz show report.json --style excel # 4-sheet 仪表盘 +evalharness viz show report.json --style errors # 失败样本下钻 ``` -## 0.6 代码执行类 bench（humaneval / bigcodebench / live_code_bench） +每样本结果保留：原始预测、抽取说明、分数明细、token 用量，agent bench 另有完整轨迹。 +`extraction_failure_rate` 作为健康指标上报 —— 抽取失败不会被静默记零分。 -自动走 docker 沙箱（需要本机 docker）： -```bash -evalharness eval run humaneval --model openai/...?qwen3-8b -# bigcodebench 需要官方镜像： -docker build -f docker/Dockerfile.bigcodebench -t bigcodebench-sandbox:latest . -evalharness eval run bigcodebench --model openai/...?qwen3-8b -# swe 需要 per-instance sweb.eval.* 镜像： -evalharness sandbox prefetch swe_bench_verified --limit 20 -evalharness eval run swe_bench_verified --model openai/...?qwen3-8b --limit 20 -``` +## 模型接入 -## 0.7 Agent 类 bench（bfcl_v3 / general_fc / tau2_bench） +任意 OpenAI 兼容端点。模型用一条 spec 字符串描述（或用等价的 `--provider/--api-url/--model` 参数）： + +| spec | 含义 | +|---|---| +| `openai/?` | 单端点（vLLM、SGLang、lmdeploy、ollama、云 API） | +| `openai-pool/?` | 端点池：轮询 + 自适应并发 + failover | +| `deploy:/` | 经 Deployer 解析（钉版本的推理环境） | +| `mock` / `mock:boxed` / `mock:fc` | 离线适配器（管线自检） | +| `!nothink` `!perf` `!textools` 后缀 | spec 内联开关（与 CLI 参数等价） | + +API key 按端点从环境变量读取（`OPENAI_API_KEY`、`ANTHROPIC_API_KEY` …）。 + +## 内置 benchmark + +| 族 | benchmark | +|---|---| +| 数学 | `gsm8k`、`competition_math`、`aime24/25/26`、`hmmt26`、`imo_answerbench` | +| 知识 / 选择题 | `mmlu`、`mmlu_pro`、`cmmlu`、`gpqa_diamond`、`arc`、`hellaswag`、`winogrande`、`bbh` | +| 问答 | `trivia_qa`、`drop`、`simple_qa`、`hle` | +| 长上下文 | `longbench_v2`、`openai_mrcr` | +| 代码（沙箱执行） | `humaneval`、`bigcodebench`、`live_code_bench` | +| Agent / 工具 | `bfcl_v3`、`general_fc`、`tau2_bench`、`swe_bench_verified` | ```bash -evalharness eval run bfcl_v3 --model openai/...?qwen3-8b --env bfcl_mock -# tau2 需要官方数据 + TAU2_DATA_DIR 环境变量： -TAU2_DATA_DIR=/path/to/tau2-bench/data \ -evalharness eval run tau2_bench --model openai/...?qwen3-8b +evalharness data list # 全部数据集：数据源、子集、默认 few-shot、split +evalharness eval list # 全部判分 recipe ``` -## 0.8 长上下文 bench（lb2 / mrcr，128k 截断） +各族注意事项： + +- **LLM-judge 类**（`hle`、`simple_qa`、`imo_answerbench`）：传 `--judge `；judge 走官方协议 + （如 SimpleQA 的分级正确性 + NOT_ATTEMPTED 兜底）。 +- **代码执行类**：模型生成的代码在硬隔离 Docker 中运行（`--network none`、cgroup 上限、只读 rootfs）； + swe 的逐实例 `sweb.eval.*` 镜像用 `evalharness sandbox prefetch swe_bench_verified` 预取。 +- **Agent 类**：`--env bfcl_mock` 驱动多轮消息泵 + 官方 call-sequence 判分；`tau2_bench` / + `swe_bench_verified` 走官方引擎 bundle（自跑环境路径）。 +- **长上下文**：`gen_kwargs={'max_input_tokens': ...}` 做 token 预算中位截断（保头尾），与 evalscope 同构。 +- **小样本高方差集**（aime/hmmt）：`temperature=1.0` 跑多次取均值 —— 用 `evalharness.run` 五行循环。 + +## 可靠性模型 + +- **断点** —— 每条预测完成即追加进当次运行的 checkpoint（`--resume`）。key 含 prompt 语义； + 改了模板要删旧断点（`rm ~/.cache/evalharness/ckpt/*.jsonl`），否则会复用旧预测。 +- **分层重试** —— 连接超时 15 秒快速失败；池内换端点（自适应闸门自动降载、病端点冷却）； + 单样本分钟级退避重试 6 次，路由抖动不会杀死批次。 +- **预测不可变** —— 判分是对存量输出的确定性计算；随便换 grader / recipe。 + +## 扩展 + +**加数据集** —— 单文件丢进 `evalharness/data/datasets/`，自动发现注册： ```python -rep = asyncio.run(run_eval( - get_dataset('longbench_v2', subset='medium'), - 'openai/http://bigctx:30000/v1?model', # 需要 262k ctx 端点 - gen_kwargs={'max_input_tokens': 128000}, # 128k 中截（同 evalscope） -)) -``` - -## 0.9 多轮采样（temp=1 × N 次取均值，aime/hmmt 系列） - -```python -runs = [] -for i in range(12): - rep = asyncio.run(run_eval(get_dataset('aime25'), MODEL, - gen_kwargs={'temperature': 1.0})) - runs.append(rep.metrics['acc']) -print(f'mean: {sum(runs)/len(runs):.4f}') -``` - -## 0.10 查看结果 - -```bash -evalharness viz show gsm8k.report.json # 控制台表格 -evalharness viz show r1.json r2.json --style md_compare # 多模型对照 -evalharness viz show report.json --style excel # 4-sheet Excel 仪表盘 -``` - -## 0.11 跑全部 28 个（编排脚本模板） - -```python -"""full_28.py — 用跑批脚本编排全部 bench""" -import asyncio, json, os -from evalharness import get_dataset -from evalharness.model import run_eval - -MODEL = 'openai-pool/http://gpu1:{8123..8130}/v1?qwen3-8b!nothink' -JUDGE = 'openai/https://judge-api.example.com/v1?judge-model' -OUT = 'results' -os.makedirs(OUT, exist_ok=True) - -BENCHES = [ - # (name, dataset, kwargs) - ('wino', 'winogrande', dict(limit=1267)), - ('arc', 'arc', dict()), - ('gsm8k', 'gsm8k', dict(limit=1319)), - ('hswag', 'hellaswag', dict(limit=10042)), - ('cmmlu', 'cmmlu', dict(subset='all')), - ('mmlu', 'mmlu', dict()), - ('mmlu_pro', 'mmlu_pro', dict()), - ('trivia', 'trivia_qa', dict()), - ('drop', 'drop', dict()), - ('math', 'competition_math', dict(subset='all')), - ('humaneval', 'humaneval', dict()), - ('bcb', 'bigcodebench', dict()), - ('lcb', 'live_code_bench', dict(subset='release_latest')), - ('bfcl', 'bfcl_v3', dict(env='bfcl_mock')), - ('gfc', 'general_fc', dict()), - # judge 类 - ('sqa', 'simple_qa', dict(judge_spec=JUDGE)), - ('hle', 'hle', dict(judge_spec=JUDGE)), - ('imo', 'imo_answerbench', dict(judge_spec=JUDGE)), - # 长上下文 - ('lb2', 'longbench_v2', dict(subset='short', gen_kwargs={'max_input_tokens': 128000})), - ('mrcr', 'openai_mrcr', dict(gen_kwargs={'max_input_tokens': 128000})), - # agent - ('tau2', 'tau2_bench', dict()), - ('swe', 'swe_bench_verified', dict(limit=70)), -] - -async def run_one(tag, name, kw): - out = f'{OUT}/{tag}.json' - if os.path.exists(out): - print(f'skip {tag}'); return - subset = kw.pop('subset', None) - ds = get_dataset(name, subset=subset) if subset else get_dataset(name) - rep = await run_eval(ds, MODEL, checkpoint=True, **kw) - json.dump({'n': rep.num_samples, 'metrics': rep.metrics}, open(out, 'w'), default=str) - print(f'## {tag}: {rep.metrics}', flush=True) - -async def main(): - for tag, name, kw in BENCHES: - await run_one(tag, name, kw) - # bbh: 27 子集循环 + 聚合 - BBH = ['boolean_expressions', 'causal_judgement', ...] # 27 个 - vals = [] - for sub in BBH: - await run_one(f'bbh_{sub}', 'bbh', dict(subset=sub, limit_per_task=10)) - vals.append(json.load(open(f'{OUT}/bbh_{sub}.json'))['metrics']['acc']) - json.dump({'acc': sum(vals)/len(vals)}, open(f'{OUT}/bbh.json', 'w')) - # aime × 3 + hmmt: t1 × 12 轮均值 - for b in ['aime24', 'aime25', 'aime26', 'hmmt26']: - runs = [] - for i in range(12): - rep = await run_eval(get_dataset(b), MODEL, - gen_kwargs={'temperature': 1.0, 'max_tokens': 32768}) - runs.append(rep.metrics['acc']) - json.dump({'runs': runs}, open(f'{OUT}/{b}.partial.json', 'w')) # 断点 - json.dump({'mean': sum(runs)/len(runs)}, open(f'{OUT}/{b}.json', 'w')) - -asyncio.run(main()) -``` - -```bash -# 后台跑 + 崩溃自动续（ckpt 断点）： -setsid python -u full_28.py > full_28.log 2>&1 < /dev/null & -tail -f full_28.log -``` - ---- - -# 一、每个插件怎么写、怎么用（每类一个完整 case） - -## 1.1 数据集插件 —— "这个 benchmark 的题目长什么样" - -**写**（`data/datasets/mybench.py`，放进去就被自动发现，无需改任何中央文件）： - -```python -from ..sample import Sample -from ..registry import register_dataset -from ..spec import DatasetSpec - @register_dataset(DatasetSpec( name='mybench', - source='org/mybench', # HF id / ModelScope id / 本地路径 + source='org/mybench', # HF id / ModelScope id / 本地路径 split='test', - task_type='mcq', # 决定判分 recipe 的大类路由 - prompt_style='cot_letter', # 引用哪个 prompt 渲染插件（见 1.2） - few_shot_split='dev', # 范例从哪个 split 取 - few_shot_num=5, - gen_config={'temperature': 0.0, 'max_tokens': 4096}, # 生成默认参数 + task_type='mcq', # 决定默认判分 recipe 的大类路由 )) def mybench(): - # 写法 A：字段名刚好对得上 → 一行声明式 return FieldSpec(input='question', choices='options', target='answer_key') - - # 写法 B：需要清洗/重排/增强 → 返回转换函数 - # def to_sample(record): - # return Sample(input=record['q'], choices=record['opts'], - # target='ABCD'[record['label']], metadata={'subject': record['sub']}) - # return to_sample + # 需要清洗/重排时改为返回 record -> Sample 函数 ``` -**用**： +**绑定判分** —— recipe 是注册原语的声明式组合： ```python -from evalharness import get_dataset -ds = get_dataset('mybench') # 惰性：零网络 -len(ds) # 首次使用才下载→转换→缓存 -for s in ds: print(s.input, s.target) -ds2 = get_dataset('mybench', subset='hard') # spec 覆盖 → 独立缓存条目 -``` - -**缓存规则**：subset/split/source/params 全部参与 hash —— 改任何一项自动新缓存目录， -永远不用写缓存失效逻辑。 - -## 1.2 Prompt 渲染插件 —— "题目怎么渲染给模型" - -> **为什么独立成层而不塞进数据插件？** 渲染是**生成层的关注点**：同一个数据集可能被 -> 不同协议渲染（zero-shot / CoT / 官方 few-shot），而数据插件只该回答"题目是什么"。 -> 但注册表是全局的 —— renderer 完全可以写在数据插件同一个文件里。 - -**写**（任意文件，包括数据插件同文件）： - -```python -from evalharness.model.prompt_renderers import register_prompt_renderer - -@register_prompt_renderer('mybench_cot') # ← DatasetSpec.prompt_style 填这个名字 -def mybench_cot(question, sample, spec, prompt_style): - # 输入：裸题面 + Sample + DatasetSpec - # 输出：{'question': 改写后的题面}，可选 'system'（变成 system 消息） - if not sample.choices: - return {} # 返回空 → 走通用兜底 - letters = 'ABCD' - opts = '\n'.join(f'{letters[i]}) {c}' for i, c in enumerate(sample.choices)) - return {'question': f'Answer the question.\n\n{question}\n\n{opts}'} -``` - -**用**：`DatasetSpec(prompt_style='mybench_cot')` —— 之后所有 `run_eval` 自动走它； -没注册的 style 走通用 MCQ/QA 兜底。**验证工具**：golden prompt 快照 —— 渲染输出 -逐字节存档，改渲染器后跑对比，保证不悄悄变。 - -## 1.3 few-shot 钩子 —— "官方手写范例" - -数据插件同文件加一个约定名函数即可（注册时自动被发现）： - -```python -def mybench_few_shot(split, subset, n): - return official_cot_text[subset] # 返回 None 则回退到 few_shot_split 自动取 -``` - -## 1.4 模型适配插件 —— "怎么调用模型" - -```python -@register_adapter('myproto') -class MyProto(ModelAdapter): - async def generate(self, messages, tools=None, **kw) -> ModelOutput: - # 任何协议：gRPC、私有 SDK、云 API…… - return ModelOutput(text=..., tool_calls=[...], usage=Usage(...)) -``` - -**用**：spec 字符串 `'myproto://host:port?model-id'`。 - -## 1.5 流量管理 —— "多端点怎么打满不打死"（内建，无需写） - -``` -spec: openai-pool/http://51.3:{30014..30014}/v1,http://51.4:{30000..30000}/v1?dp4-flash - → 每端点一个 AdaptiveGate（AIMD）： - /metrics 显示没喂饱 → 并发 +1（每 5s） - 服务端排队 → 并发 -1 - 请求失败 → 并发 ×0.7（保命） - + 连续失败健康冷却 60s + 端点假死探活（哨兵 docker restart） -``` - -## 1.6 判分三件套 —— extractor / scorer / aggregator - -```python -@register_extractor('my_answer') -def my_answer(raw, sample): # → (value, ok, note) - m = re.search(r'MY ANSWER: (.+)', raw) - return (m.group(1), True, 'regex') if m else ('', False, 'no match') - -@register_scorer('my_metric') -def my_metric(pred, target, sample, ctx): # → ({metric: 分数}, {metric: 详情}) - return ({'acc': float(pred == target)}, {'acc': {'pred': pred}}) - -@register_aggregator('my_group') -def my_group(results, metric): - ... # → float 或 {子组名: 分数} - -# recipe = 三件套的声明式组合（每 bench 5-20 行） @register_eval('mybench') def mybench(): return EvalRecipe( name='mybench', extract=['my_answer', 'answer_phrase'], # 级联：首个成功者胜 - scorers={'acc': 'my_metric'}, - aggregators={'acc': 'my_group'}, - exec_workers=8, # execution 类并行判分 + scorers={'acc': 'exact'}, # 或 math_equal / em_f1 / execution / env_reward / llm_judge + aggregators={'acc': 'mean'}, # 或 pass_at_k / grouped_avg / binned_avg ) ``` -## 1.7 沙箱插件 —— "在哪儿跑模型生成的代码" +同一模式覆盖全部扩展点 —— `@register_prompt_renderer`（题面如何渲染，含 system 契约）、 +`@register_extractor` / `@register_scorer` / `@register_aggregator`、`@register_adapter`（模型协议）、 +`@register_sandbox`（执行环境）、`@register_env`（agent 世界）、`@register_renderer`（报告呈现）。 +生成参数预设注册为 profile（`--profile`）。 -```python -@register_sandbox('myvm') -class MyVM: - def exec(self, files: Dict[str, str], entry: str, - timeout_s: int, image: str) -> ExecResult: - # files: {filename: content} 写入容器 - # entry: 容器里跑的入口文件 - # 返回 ExecResult(exit_code, stdout, stderr, timed_out, duration_s) - ... -``` +数据插件还可以导出 `_few_shot(split, subset, n)` 钩子注入官方手写范例（BBH CoT 即此实现）； +DatasetSpec 声明每 bench 的默认值（few-shot 数量/split、`gen_config`、prompt 风格）， +常见场景零配置。 -内建两个： -- `docker`：硬隔离（`--network none` + cpu/mem/pids 上限 + tmpfs /tmp），支持任意镜像 -- `local`：子进程直跑（开发调试用，无隔离） - -## 1.8 Agent 环境插件 —— "多轮工具调用的世界" - -两种模式： - -```python -@register_env('my_sim') -class MySim(Environment): - # 模式 A：消息泵（框架驱动循环） - def reset(self, sample) -> List[ChatMessage]: - return [] # 初始观察 - - async def step(self, tool_calls, text, sample) -> List[ChatMessage]: - # 执行模型的 tool_calls，返回观察消息 - return [ChatMessage(role='tool', content=json.dumps(result))] - - def final_state(self) -> dict: - return {'calls': self.calls} # 传给 env_reward scorer - - # 模式 B：自跑旁路（官方引擎 bundle） - async def run_task(self, adapter, sample, **kw) -> Optional[dict]: - # 整个模拟在引擎内部完成，返回 prediction dict - # 返回 None 则回退到模式 A 的消息泵 -``` - -内建：`bfcl_mock`（BFCL 官方 ast_checker 判定）、`tau2_official`（tau2 官方引擎）。 - -## 1.9 渲染插件 —— "报告怎么展示" - -```python -@register_renderer('my_style') -def my_style(reports: List[EvalReport]) -> str: - return '...' # 任意格式的字符串 -``` - -内建：`text`（控制台表格）、`md`/`md_compare`（单/多模型 Markdown 对照）、 -`excel`（4-sheet 仪表盘）、`radar`、`errors`（失败样本下钻）。 - -## 1.10 生成参数 Profile —— "不同模型不同参数" - -```yaml -# gen_profiles.yaml（当前目录或 ~/.config/evalharness/） -my-protocol: - default: - temperature: 0.0 - max_tokens: 32768 - simple_qa: # 单 bench 覆盖 - max_tokens: 512 -``` - -```bash -evalharness eval run hle --model ... --profile my-protocol -``` - -优先级：插件默认 < profile.default < profile[bench] < 显式 kwargs。 - ---- - -# 二、还不够插件化的地方（Before → After 全对照） - -## P0 跑批编排层 —— 最大的硬编码 - -**现在**：所有编排逻辑住在 `/tmp/opencode/dp4_run.py` + 5 个 tail 脚本 + 哨兵 bash， -共 ~400 行一次性代码。 - -**改后**： -```python -@register_protocol('dp4-full') -def dp4_full(): - return Protocol( - model='openai-pool/...', - runs=[FullRun('mmlu'), MeanRun('aime24', k=12), JudgedRun('hle', judge='...')], - sentinel=True) -``` -```bash -evalharness run --protocol dp4-full -evalharness status -``` - -## P1 重判分 CLI - -**现在**：判分出问题手写 40 行 rejudge 脚本（处理 ckpt key 三种形态）。 - -**改后**： -```bash -evalharness eval rescore hle --ckpt latest --judge pool:judge -``` - -## P2-P18（详见各节） - -- P2 数据源 variant + 选项排列策略（gpqa 的 es-dump 已实现） -- P3 ckpt key 统一（指纹匹配） -- P4 judge 走池容灾 -- P5 沙箱 warm pool（容器复用，bcb 再快 3-5×） -- P6 截断策略插件 -- P7 选样语义插件 -- P8 few-shot 渲染进 renderer -- P9 运行时心跳监控 -- P10 模型策略外置 -- P11 协议 profile（已实现 gen_profiles） -- P12 依赖校验 -- P13 judge prompt 版本化 -- P14 聚合视图插件 -- P15 Web/API -- P16 工具层（filter/synthesis/dedup） -- P17 Skill 层 -- P18 结果对比器 - ---- - -# 三、系统运行全景 +## 架构 ``` -get_dataset('mmlu') ──惰性物化+flock缓存──▶ Dataset[Sample] - │ -run_eval(ds, model_spec, judge_spec, gen_profile) - │ few-shot hook / 域匹配范例 - │ prompt renderer 插件改写题面 - │ 截断（token 中截，budget = ctx − max_tokens − 2k） - ▼ -PooledAdapter ──round-robin──▶ N 端点 × AdaptiveGate(AIMD) - │ 失败：换端点 × N + gate ×0.7 + 冷却 - │ 断网：run_one 六次分钟级退避 - │ 每条预测 append 进 ckpt（key 含 prompt 语义） - ▼ -evaluate(samples, preds, recipe) - │ extractor 级联 → scorer → aggregator - │ execution 类：exec_workers 线程并行 docker/subprocess - ▼ -EvalReport（raw_prediction 永不丢 → 换 recipe 重判不重跑） - ▼ -viz render（text/md_compare/excel/radar/errors） +evalharness/ +├── cli.py data | eval | sandbox | viz 子命令 +├── progress/ Rich 每样本终端进度（无 rich 自动降级） +├── data/ 统一 Sample schema、惰性物化、内容寻址缓存 +│ └── datasets/ 28 个单文件插件 +├── model/ adapter（怎么调）+ deployer（怎么部署） +│ ├── pool.py 端点池、自适应闸门、failover +│ ├── prompt_renderers 逐 bench 官方 prompt 模板 +│ ├── gen_profiles.py 命名生成参数预设 +│ └── runner.py 异步生成 → 同步判分；断点、重试、进度钩子 +├── eval/ extract → score → aggregate 流水线 + recipes +├── sandbox/ docker（硬隔离执行）/ local；镜像引用计数 +├── agent/ 消息泵 + Environment 插件（bfcl / tau2 / swe） +└── viz/ text / md / md_compare / radar / excel / errors ``` -# 四、对齐战绩与残差定性 +设计原则： -- **Qwen3-8B**：23/28 同题达标 -- **DeepSeek-V4-Flash**：20+/25 达标；mmlu_pro diff 0.0000 -- es 侧无效分：imo 0.0（judge 白跑）、bigcodebench 0.9956（执行器空跑） -- 已定性残差：drop（es 多金标）、gpqa（排列敏感，es-dump 口径 0.046 ✅） +- **层间严格分离** —— 数据层回答"题目与金标是什么"；判分层回答"如何评判一个回复"；模型层回答 + "如何触达模型"。下游只见 `Sample`，不见原始数据格式。 +- **声明优先于执行** —— 样本携带沙箱/工具声明，由执行层物化；数据层永不执行任何东西。 +- **注册表而非配置文件** —— 插件 import 即注册；`list` 命令枚举不触网。 + +## 对齐验证 + +prompt 模板与判分器在两个层面与 evalscope 对齐过：字符串级（同一条记录过两侧管线，prompt 逐字节 +一致）与分数级（同题同生成参数）： + +- **Qwen3-8B**：28 个 benchmark 中 23 个同题分差 < 0.05。 +- **DeepSeek-V4-Flash**：25 个中 20+ 个分差 < 0.05（mmlu_pro 分差 0.0000）。 + +残差分歧是定性而非掩盖：已知原因包括金标集差异（DROP 的 validated answers）、排列敏感性（GPQA）、 +以及 benchmark 侧缺陷（evalscope 的 bigcodebench 执行器空跑、BFCL 单轮格式提示被剥 —— 均有 +代码级证据记录，我方侧已规避且未改动 evalscope）。 + +## License + +尚未声明 —— 公开发布前请添加 `LICENSE`。第三方 benchmark 数据与内置的官方判分片段 +（如 BBH CoT 范例、SimpleQA judge prompt）保留上游许可；各数据集插件头部记录了数据来源。 diff --git a/evalharness/__init__.py b/evalharness/__init__.py index 165a512..0408fb9 100644 --- a/evalharness/__init__.py +++ b/evalharness/__init__.py @@ -15,7 +15,50 @@ from .data import ( __version__ = '0.1.0' + +async def arun(bench, model, **kwargs): + """ASYNC entry: for callers already inside an event loop. + + import evalharness + rep = await evalharness.arun('gsm8k', 'openai/http://...?m', limit=200) + """ + return await _run_dispatch(bench, model, **kwargs) + + +def run(bench, model, **kwargs): + """SYNC entry (primary API, mirroring inspect_ai.run / lm_eval). + + import evalharness + rep = evalharness.run('gsm8k', 'openai/http://...?m', limit=200) + + Creates its own event loop; safe to call from scripts/notebooks. + """ + import asyncio + + try: # already inside a loop (notebook)? run in a thread + asyncio.get_running_loop() + import concurrent.futures + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, _run_dispatch(bench, model, **kwargs)).result() + except RuntimeError: + return asyncio.run(_run_dispatch(bench, model, **kwargs)) + + +async def _run_dispatch(bench, model, **kwargs): + from .data import Dataset as _Dataset, get_dataset as _gd + from .model import run_eval as _run_eval + + if isinstance(bench, str): + ds = _gd(bench, subset=kwargs.pop('subset', None)) + elif isinstance(bench, (_Dataset, list)): + ds = bench + else: + raise TypeError(f'bench must be str/Dataset/list, got {type(bench)}') + return await _run_eval(ds, model, **kwargs) + + __all__ = [ 'Dataset', 'DatasetSpec', 'FieldSpec', 'Sample', 'ChatMessage', 'SandboxSpec', 'ToolInfo', - 'get_dataset', 'list_datasets', 'register_dataset', + 'get_dataset', 'list_datasets', 'register_dataset', 'run', 'arun', ] diff --git a/evalharness/agent/envs/bfcl_mock.py b/evalharness/agent/envs/bfcl_mock.py index ab59e41..fb430be 100644 --- a/evalharness/agent/envs/bfcl_mock.py +++ b/evalharness/agent/envs/bfcl_mock.py @@ -44,8 +44,12 @@ def bfcl_official(calls: List[Dict[str, Any]], sample: Sample) -> Optional[bool] the official decoded shape (incl. the dot->underscore name convention the checker itself applies via convert_func_name).""" try: - from bfcl_eval.eval_checker.ast_eval.ast_checker import ast_checker - except ImportError: + # vendored copy of the official checker (Apache-2.0, see + # evalharness/third_party/bfcl) -- verified in agreement with the + # bfcl-eval package on real BFCL rows; no cloud-SDK dependency tree + from ...third_party.bfcl.ast_checker import ast_checker + except ImportError as e: + print(f'bfcl_official unavailable: {e}', flush=True) return None gt = _parse_ground_truth(sample.target) if isinstance(gt, dict): @@ -81,7 +85,8 @@ def bfcl_official(calls: List[Dict[str, Any]], sample: Sample) -> Optional[bool] funcs = [{'name': t.name, 'description': t.description or '', 'parameters': t.parameters} for t in (sample.tools or [])] try: - result = ast_checker(funcs, model_output, possible, lang, category, convention) + result = ast_checker(funcs, model_output, possible, lang, category, convention, + underscore_to_dot=True) return bool(result.get('valid')) except Exception: # official checker chokes on some v3 schemas; evalscope's adapter diff --git a/evalharness/cli.py b/evalharness/cli.py index 012d95a..84f75a9 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -1,6 +1,7 @@ """EvalHarness CLI. Zero third-party deps beyond the data layer (pydantic).""" import argparse +import time import json import sys from concurrent.futures import ThreadPoolExecutor, as_completed @@ -123,6 +124,110 @@ def _cmd_eval_list(_args) -> int: return 0 +def _print_run_progress(done, total, name='', status='running', started=None): + """Print one live progress line for a multi-benchmark run.""" + import time + + width = 28 + filled = int(width * done / max(total, 1)) + bar = '#' * filled + '-' * (width - filled) + elapsed = time.time() - started if started else 0 + label = f'{done}/{total} [{bar}] {status}: {name}' + print(f'\r{label} ({elapsed:.0f}s)', end='\n' if done >= total else '', flush=True) + + +def _rich_console(): + """Return a Rich console when installed; keep the CLI dependency-free.""" + try: + from rich.console import Console + + return Console() + except ImportError: + return None + + +def _print_run_plan(console, args, model_spec): + """Print the important run facts before any dataset work starts.""" + title = 'EvalHarness · Run Plan' + provider = getattr(args, 'provider', 'openai-chat') if model_spec else '—' + api_url = getattr(args, 'api_url', '') or '—' + model_name = getattr(args, 'model', '') or 'predictions file' + if console is None: + print(f'=== {title} ===') + print(f'Provider: {provider}') + print(f'API URL: {api_url}') + print(f'Model: {model_name}') + print(f'Benchmarks: {len(args.datasets)} -> {", ".join(args.datasets)}') + print(f'Concurrency: {args.concurrency} | Thinking: ' + f'{"enabled" if not args.disable_thinking else "disabled"} | ' + f'Performance: {"on" if args.perf else "off"}') + print(f'Resume: {"on" if args.resume else "off"} | Output: {args.out_dir or "(none)"}') + print('Samples: counted after each dataset is loaded') + return + + from rich.panel import Panel + from rich.table import Table + + table = Table(show_header=False, box=None, padding=(0, 1)) + table.add_column('Item', style='cyan', no_wrap=True) + table.add_column('Value', style='white') + table.add_row('Provider', provider) + table.add_row('API URL', api_url) + table.add_row('Model', model_name) + table.add_row('Benchmarks', f'{len(args.datasets)} · {", ".join(args.datasets)}') + table.add_row('Samples', 'counted while loading each benchmark') + table.add_row('Concurrency', str(args.concurrency)) + table.add_row('Thinking', '[red]disabled[/red]' if args.disable_thinking else '[green]enabled[/green]') + table.add_row('Performance', '[green]enabled[/green]' if args.perf else '[dim]disabled[/dim]') + table.add_row('Checkpoint', '[green]resume[/green]' if args.resume else '[dim]new run[/dim]') + table.add_row('Output', args.out_dir or '[dim](not specified)[/dim]') + console.print(Panel(table, title=title, border_style='blue', expand=False)) + + +def _print_phase(console, index, total, name, message): + text = f'[{index}/{total}] {name}: {message}' + if console is not None: + console.print(f'[cyan]{text}[/cyan]') + else: + print(text, flush=True) + + +def _print_benchmark_result(console, index, total, name, status, elapsed): + if console is not None: + color = 'green' if status == 'done' else 'red' + icon = '✓' if status == 'done' else '✗' + console.print(f'[{color}]{icon}[/{color}] benchmark {index}/{total} ' + f'{name} · {status} · elapsed={elapsed:.0f}s') + else: + _print_run_progress(index, total, name, status, time.time() - elapsed) + + +def _model_with_flags(model, args): + """Translate explicit CLI flags to the adapter's internal options.""" + for enabled, flag in ((getattr(args, 'disable_thinking', False), '!nothink'), + (getattr(args, 'perf', False), '!perf'), + (getattr(args, 'textools', False), '!textools')): + if enabled and not model.endswith(flag): + model += flag + return model + + +def _compose_model_spec(args): + """Build the internal model spec from separate provider fields.""" + model = args.model or '' + api_url = getattr(args, 'api_url', '') or '' + provider = getattr(args, 'provider', 'openai-chat') or 'openai-chat' + # openai-chat is the public name; the current client implementation + # remains registered as openai internally. + internal_provider = {'openai-chat': 'openai', + 'openai-pool': 'openai-pool'}.get(provider, provider) + if api_url: + if not model: + raise SystemExit('error: --api-url requires --model (model name)') + model = f'{internal_provider}/{api_url.rstrip("/")}?{model}' + return _model_with_flags(model, args) + + def _cmd_eval_run(args) -> int: import asyncio import time as _time @@ -137,40 +242,73 @@ def _cmd_eval_run(args) -> int: Path(out_dir).mkdir(parents=True, exist_ok=True) (Path(out_dir) / 'viz').mkdir(exist_ok=True) + (Path(out_dir) / 'reports').mkdir(exist_ok=True) rows = [] + run_started = _time.time() + total_runs = len(args.datasets) + model_spec = _compose_model_spec(args) + console = _rich_console() + _print_run_plan(console, args, model_spec) for i, name in enumerate(args.datasets): t0 = _time.time() try: + _print_phase(console, i + 1, total_runs, name, + 'loading/downloading dataset') ds = get_dataset(name, **overrides) - if args.model: # generate + score in one go + sample_count = len(ds) + origin = ds.lineage.get('from', 'unknown') + _print_phase(console, i + 1, total_runs, name, + f'dataset ready · samples={sample_count} · source={origin}') + if model_spec: # generate + score in one go from evalharness.model import run_eval + progress_reporter = None + if args.progress: + from evalharness.progress import RichTerminalProgress + + if RichTerminalProgress is not None: + # share ONE console: phase lines printed by another + # writer during the live bar interleave incorrectly + progress_reporter = RichTerminalProgress(console=console) + + def status_callback(msg, _idx=i, _name=name, + _reporter=progress_reporter, + _console=console): + if _reporter is not None: + _reporter.log(f'[{_idx}/{total_runs}] {_name}: {msg}') + else: + _print_phase(_console, _idx, total_runs, _name, msg) report = asyncio.run(run_eval( - ds, args.model, concurrency=args.concurrency, limit=args.limit, + ds, model_spec, concurrency=args.concurrency, limit=args.limit, limit_per_task=args.limit_per_task, checkpoint=args.resume, judge_spec=args.judge, env=args.env, - gen_profile=getattr(args, 'profile', ''))) + gen_profile=getattr(args, 'profile', ''), + progress_reporter=progress_reporter, + status_callback=status_callback)) else: from evalharness.eval import evaluate if not args.predictions: raise SystemExit('error: provide --model or a predictions file') + _print_phase(console, i + 1, total_runs, name, 'loading predictions') preds_path = args.predictions[i] if len(args.predictions) > i else args.predictions[0] preds = [json.loads(line) for line in open(preds_path, encoding='utf-8') if line.strip()] preds = [p.get('raw', p.get('prediction', '')) if isinstance(p, dict) else p for p in preds] - report = evaluate(ds, preds, model=args.model or 'preds') + report = evaluate(ds, preds, model=model_spec or 'preds') + _print_phase(console, i + 1, total_runs, name, 'scoring complete') if args.out: report.save(args.out) if out_dir: + _print_phase(console, i + 1, total_runs, name, + 'writing report and visualization files') report.save(f'{out_dir}/reports/{name}.report.json') with open(f'{out_dir}/viz/{name}.txt', 'w', encoding='utf-8') as f: f.write(render(report, style='text')) if len(args.datasets) == 1 or args.verbose: print(render(report, style=args.style)) - print(render(report, style=args.style)) primary = next(iter(report.metrics), '') secs_total = sum(float((s.usage or {}).get('latency_s', 0) or 0) for s in report.samples) @@ -183,11 +321,15 @@ def _cmd_eval_run(args) -> int: 'secs': round(secs_total, 1), 'hours': round(secs_total / 3600, 2), 'groups': groups, 'ok': True}) + _print_benchmark_result(console, i + 1, total_runs, name, + 'done', _time.time() - t0) except Exception as e: rows.append({'name': name, 'metric': '-', 'value': None, 'secs': round(_time.time() - t0, 1), 'ok': False, 'err': f'{type(e).__name__}: {str(e)[:100]}'}) print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr) + _print_benchmark_result(console, i + 1, total_runs, name, + 'failed', _time.time() - t0) if len(rows) > 1: print(f'\n{"benchmark":<20} {"metric":<14} {"score":>8} {"n":>5} {"time":>9}') @@ -303,15 +445,29 @@ def build_parser() -> argparse.ArgumentParser: p.add_argument('datasets', nargs='+', help='dataset name(s) (recipe auto-resolved)') p.add_argument('predictions', nargs='?', help='jsonl: one raw string or {"raw": ...} per sample') p.add_argument('--model', default='', - help="generate with model spec: mock | mock:boxed | " - "openai/http://host:8000/v1?model | deploy:vllm/model") + help='served model name when --api-url is used; or full legacy model spec') + p.add_argument('--api-url', default='', + help='API base URL when --model is only the served model name') + p.add_argument('--provider', default='openai-chat', + choices=('openai-chat', 'openai-pool'), + help='API protocol/provider (default: openai-chat)') p.add_argument('--judge', default='', help='judge model spec for llm_judge recipes') p.add_argument('--profile', default='', help='named gen-params profile (dp4-nothink | qwen3-es-parity | t1-short ' 'or any @register_gen_profile name); layers: plugin default < ' "profile.default < profile[''] < explicit kwargs") + p.add_argument('--disable-thinking', action='store_true', + help='send enable_thinking=false to the OpenAI-compatible model') + p.add_argument('--perf', action='store_true', + help='collect streaming TTFT and ITL metrics') + p.add_argument('--textools', action='store_true', + help='send tools as text instead of native tool calls') p.add_argument('--env', default='', help="agent environment (e.g. 'bfcl_mock') -> message pump") p.add_argument('--concurrency', type=int, default=32, help='parallel model calls (default 32)') + p.add_argument('--progress', action='store_true', default=True, + help='show per-sample Rich terminal progress (default: on)') + p.add_argument('--no-progress', dest='progress', action='store_false', + help='disable per-sample Rich terminal progress') p.add_argument('--limit', type=int, help='evaluate only the first N samples total') p.add_argument('--resume', nargs='?', const=True, default=False, help='resume from per-sample checkpoint (default path auto-derived; ' diff --git a/evalharness/data/datasets/drop.py b/evalharness/data/datasets/drop.py index fc4fbce..fcbf3dc 100644 --- a/evalharness/data/datasets/drop.py +++ b/evalharness/data/datasets/drop.py @@ -14,9 +14,11 @@ def drop_few_shot(split, subset, n): hook path bypassed it, sending bare exemplars. Byte-diff against es's actual sent prompt showed the missing wrapper cost ~11 EM points on dp4. """ + # NOTE: do NOT append '# Your Task\n---\n' here — the runner's + # drop_style branch adds it (having it in both = doubled in the prompt) return ('You will be asked to read a passage and answer a question. ' 'Some examples of passages and Q&A are provided below.\n\n' - '# Examples\n---\n' + _DROP_FEWSHOT + '\n\n# Your Task\n---\n') + '# Examples\n---\n' + _DROP_FEWSHOT) from ..spec import DatasetSpec diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index d788f62..968369c 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -31,6 +31,8 @@ async def generate_predictions( limit: Optional[int] = None, gen_kwargs: Optional[Dict[str, Any]] = None, progress: bool = True, + progress_reporter=None, + status_callback=None, env_factory=None, env_user_spec: str = '', no_shuffle: bool = False, @@ -270,21 +272,30 @@ async def generate_predictions( from ..agent.loop import Environment, Usage as _U # noqa: F401 async with sem: - env = env_factory() - if type(env).run_task is not Environment.run_task: - # self-running env (official engine bundles: tau2/swe) - pred = await env.run_task(adapter, sample, - max_turns=max_turns, system=system, - user_adapter=_env_user_adapter(env_user_spec) if env_user_spec else None, - gen_kwargs=gen_kwargs) - if pred is None: + if progress_reporter is not None: + # begin AFTER acquiring the slot: "in flight" must mean + # actually generating, not queued on the semaphore + progress_reporter.begin_sample(f'sample {sample.id}') + try: + env = env_factory() + if type(env).run_task is not Environment.run_task: + # self-running env (official engine bundles: tau2/swe) + pred = await env.run_task(adapter, sample, + max_turns=max_turns, system=system, + user_adapter=_env_user_adapter(env_user_spec) if env_user_spec else None, + gen_kwargs=gen_kwargs) + if pred is None: + traj = await drive(adapter, sample, env=env, + max_turns=max_turns, system=system) + pred = trajectory_to_prediction(traj) + else: traj = await drive(adapter, sample, env=env, max_turns=max_turns, system=system) pred = trajectory_to_prediction(traj) - else: - traj = await drive(adapter, sample, env=env, - max_turns=max_turns, system=system) - pred = trajectory_to_prediction(traj) + except Exception: + if progress_reporter is not None: + progress_reporter.rollback() + raise if not pred.get('usage'): pred['usage'] = traj.usage.model_dump() if 'traj' in dir() else {} pred.setdefault('group_key', str(sample.metadata.get('test_category') @@ -298,7 +309,10 @@ async def generate_predictions( total_tokens=int(u.get('total_tokens', 0) or 0), latency_s=float(u.get('latency_s', 0) or 0)) done_count += 1 - _progress(progress, done_count, len(samples), t0, total_usage) + if progress_reporter is not None: + progress_reporter.advance(success=True) + else: + _progress(progress, done_count, len(samples), t0, total_usage) return pred messages = ([ChatMessage(role='user', content=assemble(sample))] @@ -320,7 +334,16 @@ async def generate_predictions( messages = messages + [ChatMessage(role='user', content=f'MOCKTARGET::{sample.target}')] async with sem: - out = await adapter.generate(messages, tools=tools, **gen_kwargs) + if progress_reporter is not None: + progress_reporter.begin_sample(f'sample {sample.id}') + try: + out = await adapter.generate(messages, tools=tools, **gen_kwargs) + except Exception: + # retry path re-enters one() and begins again: pair this + # begin here or the in-flight count leaks upward + if progress_reporter is not None: + progress_reporter.rollback() + raise total_usage = total_usage + out.usage text = out.text if out.tool_calls: # fc tasks: serialize calls as the prediction @@ -329,10 +352,15 @@ async def generate_predictions( text = (text + '\n' if text else '') + json.dumps( [c.to_openai()['function'] for c in out.tool_calls], ensure_ascii=False) done_count += 1 - _progress(progress, done_count, len(samples), t0, total_usage) + if progress_reporter is not None: + progress_reporter.advance(success=True) + else: + _progress(progress, done_count, len(samples), t0, total_usage) return {'raw': text, 'usage': out.usage.model_dump()} work = _apply_limits(samples, limit, limit_per_task, shuffle=not no_shuffle) + if status_callback: + status_callback(f'preparing {len(work)} evaluation samples') # checkpointing: restore completed samples, generate only the rest ckpt_store = None if checkpoint: @@ -369,6 +397,11 @@ async def generate_predictions( if ckpt_store is not None and restored: print(f'checkpoint: restored {len(restored)} predictions ' f'({len(pending)} to generate) -> {ckpt_store.path}', flush=True) + if status_callback: + status_callback(f'checkpoint restored: {len(restored)} ready, {len(pending)} pending') + + if progress_reporter is not None: + progress_reporter.start(len(work), dataset_name, completed=len(restored)) async def run_one(i_s): i, s = i_s @@ -382,6 +415,8 @@ async def generate_predictions( break except Exception: if attempt == 5: + if progress_reporter is not None: + progress_reporter.advance(success=False) raise # minute-scale backoff: cluster routes flap in multi-minute # bursts; short retries exhaust inside one dead window @@ -390,12 +425,20 @@ async def generate_predictions( ckpt_store.append(keys[i], pred) return i, pred - fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending)) - for i, pred in fresh: - preds_by_key[keys[i]] = pred - preds = [preds_by_key[k] for k in keys] - usages = [p.get('usage', {}) for p in preds] - return preds, usages, total_usage + try: + if status_callback: + status_callback(f'generating model responses: {len(pending)} pending') + fresh = await asyncio.gather(*(run_one((i, s)) for i, s in pending)) + for i, pred in fresh: + preds_by_key[keys[i]] = pred + preds = [preds_by_key[k] for k in keys] + usages = [p.get('usage', {}) for p in preds] + if status_callback: + status_callback(f'generation complete: {len(preds)} responses') + return preds, usages, total_usage + finally: + if progress_reporter is not None: + progress_reporter.close() def _apply_limits(samples: List[Sample], total: Optional[int], @@ -444,7 +487,8 @@ def _apply_limits(samples: List[Sample], total: Optional[int], def _progress(progress: bool, done: int, total: int, t0: float, usage: Usage) -> None: - if progress and (done % 20 == 0 or done == total): + interval = max(1, min(20, total)) + if progress and (done % interval == 0 or done == total): rate = done / max(time.time() - t0, 1e-6) print(f' [{done}/{total}] {rate:.1f} samples/s tokens={usage.total_tokens}', flush=True) @@ -460,6 +504,8 @@ async def run_eval( judge_spec: Optional[str] = None, judge: Optional[Any] = None, progress: bool = True, + progress_reporter=None, + status_callback=None, env: str = '', env_user_spec: str = '', no_shuffle: bool = False, # fixed-order selection: raw first-N (same-questions parity) @@ -489,6 +535,9 @@ async def run_eval( if few_shot_num < 0: few_shot_num = (spec.few_shot_num if spec is not None else 0) adapter = _make_adapter(model_spec) + # reports carry a string model label: pre-built adapter objects need one + model_spec = model_spec if isinstance(model_spec, str) \ + else (getattr(model_spec, 'model', '') or repr(model_spec)) name = spec.name if spec is not None else 'adhoc' if recipe is None: from ..eval.recipe import EvalRecipe, get_eval @@ -502,7 +551,11 @@ async def run_eval( scorers={'acc': {'name': 'exact', 'mode': 'raw'}}) # materialize in a worker thread: hub downloads here are synchronous # (requests/ssl) and would otherwise stall the whole event loop + if status_callback: + status_callback('materializing dataset: cache check/download and parsing') raw_samples = await asyncio.to_thread(lambda: list(dataset)) + if status_callback: + status_callback(f'dataset materialized: {len(raw_samples)} samples') if limit: raw_samples = raw_samples[:limit] # generate_predictions applies the SAME deterministic limiting internally; @@ -532,6 +585,8 @@ async def run_eval( few_shot_samples = None few_shot_text = None if few_shot_num: + if status_callback: + status_callback(f'loading few-shot examples: {few_shot_num}') from ..data.registry import get_dataset_provider prov = get_dataset_provider(name) @@ -574,6 +629,8 @@ async def run_eval( from .gen_profiles import merge_gen_kwargs preds, _usages, usage = await generate_predictions( adapter, list(raw_samples), concurrency, progress=progress, + progress_reporter=progress_reporter, + status_callback=status_callback, gen_kwargs=merge_gen_kwargs(name, spec, gen_kwargs, gen_profile), env_factory=env_factory, env_user_spec=env_user_spec, @@ -590,9 +647,13 @@ async def run_eval( finally: await adapter.close() if judge is None and judge_spec: + if status_callback: + status_callback('loading judge model') judge_adapter = _make_adapter(judge_spec) judge = _judge_callable(judge_adapter) + if status_callback: + status_callback('scoring predictions') report = evaluate( samples, preds, recipe, model=model_spec, @@ -603,6 +664,8 @@ async def run_eval( ) report.model = model_spec report.dataset = name + if status_callback: + status_callback('scoring complete') # performance profile: pool success rate + latency/ttft percentiles try: from .aggregator import get_aggregator @@ -640,7 +703,10 @@ def _make_adapter(spec: str) -> ModelAdapter: round-robin counter stays global (independent pools would each restart at the first backend and starve the rest). """ - from .adapter import _ADAPTER_CACHE as _CACHE + from .adapter import _ADAPTER_CACHE as _CACHE, ModelAdapter + + if isinstance(spec, ModelAdapter): # pre-built adapter (tests/custom) + return spec cache_key = spec if cache_key in _CACHE: diff --git a/evalharness/progress/__init__.py b/evalharness/progress/__init__.py new file mode 100644 index 0000000..f0c9c72 --- /dev/null +++ b/evalharness/progress/__init__.py @@ -0,0 +1,12 @@ +"""Optional runtime progress renderers. + +Degrades to None when rich is absent so callers fall back to plain-text +progress (the CLI stays dependency-free in its fallback path). +""" + +try: + from .rich_terminal import RichTerminalProgress +except ImportError: # rich not installed + RichTerminalProgress = None + +__all__ = ["RichTerminalProgress"] diff --git a/evalharness/progress/rich_terminal.py b/evalharness/progress/rich_terminal.py new file mode 100644 index 0000000..813793e --- /dev/null +++ b/evalharness/progress/rich_terminal.py @@ -0,0 +1,122 @@ +"""Rich terminal progress reporter for per-sample model generation.""" + +import asyncio +import time + +from rich.console import Console +from rich.progress import ( + BarColumn, + Progress, + SpinnerColumn, + TaskProgressColumn, + TextColumn, + TimeElapsedColumn, + TimeRemainingColumn, +) + + +class RichTerminalProgress: + def __init__(self, console=None): + # accept an EXTERNAL console: CLI phase messages and the live bar must + # share one console, or the two writers interleave and repaint wrongly + self.console = console or Console() + self.progress = Progress( + SpinnerColumn(), + TextColumn("[progress.description]{task.description}"), + BarColumn(complete_style="green", finished_style="bold green"), + TaskProgressColumn(), + TextColumn("• Completed {task.completed}/{task.total}"), + TextColumn("• In flight {task.fields[inflight]}"), + TextColumn("• Waiting {task.fields[waiting]}"), + TextColumn("• Last {task.fields[last_result]}"), + TextColumn("• {task.fields[rate]} sample/s"), + TextColumn("•"), + TimeElapsedColumn(), + TextColumn("•"), + TimeRemainingColumn(), + console=self.console, + refresh_per_second=4, + ) + self.task_id = None + self.started = 0.0 + self.current_started = 0.0 + self.inflight = 0 + self.heartbeat_task = None + + def start(self, total: int, description: str, completed: int = 0): + self.started = time.monotonic() + self.progress.start() + self.task_id = self.progress.add_task( + f"[green]{description}", + total=total, + completed=min(completed, total), + success=completed, + failed=0, + rate="0.00", + inflight=0, + waiting="00:00", + last_result="restored", + ) + self.heartbeat_task = asyncio.create_task(self._heartbeat()) + + def begin_sample(self, label: str): + if self.task_id is None: + return + self.inflight += 1 + self.current_started = time.monotonic() + self.progress.update(self.task_id, inflight=self.inflight, + waiting="00:00", last_result=f"waiting {label}") + + def rollback(self): + """Pair a begin_sample that will NOT reach advance (retry path): + just decrement the in-flight count, no success/fail bookkeeping.""" + self.inflight = max(0, self.inflight - 1) + if self.task_id is not None: + self.progress.update(self.task_id, inflight=self.inflight) + + def advance(self, success: bool = True): + if self.task_id is None: + return + task = self.progress.tasks[self.task_id] + completed = task.completed + 1 + ok = task.fields["success"] + (1 if success else 0) + failed = task.fields["failed"] + (0 if success else 1) + self.inflight = max(0, self.inflight - 1) + elapsed = max(time.monotonic() - self.started, 1e-6) + self.progress.update( + self.task_id, + advance=1, + success=ok, + failed=failed, + rate=f"{completed / elapsed:.2f}", + inflight=self.inflight, + waiting="00:00", + last_result="success" if success else "failed", + ) + + async def _heartbeat(self): + while self.task_id is not None: + if self.task_id is not None and self.inflight: + waiting = int(time.monotonic() - self.current_started) + self.progress.update(self.task_id, waiting=f"{waiting // 60:02d}:{waiting % 60:02d}") + await asyncio.sleep(1) + + def log(self, message: str): + """Print a status line ABOVE the live bar (safe during live display). + + Plain console.print from another writer while Progress is live causes + interleaved/repainted output; Progress.print routes through the live + region correctly. + """ + if self.task_id is not None: + self.progress.print(message) + else: + self.console.print(message) + + def close(self): + if self.task_id is not None: + if self.heartbeat_task is not None: + self.heartbeat_task.cancel() + self.heartbeat_task = None + self.progress.stop() + self.task_id = None diff --git a/evalharness/third_party/bfcl/__init__.py b/evalharness/third_party/bfcl/__init__.py new file mode 100644 index 0000000..e35e223 --- /dev/null +++ b/evalharness/third_party/bfcl/__init__.py @@ -0,0 +1,8 @@ +"""Vendored BFCL official AST checker (from bfcl-eval, Apache-2.0). + +Source: bfcl_eval/eval_checker/ast_eval/{ast_checker.py,type_convertor/} + + bfcl_eval/constants/type_mappings.py +Only change: imports rerouted locally and the MODEL_CONFIG_MAPPING lookup +replaced by an explicit ``underscore_to_dot`` argument. Upstream license +and notice apply to the files in this directory. +""" diff --git a/evalharness/third_party/bfcl/ast_checker.py b/evalharness/third_party/bfcl/ast_checker.py new file mode 100644 index 0000000..28306c3 --- /dev/null +++ b/evalharness/third_party/bfcl/ast_checker.py @@ -0,0 +1,636 @@ +from .type_mappings import ( + JAVA_TYPE_CONVERSION, + JS_TYPE_CONVERSION, +) +from .type_convertor.java_type_converter import java_type_converter +from .type_convertor.js_type_converter import js_type_converter +import re + +#### Constants #### +PYTHON_TYPE_MAPPING = { + "string": str, + "integer": int, + "float": float, + "boolean": bool, + "array": list, + "tuple": list, + "dict": dict, + "any": str, +} + +# This is the list of types that we need to recursively check its values +PYTHON_NESTED_TYPE_CHECK_LIST = ["array", "tuple"] + + +NESTED_CONVERSION_TYPE_LIST = ["Array", "ArrayList", "array"] + + +#### Main function #### +def ast_checker( + func_description, model_output, possible_answer, language, test_category, model_name, + underscore_to_dot=True, +): + if "parallel" in test_category: + return parallel_function_checker_no_order( + func_description, model_output, possible_answer, language, model_name + ) + + elif "multiple" in test_category: + return multiple_function_checker( + func_description, model_output, possible_answer, language, model_name + ) + + else: + if len(model_output) != 1: + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "simple_function_checker:wrong_count", + } + + return simple_function_checker( + func_description[0], model_output[0], possible_answer[0], language, model_name + ) + + +#### Helper functions for AST #### +def find_description(func_descriptions, name): + if type(func_descriptions) == list: + for func_description in func_descriptions: + if func_description["name"] == name: + return func_description + return None + else: + # it is a dict, there is only one function + return func_descriptions + + +def get_possible_answer_type(possible_answer: list): + for answer in possible_answer: + if answer != "": # Optional parameter + return type(answer) + return None + + +def convert_func_name(function_name, model_name: str): + model_name_escaped = model_name.replace("_", "/") + if "." in function_name: + if underscore_to_dot: + # OAI does not support "." in the function name so we replace it with "_". ^[a-zA-Z0-9_-]{1,64}$ is the regex for the name. + # This happens for OpenAI, Mistral, and Google models + return re.sub(r"\.", "_", function_name) + return function_name + + +def type_checker( + param: str, + value, + possible_answer: list, + expected_type_description: str, + expected_type_converted, + nested_type_converted, +): + # NOTE: This type checker only supports nested type checking for one level deep. + # We didn't implement recursive type checking for nested types, as it's not needed for the current use case and it's very complex. + + result = { + "valid": True, + "error": [], + "is_variable": False, + "error_type": "type_error:simple", + } + + is_variable = False + # check for the case where a variable is used instead of a actual value. + # use the type in possible_answer as the expected type + possible_answer_type = get_possible_answer_type(possible_answer) + # if possible_answer only contains optional parameters, we can't determine the type + if possible_answer_type != None: + # we are being precise here. + # in fact, possible_answer_type should always be string, as that's how we treat varibale in possible_answer + if possible_answer_type != expected_type_converted: + is_variable = True + + # value is the same type as in function description + if type(value) == expected_type_converted: + # We don't need to do recursive check for simple types + if nested_type_converted == None: + result["is_variable"] = is_variable + return result + else: + for possible_answer_item in possible_answer: + flag = True # Each parameter should match to at least one possible answer type. + # Here, we assume that each item should be the same type. We could also relax it. + if type(possible_answer_item) == list: + for value_item in value: + checker_result = type_checker( + param, + value_item, + possible_answer_item, + str(nested_type_converted), + nested_type_converted, + None, + ) + if not checker_result["valid"]: + flag = False + break + + if flag: + return {"valid": True, "error": [], "is_variable": is_variable} + + result["valid"] = False + result["error"] = [ + f"Nested type checking failed for parameter {repr(param)}. Expected outer type {expected_type_description} with inner type {str(nested_type_converted)}. Parameter value: {repr(value)}." + ] + result["error_type"] = "type_error:nested" + + # value is not as expected, check for the case where a variable is used instead of a actual value + # use the type in possible_answer as the expected type + possible_answer_type = get_possible_answer_type(possible_answer) + # if possible_answer only contains optional parameters, we can't determine the type + if possible_answer_type != None: + # we are being precise here. + # in fact, possible_answer_type should always be string, as that's how we treat varibale in possible_answer + if type(value) == possible_answer_type: + result["is_variable"] = True + return result + + result["valid"] = False + result["error"].append( + f"Incorrect type for parameter {repr(param)}. Expected type {expected_type_description}, got {type(value).__name__}. Parameter value: {repr(value)}." + ) + result["error_type"] = "type_error:simple" + return result + + +def standardize_string(input_string: str): + # This function standardizes the string by removing all the spaces, ",./-_*^" punctuation, and converting it to lowercase + # It will also convert all the single quotes to double quotes + # This is used to compare the model output with the possible answers + # We don't want to punish model for answer like April 1, 2024 vs April 1,2024, vs April 1 2024 + regex_string = r"[ \,\.\/\-\_\*\^]" + return re.sub(regex_string, "", input_string).lower().replace("'", '"') + + +def string_checker(param: str, model_output: str, possible_answer: list): + standardize_possible_answer = [] + standardize_model_output = standardize_string(model_output) + for i in range(len(possible_answer)): + if type(possible_answer[i]) == str: + standardize_possible_answer.append(standardize_string(possible_answer[i])) + + if standardize_model_output not in standardize_possible_answer: + return { + "valid": False, + "error": [ + f"Invalid value for parameter {repr(param)}: {repr(model_output)}. Expected one of {possible_answer}. Case insensitive." + ], + "error_type": "value_error:string", + } + + return {"valid": True, "error": []} + + +def list_checker(param: str, model_output: list, possible_answer: list): + # Convert the tuple to a list + + standardize_model_output = list(model_output) + + # If the element in the list is a string, we need to standardize it + for i in range(len(standardize_model_output)): + if type(standardize_model_output[i]) == str: + standardize_model_output[i] = standardize_string(model_output[i]) + + standardize_possible_answer = [] + # We also need to standardize the possible answers + for i in range(len(possible_answer)): + standardize_possible_answer.append([]) + for j in range(len(possible_answer[i])): + if type(possible_answer[i][j]) == str: + standardize_possible_answer[i].append( + standardize_string(possible_answer[i][j]) + ) + else: + standardize_possible_answer[i].append(possible_answer[i][j]) + + if standardize_model_output not in standardize_possible_answer: + return { + "valid": False, + "error": [ + f"Invalid value for parameter {repr(param)}: {repr(model_output)}. Expected one of {possible_answer}." + ], + "error_type": "value_error:list/tuple", + } + + return {"valid": True, "error": []} + + +def dict_checker(param: str, model_output: dict, possible_answers: list): + # This function works for simple dictionaries, but not dictionaries with nested dictionaries. + # The current dataset only contains simple dictionaries, so this is sufficient. + + result = {"valid": False, "error": [], "error_type": "dict_checker:unclear"} + for i in range(len(possible_answers)): + + if possible_answers[i] == "": + continue + + result = {"valid": False, "error": [], "error_type": "dict_checker:unclear"} + + flag = True + + possible_answer = possible_answers[i] + # possible_anwer is a single dictionary + + for key, value in model_output.items(): + if key not in possible_answer: + result["valid"] = False + result["error"].append(f"Unexpected dict key parameter: '{key}'.") + result["error_type"] = "value_error:dict_key" + flag = False + break + + standardize_value = value + # If the value is a string, we need to standardize it + if type(value) == str: + standardize_value = standardize_string(value) + + # We also need to standardize the possible answers if they are string + standardize_possible_answer = [] + for i in range(len(possible_answer[key])): + if type(possible_answer[key][i]) == str: + standardize_possible_answer.append( + standardize_string(possible_answer[key][i]) + ) + else: + standardize_possible_answer.append(possible_answer[key][i]) + + if standardize_value not in standardize_possible_answer: + result["valid"] = False + result["error"].append( + f"Invalid value for parameter {repr(key)}: {repr(value)}. Expected one of {standardize_possible_answer}." + ) + result["error_type"] = "value_error:dict_value" + flag = False + break + + for key, value in possible_answer.items(): + if key not in model_output and "" not in value: + result["valid"] = False + result["error"].append(f"Missing dict key parameter: '{key}'.") + result["error_type"] = "value_error:dict_key" + flag = False + break + + if flag: + return {"valid": True, "error": []} + + return result + + +def list_dict_checker(param: str, model_output: list, possible_answers: list): + # This function takes in a list of dictionaries and checks if each dictionary is valid + # The order of the dictionaries in the list must match the order of the possible answers + + result = {"valid": False, "error": [], "error_type": "list_dict_checker:unclear"} + + for answer_index in range(len(possible_answers)): + flag = True # True means so far, all dictionaries are valid + + # Only proceed if the number of dictionaries in the list matches the number of dictionaries in the possible answers + if len(model_output) != len(possible_answers[answer_index]): + result["valid"] = False + result["error"] = ["Wrong number of dictionaries in the list."] + result["error_type"] = "value_error:list_dict_count" + flag = False + continue + + for dict_index in range(len(model_output)): + result = dict_checker( + param, + model_output[dict_index], + [possible_answers[answer_index][dict_index]], + ) + if not result["valid"]: + flag = False + break + if flag: + return {"valid": True, "error": []} + + return result + + +def simple_function_checker( + func_description: dict, + model_output: dict, + possible_answer: dict, + language: str, + model_name: str, +): + possible_answer = list(possible_answer.values())[0] + # Extract function name and parameters details + func_name = func_description["name"] + param_details = func_description["parameters"]["properties"] + required_params = func_description["parameters"]["required"] + + # Initialize a result dictionary + result = { + "valid": True, + "error": [], + "error_type": "simple_function_checker:unclear", + } + + func_name = convert_func_name(func_name, model_name) + + # Check if function name matches + if func_name not in model_output: + result["valid"] = False + result["error"].append( + f"Function name {repr(func_name)} not found in model output." + ) + result["error_type"] = "simple_function_checker:wrong_func_name" + return result + + model_params = model_output[func_name] + + # Check for required parameters in model output + for param in required_params: + if param not in model_params: + result["valid"] = False + result["error"].append(f"Missing required parameter: {repr(param)}.") + result["error_type"] = "simple_function_checker:missing_required" + return result + + # Validate types and values for each parameter in model output + for param, value in model_params.items(): + if param not in param_details or param not in possible_answer: + result["valid"] = False + result["error"].append(f"Unexpected parameter: {repr(param)}.") + result["error_type"] = "simple_function_checker:unexpected_param" + return result + + full_param_details = param_details[param] + expected_type_description = full_param_details["type"] # This is a string + is_variable = False + nested_type_converted = None + + if language == "Java": + expected_type_converted = JAVA_TYPE_CONVERSION[expected_type_description] + + if expected_type_description in JAVA_TYPE_CONVERSION: + if type(value) != str: + result["valid"] = False + result["error"].append( + f"Incorrect type for parameter {repr(param)}. Expected type String, got {type(value).__name__}. Parameter value: {repr(value)}." + ) + result["error_type"] = "type_error:java" + return result + + if expected_type_description in NESTED_CONVERSION_TYPE_LIST: + nested_type = param_details[param]["items"]["type"] + nested_type_converted = JAVA_TYPE_CONVERSION[nested_type] + value = java_type_converter( + value, expected_type_description, nested_type + ) + else: + value = java_type_converter(value, expected_type_description) + + elif language == "JavaScript": + expected_type_converted = JS_TYPE_CONVERSION[expected_type_description] + + if expected_type_description in JS_TYPE_CONVERSION: + if type(value) != str: + result["valid"] = False + result["error"].append( + f"Incorrect type for parameter {repr(param)}. Expected type String, got {type(value).__name__}. Parameter value: {repr(value)}." + ) + result["error_type"] = "type_error:js" + return result + + if expected_type_description in NESTED_CONVERSION_TYPE_LIST: + nested_type = param_details[param]["items"]["type"] + nested_type_converted = JS_TYPE_CONVERSION[nested_type] + value = js_type_converter( + value, expected_type_description, nested_type + ) + else: + value = js_type_converter(value, expected_type_description) + + elif language == "Python": + expected_type_converted = PYTHON_TYPE_MAPPING[expected_type_description] + if expected_type_description in PYTHON_NESTED_TYPE_CHECK_LIST: + nested_type = param_details[param]["items"]["type"] + nested_type_converted = PYTHON_TYPE_MAPPING[nested_type] + + # We convert all tuple value to list when the expected type is tuple. + # The conversion is necessary because any tuple in the possible answer would become a list after being processed through json.dump() and json.load(). + # This does introduce some false positive (eg, when the model provides a list value instead of tuple). We hope to find a better solution in the future. + if expected_type_description == "tuple" and type(value) == tuple: + value = list(value) + + # Allow python auto conversion from int to float + if ( + language == "Python" + and expected_type_description == "float" + and type(value) == int + ): + value = float(value) + + # Type checking + # In fact, we only check for Python here. + # Type check for other languages are handled by the type converter, and so their value (after conversion) is always correct. + type_check_result = type_checker( + param, + value, + possible_answer[param], + expected_type_description, + expected_type_converted, + nested_type_converted, + ) + is_variable = type_check_result["is_variable"] + if not type_check_result["valid"]: + return type_check_result + + # It doesn't make sense to special handle dictionaries and list of dictionaries if the value is a variable. + # We can just treat the variable as a string and use the normal flow. + if not is_variable: + # Special handle for dictionaries + if expected_type_converted == dict: + result = dict_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + # Special handle for list of dictionaries + elif expected_type_converted == list and nested_type_converted == dict: + result = list_dict_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + # Special handle for strings + elif expected_type_converted == str: + # We don't check for case sensitivity for string, as long as it's not a variable + result = string_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + elif expected_type_converted == list: + result = list_checker(param, value, possible_answer[param]) + if not result["valid"]: + return result + continue + + # Check if the value is within the possible answers + if value not in possible_answer[param]: + result["valid"] = False + result["error"].append( + f"Invalid value for parameter {repr(param)}: {repr(value)}. Expected one of {possible_answer[param]}." + ) + result["error_type"] = "value_error:others" + return result + + # Check for optional parameters not provided but allowed + for param in possible_answer: + if param not in model_params and "" not in possible_answer[param]: + result["valid"] = False + result["error"].append( + f"Optional parameter {repr(param)} not provided and not marked as optional." + ) + result["error_type"] = "simple_function_checker:missing_optional" + return result + + return result + + +def parallel_function_checker_enforce_order( + func_descriptions: list, + model_output: list, + possible_answers: dict, + language: str, + model_name: str, +): + if len(model_output) != len(possible_answers): + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "parallel_function_checker_enforce_order:wrong_count", + } + + func_name_list = list(possible_answers.keys()) + possible_answers_list = [] + + for key, value in possible_answers.items(): + possible_answers_list.append({key: value}) + + for i in range(len(possible_answers_list)): + func_description = find_description(func_descriptions, func_name_list[i]) + + result = simple_function_checker( + func_description, + model_output[i], + possible_answers_list[i], + language, + model_name, + ) + if not result["valid"]: + return result + + return {"valid": True, "error": []} + + +def parallel_function_checker_no_order( + func_descriptions: list, + model_output: list, + possible_answers: list, + language: str, + model_name: str, +): + if len(model_output) != len(possible_answers): + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "parallel_function_checker_no_order:wrong_count", + } + + matched_indices = [] + + # We go throught the possible answers one by one, and eliminate the model output that matches the possible answer + # It must be this way because we need ground truth to fetch the correct function description + for i in range(len(possible_answers)): + # possible_answers[i] is a dictionary with only one key + func_name_expected = list(possible_answers[i].keys())[0] + func_description = find_description(func_descriptions, func_name_expected) + + + all_errors = [] + + for index in range(len(model_output)): + if index in matched_indices: + continue + + result = simple_function_checker( + func_description, + model_output[index], + possible_answers[i], + language, + model_name, + ) + + if result["valid"]: + matched_indices.append(index) + break + else: + all_errors.append( + { + f"Model Result Index {index}": { + "sub_error": result["error"], + "sub_error_type": result["error_type"], + "model_output_item": model_output[index], + "possible_answer_item": possible_answers[i], + } + } + ) + + if not result["valid"]: + considered_indices = [ + i for i in range(len(model_output)) if i not in matched_indices + ] + all_errors.insert( + 0, + f"Could not find a matching function among index {considered_indices} of model output for index {i} of possible answers.", + ) + return { + "valid": False, + "error": all_errors, + "error_type": "parallel_function_checker_no_order:cannot_find_match", + } + + return {"valid": True, "error": []} + + +def multiple_function_checker( + func_descriptions: list, + model_output: list, + possible_answers: list, + language: str, + model_name: str, +): + if len(model_output) != len(possible_answers): + return { + "valid": False, + "error": ["Wrong number of functions."], + "error_type": "multiple_function_checker:wrong_count", + } + + # possible_answers is a list of only one dictionary with only one key + func_name_expected = list(possible_answers[0].keys())[0] + func_description = find_description(func_descriptions, func_name_expected) + return simple_function_checker( + func_description, + model_output[0], + possible_answers[0], + language, + model_name, + ) diff --git a/evalharness/third_party/bfcl/type_convertor/__init__.py b/evalharness/third_party/bfcl/type_convertor/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/evalharness/third_party/bfcl/type_convertor/java_type_converter.py b/evalharness/third_party/bfcl/type_convertor/java_type_converter.py new file mode 100644 index 0000000..c281786 --- /dev/null +++ b/evalharness/third_party/bfcl/type_convertor/java_type_converter.py @@ -0,0 +1,407 @@ +import re +from typing import List, Dict, Union +from ..type_mappings import JAVA_TYPE_CONVERSION + + +def java_type_converter(value, expected_type, nested_type=None): + if expected_type not in JAVA_TYPE_CONVERSION: + raise ValueError(f"Unsupported type: {expected_type}") + if ( + expected_type == "byte" + or expected_type == "short" + or expected_type == "integer" + ): + if not re.match(r"^-?\d+$", value): + return str(value) # default to string + return int(value) + elif expected_type == "float": + if not re.match(r"^-?\d+(\.\d+)?([eE][+-]?\d+)?[fF]$", value): + return str(value) # default to string + return float(re.sub(r"[fF]$", "", value)) + elif expected_type == "double": + if not re.match(r"^-?\d+(\.\d+)?([eE][+-]?\d+)?$", value): + return str(value) # default to string + return float(value) + elif expected_type == "long": + if not re.match(r"^-?\d+[lL]$", value): + return str(value) # default to string + return int(re.sub(r"[lL]$", "", value)) + elif expected_type == "boolean": + if value not in ["true", "false"]: + return str(value) # default to string + return parse_java_boolean(value) + elif expected_type == "char": + if not re.match(r"^\'.$\'", value): + return str(value) # default to string + return value # Remove the single quotes + elif expected_type == "Array" or expected_type == "ArrayList": + return parse_java_collection(value, expected_type, nested_type) + elif expected_type == "Set": + raise NotImplementedError("Set conversion is not implemented") + elif expected_type == "HashMap": + return parse_java_collection(value, expected_type, nested_type) + elif expected_type == "Hashtable": + raise NotImplementedError("Set conversion is not implemented") + elif expected_type == "Queue" or expected_type == "Stack": + raise NotImplementedError(f"{expected_type} conversion is not implemented") + elif expected_type == "String" or expected_type == "any": + return str(value) # we output as string for `any` type + else: + raise ValueError(f"Unsupported type: {expected_type}") + + +def parse_java_boolean(value): + return value == "true" + + +def parse_java_collection( + input_str: str, type_str: str, nested_type=None +) -> Union[List, Dict]: + if type_str == "ArrayList": + return parse_arraylist(input_str, nested_type) + elif type_str == "Array": + return parse_array(input_str, nested_type) + elif type_str == "HashMap": + return parse_hashmap(input_str) + else: + raise ValueError(f"Unsupported type: {type_str}") + + +def parse_arraylist(input_str: str, nested_type=None) -> List: + match_asList = re.search( + r"new\s+ArrayList<\w*>\(Arrays\.asList\((.+?)\)\)", input_str + ) + if match_asList: + elements_str = match_asList.group(1) + elements = [] + for element_str in elements_str.split(","): + element_str = element_str.strip() + if nested_type == "char": + element = element_str[1:-1] # Remove the single quotes + elif nested_type == "String": + element = element_str[1:-1] # Remove the double quotes + else: + element = ( + java_type_converter(element_str, nested_type) + if nested_type + else parse_java_value(element_str) + ) + elements.append(element) + return elements + + match_add = re.search( + r"new\s+ArrayList<\w*>\(\)\s*\{\{\s*(.+?)\s*\}\}", input_str, re.DOTALL + ) + if match_add: + adds_str = match_add.group(1) + elements = [] + matches = re.findall(r"add\((.+?)\)", adds_str) + for match in matches: + value_str = match.strip() + if nested_type == "char": + value = value_str[1:-1] # Remove the single quotes + elif nested_type == "String": + value = value_str[1:-1] # Remove the double quotes + else: + value = ( + java_type_converter(value_str, nested_type) + if nested_type + else parse_java_value(value_str) + ) + elements.append(value) + return elements + + match_empty = re.search(r"new\s+ArrayList<\w*>\(\)", input_str) + if match_empty: + return [] # Return an empty list for an empty ArrayList + + return input_str # default to string + + +def parse_array(input_str: str, nested_type=None) -> List: + match = re.search(r"new\s+\w+\[\]\s*\{(.*?)\}", input_str) + if match: + elements_str = match.group(1) + if nested_type: + elements = [ + java_type_converter(x.strip(), nested_type) + for x in elements_str.split(",") + if x.strip() + ] + else: + elements = [ + parse_java_value(x.strip()) + for x in elements_str.split(",") + if x.strip() + ] + + return elements + else: + return input_str # default to string + + +def parse_hashmap(input_str: str) -> Dict: + elements = {} + match = re.search( + r"new\s+HashMap<.*?>\s*\(\)\s*\{\s*\{?\s*(.*?)\s*\}?\s*\}", input_str, re.DOTALL + ) + if match: + puts_str = match.group(1) + if puts_str.strip(): + matches = re.findall(r"put\(\"(.*?)\",\s*(.*?)\)", puts_str) + for match in matches: + key = match[0] + value = parse_java_value(match[1].strip()) + elements[key] = value + return elements + + match_empty = re.search(r"new\s+HashMap<.*?>\s*\(\)", input_str) + if match_empty: + return {} # Return an empty dictionary for an empty HashMap + + return input_str # default to string + + +# This method parses without the information of what each element type is, contrary of the previous +def parse_java_value(value_str: str): + # check if it's boolean + if value_str == "true": + return True + elif value_str == "false": + return False + # check if it's a string + elif value_str.startswith('"') and value_str.endswith('"'): + return value_str[1:-1] + # check if it's a long + elif re.match(r"^-?\d+[lL]$", value_str): + return int(value_str[:-1]) + # check if it's a float + elif re.match(r"^-?\d+(\.\d+)?([eE][+-]?\d+)?[fF]$", value_str): + return float(re.sub(r"[fF]$", "", value_str)) + # check if it's a integer-like and float-like types (including byte, short, integer, double, etc) + else: + try: + return int(value_str) + except ValueError: + try: + return float(value_str) + except ValueError: + # this assuming all other types are converted to string + return value_str + + +# Write tests for the `java_type_converter` function +def test_java_type_converter(): + # Test valid conversions + assert java_type_converter("true", "boolean") == True + assert java_type_converter("false", "boolean") == False + assert java_type_converter("123", "integer") == 123 + assert java_type_converter("-123", "integer") == -123 + assert java_type_converter("3.14f", "float") == 3.14 + assert java_type_converter("-3.14f", "float") == -3.14 + assert java_type_converter("3.14", "double") == 3.14 + assert java_type_converter("-3.14", "double") == -3.14 + assert java_type_converter("123L", "long") == 123 + assert java_type_converter("-123L", "long") == -123 + assert java_type_converter("a", "char") == "a" + assert java_type_converter("abc", "String") == "abc" + assert java_type_converter("new int[]{1, 2, 3}", "Array") == [1, 2, 3] + assert java_type_converter( + 'new ArrayList<>(Arrays.asList("a", "b"))', "ArrayList" + ) == ["a", "b"] + assert java_type_converter( + 'new HashMap() {{ put("key", "value"); }}', "HashMap" + ) == {"key": "value"} + assert java_type_converter("3f", "float") == 3.0 + assert java_type_converter("3e3F", "float") == 3e3 + assert java_type_converter("3e-3F", "float") == 3e-3 + assert java_type_converter("3.14e2", "double") == 3.14e2 + assert java_type_converter("3.14e-2", "double") == 3.14e-2 + assert java_type_converter("127", "byte") == 127 + assert java_type_converter("-128", "byte") == -128 + assert java_type_converter("32767", "short") == 32767 + assert java_type_converter("-32768", "short") == -32768 + assert java_type_converter("9223372036854775807L", "long") == 9223372036854775807 + assert java_type_converter("-9223372036854775808L", "long") == -9223372036854775808 + assert java_type_converter("123", "any") == "123" + assert java_type_converter("abc", "any") == "abc" + + # Test empty collections + assert java_type_converter("new int[]{}", "Array") == [] + assert java_type_converter("new ArrayList<>()", "ArrayList") == [] + assert java_type_converter("new HashMap<>()", "HashMap") == {} + + # Test collections with mixed types + assert java_type_converter('new Object[]{1, "abc", true}', "Array") == [ + 1, + "abc", + True, + ] + assert java_type_converter( + 'new ArrayList<>(Arrays.asList(1, "abc", true))', "ArrayList" + ) == [1, "abc", True] + assert java_type_converter( + 'new HashMap() {{ put("key1", 1); put("key2", "value"); put("key3", true); }}', + "HashMap", + ) == {"key1": 1, "key2": "value", "key3": True} + + # Test invalid values + try: + java_type_converter("true", "integer") + except ValueError as e: + assert str(e) == "Invalid integer value: true" + + try: + java_type_converter("abc", "integer") + except ValueError as e: + assert str(e) == "Invalid integer value: abc" + + try: + java_type_converter("abc", "long") + except ValueError as e: + assert str(e) == "Invalid long value: abc" + + try: + java_type_converter("3.14", "float") + except ValueError as e: + assert str(e) == "Invalid float value: 3.14" + + try: + java_type_converter("3.14f", "double") + except ValueError as e: + assert str(e) == "Invalid double value: 3.14f" + + try: + java_type_converter("128", "byte") + except ValueError as e: + assert str(e) == "Invalid byte value: 128" + + try: + java_type_converter("32768", "short") + except ValueError as e: + assert str(e) == "Invalid short value: 32768" + + try: + java_type_converter("invalid", "boolean") + except ValueError as e: + assert str(e) == "Invalid boolean value: invalid" + + try: + java_type_converter("abc", "char") + except ValueError as e: + assert str(e) == "Invalid char value: abc" + + # Test unsupported types + try: + java_type_converter("abc", "Set") + except NotImplementedError as e: + assert str(e) == "Set conversion is not implemented" + + try: + java_type_converter("abc", "Hashtable") + except NotImplementedError as e: + assert str(e) == "Set conversion is not implemented" + + try: + java_type_converter("abc", "Queue") + except NotImplementedError as e: + assert str(e) == "Queue conversion is not implemented" + + try: + java_type_converter("abc", "Stack") + except NotImplementedError as e: + assert str(e) == "Stack conversion is not implemented" + + # extra array testing + assert java_type_converter("new int[]{}", "Array") == [] + assert java_type_converter("new int[] {}", "Array") == [] + assert java_type_converter("new int[] { }", "Array") == [] + assert java_type_converter("new int[]{1,2,3}", "Array") == [1, 2, 3] + assert java_type_converter("new int[]{1, 2, 3}", "Array") == [1, 2, 3] + assert java_type_converter("new int[] {1, 2, 3}", "Array") == [1, 2, 3] + assert java_type_converter("new int[] { 1, 2, 3 }", "Array") == [1, 2, 3] + + # extra hashmap testing + assert java_type_converter("new HashMap<>()", "HashMap") == {} + assert java_type_converter("new HashMap<>() {}", "HashMap") == {} + assert java_type_converter("new HashMap<>() {{}}", "HashMap") == {} + assert java_type_converter("new HashMap<>() {{ }}", "HashMap") == {} + assert java_type_converter( + 'new HashMap() {{ put("key", "value"); }}', "HashMap" + ) == {"key": "value"} + assert java_type_converter( + 'new HashMap() {{put("key", "value");}}', "HashMap" + ) == {"key": "value"} + assert java_type_converter( + 'new HashMap() { { put("key", "value"); } }', "HashMap" + ) == {"key": "value"} + assert java_type_converter( + 'new HashMap() {{ put("key1", 123); put("key2", true); }}', + "HashMap", + ) == {"key1": 123, "key2": True} + assert java_type_converter( + 'new HashMap() {{ put("key1", "value 1"); put("key2", "value 2"); }}', + "HashMap", + ) == {"key1": "value 1", "key2": "value 2"} + + def test_parse_array_long(): + input_str = "new long[]{1L, 2L, 3L}" + expected_output = [1, 2, 3] + assert parse_array(input_str, nested_type="long") == expected_output + + def test_parse_array_mixed_long(): + input_str = "new long[]{1L, 2, 3L}" + expected_output = [1, "2", 3] + assert parse_array(input_str, nested_type="long") == expected_output + + def test_parse_array_invalid_long(): + input_str = "new long[]{1L, 2.0, 3L}" + expected_output = [1, "2.0", 3] + assert parse_array(input_str, nested_type="long") == expected_output + + def test_parse_arraylist_int(): + input_str = "new ArrayList(Arrays.asList(1, 2, 3))" + expected_output = [1, 2, 3] + assert parse_arraylist(input_str, nested_type="integer") == expected_output + + def test_parse_arraylist_float(): + input_str = "new ArrayList() {{ add(1.0f); add(2.0f); add(3.0f); }}" + expected_output = [1.0, 2.0, 3.0] + assert parse_arraylist(input_str, nested_type="float") == expected_output + + def test_parse_arraylist_double(): + input_str = "new ArrayList() {{ add(1.0); add(2.0); add(3.0); }}" + expected_output = [1.0, 2.0, 3.0] + assert parse_arraylist(input_str, nested_type="double") == expected_output + + def test_parse_arraylist_boolean(): + input_str = "new ArrayList(Arrays.asList(true, false, true))" + expected_output = [True, False, True] + assert parse_arraylist(input_str, nested_type="boolean") == expected_output + + def test_parse_arraylist_char(): + input_str = "new ArrayList() {{ add('a'); add('b'); add('c'); }}" + expected_output = ["a", "b", "c"] + print(parse_arraylist(input_str, nested_type="char")) + assert parse_arraylist(input_str, nested_type="char") == expected_output + + def test_parse_arraylist_string(): + input_str = 'new ArrayList() {{ add("aasdasd"); add("basdasd"); add("casdasd"); }}' + expected_output = ["aasdasd", "basdasd", "casdasd"] + print(parse_arraylist(input_str)) + assert parse_arraylist(input_str) == expected_output + + test_parse_array_long() + test_parse_array_mixed_long() + test_parse_array_invalid_long() + test_parse_arraylist_int() + test_parse_arraylist_float() + test_parse_arraylist_double() + test_parse_arraylist_boolean() + test_parse_arraylist_char() + test_parse_arraylist_string() + print("All tests passed successfully!") + + +if __name__ == "__main__": + test_java_type_converter() diff --git a/evalharness/third_party/bfcl/type_convertor/js_type_converter.py b/evalharness/third_party/bfcl/type_convertor/js_type_converter.py new file mode 100644 index 0000000..d220468 --- /dev/null +++ b/evalharness/third_party/bfcl/type_convertor/js_type_converter.py @@ -0,0 +1,311 @@ +import re +from ..type_mappings import JS_TYPE_CONVERSION + + +def js_type_converter(value, expected_type, nested_type=None): + if expected_type not in JS_TYPE_CONVERSION: + raise ValueError(f"Unsupported type: {expected_type}") + + if expected_type == "String": + if not (value.startswith('"') and value.endswith('"')) and not ( + value.startswith("'") and value.endswith("'") + ): + return str(value) + return value[1:-1] + + elif expected_type == "integer": + if not re.match(r"^-?\d+$", value): + return str(value) # default to string + return int(value) + elif expected_type == "float": + if not re.match(r"^-?\d+(\.\d+)?$", value): + return str(value) # default to string + return float(value) + elif expected_type == "Bigint": + if not re.match(r"^-?\d+n$", value): + return str(value) # default to string + return int(value[:-1]) + elif expected_type == "Boolean": + if value not in ["true", "false"]: + return str(value) # default to string + return value == "true" + elif expected_type == "dict": + return parse_js_collection(value, "dict", nested_type) + elif expected_type == "array": + return parse_js_collection(value, "array", nested_type) + elif expected_type == "any": + return str(value) + else: + raise ValueError(f"Unsupported type: {expected_type}") + + +def parse_js_collection(code, type_str, nested_type=None): + code = code.strip() + if type_str == "array": + # Regular expression patterns + array_2d_pattern = r"\[\s*\[.*?\]\s*(,\s*\[.*?\]\s*)*\]|\bnew\s+Array\(\s*\[.*?\]\s*(,\s*\[.*?\]\s*)*\)" + array_pattern = r"\[(.*?)\]|\bnew\s+Array\((.*?)\)" + + # Check if the code is a 2D array + array_2d_match = re.match(array_2d_pattern, code) + try: + if array_2d_match: + elements_str = array_2d_match.group(0) + inner_arrays = re.findall(r"\[(.*?)\]", elements_str) + elements = [] + for idx, inner_array_str in enumerate(inner_arrays): + inner_array_str = inner_array_str.strip() + if idx == 0 and inner_array_str.startswith("["): + inner_array_str = inner_array_str[1:] + inner_array_elements = [ + e.strip() for e in inner_array_str.split(",") + ] + if nested_type: + inner_array = [parse_js_value(e) for e in inner_array_elements] + else: + inner_array = [parse_js_value(e) for e in inner_array_elements] + elements.append(inner_array) + return elements + + # Check if the code is a 1D array + array_match = re.match(array_pattern, code) + if array_match: + if array_match.group(1) is not None: + elements_str = array_match.group(1).strip() + if elements_str: + elements = elements_str.split(",") + else: + elements = [] + elif array_match.group(2) is not None: + elements_str = array_match.group(2).strip() + if elements_str: + elements = elements_str.split(",") + else: + elements = [] + else: + elements = [] + if nested_type: + elements = [ + ( + js_type_converter(e.strip(), nested_type, "String") + if (e.strip().startswith("'") or e.strip().startswith('"')) + else js_type_converter(e.strip(), nested_type) + ) + for e in elements + ] + else: + elements = [parse_js_value(e.strip()) for e in elements] + return elements + else: + return code + except: + return code + + elif type_str == "dict": + if code == "{}": + return {} # Return an empty dictionary for an empty object + dict_pattern = r"\{(.*?)\}" + # Check if the code is a dictionary + dict_match = re.match(dict_pattern, code) + if dict_match: + try: + content = dict_match.group(1) + pairs = re.findall(r"([^:]+):\s*(.*?)(?:,\s*(?=[^,]+:)|$)", content) + dictionary = {} + for key, value in pairs: + key = key.strip().strip("'\"") + value = value.strip() + if value.startswith("[") and value.endswith("]"): + # Handle array values + dictionary[key] = parse_js_collection(value, "array") + elif value.startswith("{") and value.endswith("}"): + # Handle nested dictionary values + dictionary[key] = parse_js_collection(value, "dict") + else: + dictionary[key] = parse_js_value(value.strip("'\"")) + return dictionary + except Exception as e: + print(f"Error parsing dictionary: {e}") + return code + else: + return code # default to string + else: + raise ValueError(f"Unsupported type: {type_str}") + + +def parse_js_value(value_str: str): + value_str = value_str.strip() + if value_str == "true": + return True + elif value_str == "false": + return False + elif (value_str.startswith('"') and value_str.endswith('"')) or ( + value_str.startswith("'") and value_str.endswith("'") + ): + return value_str[1:-1] + else: + try: + return int(value_str) + except ValueError: + try: + return float(value_str) + except ValueError: + return value_str + + +# Write tests for the `js_type_converter` function +def test_js_type_converter(): + assert js_type_converter("true", "Boolean") == True + assert js_type_converter("false", "Boolean") == False + assert js_type_converter("123", "integer") == 123 + assert js_type_converter("3.14", "float") == 3.14 + assert js_type_converter("123n", "Bigint") == 123 + assert js_type_converter("abc", "String") == "abc" + assert js_type_converter("[1, 2, 3]", "array") == [1, 2, 3] + assert js_type_converter("new Array(1, 2, 3)", "array") == [1, 2, 3] + assert js_type_converter("{'key': 'value'}", "dict") == {"key": "value"} + assert js_type_converter("{'key': 123}", "dict") == {"key": 123} + assert js_type_converter("{'key': true}", "dict") == {"key": True} + + # Additional test cases + # Test empty array and dictionary + assert js_type_converter("[]", "array") == [] + assert js_type_converter("{}", "dict") == {} + + # Test array with mixed types + assert js_type_converter("[1, 'two', true]", "array") == [1, "two", True] + + # Test dictionary with mixed types + assert js_type_converter( + "{'key1': 123, 'key2': 'value', 'key3': false}", "dict" + ) == {"key1": 123, "key2": "value", "key3": False} + + # Test string with special characters + + # Test negative integer and float values + assert js_type_converter("-123", "integer") == -123 + assert js_type_converter("-3.14", "float") == -3.14 + + # Test invalid type + try: + js_type_converter("123", "InvalidType") + except ValueError as e: + assert str(e) == "Unsupported type: InvalidType" + + # Test invalid integer value + try: + js_type_converter("123.45", "integer") + except ValueError as e: + assert str(e) == "Invalid integer value: 123.45" + + # Test invalid float value + try: + js_type_converter("3.14abc", "float") + except ValueError as e: + assert str(e) == "Invalid float value: 3.14abc" + + # Test invalid Bigint value + try: + js_type_converter("123", "Bigint") + except ValueError as e: + assert str(e) == "Invalid Bigint value: 123" + + # Test invalid boolean value + try: + js_type_converter("not_a_boolean", "Boolean") + except ValueError as e: + assert str(e) == "Invalid boolean value: not_a_boolean" + + print("All tests passed successfully!") + + +def test_js_type_converter_nested_array(): + # Test array with nested integers + assert js_type_converter("[1, 2, 3]", "array", "integer") == [1, 2, 3] + assert js_type_converter("new Array(4, 5, 6)", "array", "integer") == [4, 5, 6] + + # Test array with nested floats + assert js_type_converter("[1.1, 2.2, 3.3]", "array", "float") == [1.1, 2.2, 3.3] + assert js_type_converter("new Array(4.4, 5.5, 6.6)", "array", "float") == [ + 4.4, + 5.5, + 6.6, + ] + + # Test array with nested Bigints + assert js_type_converter("[1n, 2n, 3n]", "array", "Bigint") == [1, 2, 3] + assert js_type_converter("new Array(4n, 5n, 6n)", "array", "Bigint") == [4, 5, 6] + + # Test array with nested booleans + assert js_type_converter("[true, false, true]", "array", "Boolean") == [ + True, + False, + True, + ] + assert js_type_converter("new Array(false, true, false)", "array", "Boolean") == [ + False, + True, + False, + ] + + # Test array with nested strings + print(js_type_converter('["hello", "world", "!"]', "array", "String")) + assert js_type_converter('["hello", "world", "!"]', "array", "String") == [ + "hello", + "world", + "!", + ] + assert js_type_converter('new Array("foo", "bar", "baz")', "array", "String") == [ + "foo", + "bar", + "baz", + ] + + # Test array with mixed nested types + assert js_type_converter('[1, "two", true]', "array") == [1, "two", True] + assert js_type_converter('new Array(3.14, "pi", false)', "array") == [ + 3.14, + "pi", + False, + ] + + # Test array with nested arrays + print(js_type_converter(" [ [1, 2], [3, 4], [5, 6]]", "array", "array")) + assert js_type_converter(" [ [ 1, 2 ], [ 3, 4], [5, 6]]", "array", "array") == [ + [1, 2], + [3, 4], + [5, 6], + ] # this example has many weird spacings + assert js_type_converter("new Array([1, 2], [3, 4], [5, 6])", "array", "array") == [ + [1, 2], + [3, 4], + [5, 6], + ] + + # Test array with nested dictionaries + assert js_type_converter( + '[{"key1": 1}, {"key2": 2}, {"key3": 3}]', "array", "dict" + ) == [{"key1": 1}, {"key2": 2}, {"key3": 3}] + assert js_type_converter( + 'new Array({"key1": 1}, {"key2": 2}, {"key3": 3})', "array", "dict" + ) == [{"key1": 1}, {"key2": 2}, {"key3": 3}] + + print("All nested array tests passed successfully!") + + +def test_js_type_converter_dictionary_with_arrays(): + complex_dict = js_type_converter( + '{"initialState": initialStateObject, "reducers": reducersMap, "middlewares": ["loggerMiddleware"], "enhancers": ["applyMiddleware", "myMiddleWare"]}', + "dict", + ) + assert isinstance(complex_dict, dict) + assert complex_dict["initialState"] == "initialStateObject" + assert complex_dict["reducers"] == "reducersMap" + assert complex_dict["middlewares"] == ["loggerMiddleware"] + assert complex_dict["enhancers"] == ["applyMiddleware", "myMiddleWare"] + print("Complex dictionary test passed successfully!") + +if __name__ == "__main__": + test_js_type_converter() + test_js_type_converter_nested_array() + test_js_type_converter_dictionary_with_arrays() diff --git a/evalharness/third_party/bfcl/type_mappings.py b/evalharness/third_party/bfcl/type_mappings.py new file mode 100644 index 0000000..fdbdde4 --- /dev/null +++ b/evalharness/third_party/bfcl/type_mappings.py @@ -0,0 +1,89 @@ +GORILLA_TO_OPENAPI = { + "integer": "integer", + "number": "number", + "float": "number", + "string": "string", + "boolean": "boolean", + "bool": "boolean", + "array": "array", + "list": "array", + "dict": "object", + "object": "object", + "tuple": "array", + "any": "string", + "byte": "integer", + "short": "integer", + "long": "integer", + "double": "number", + "char": "string", + "ArrayList": "array", + "Array": "array", + "HashMap": "object", + "Hashtable": "object", + "Queue": "array", + "Stack": "array", + "Any": "string", + "String": "string", + "Bigint": "integer", +} + +GORILLA_TO_PYTHON = { + "integer": "int", + "number": "float", + "float": "float", + "string": "str", + "boolean": "bool", + "bool": "bool", + "array": "list", + "list": "list", + "dict": "dict", + "object": "dict", + "tuple": "tuple", + "any": "str", + "byte": "int", + "short": "int", + "long": "int", + "double": "float", + "char": "str", + "ArrayList": "list", + "Array": "list", + "HashMap": "dict", + "Hashtable": "dict", + "Queue": "list", + "Stack": "list", + "Any": "str", + "String": "str", + "Bigint": "int", +} + + +JAVA_TYPE_CONVERSION = { + "byte": int, + "short": int, + "integer": int, + "float": float, + "double": float, + "long": int, + "boolean": bool, + "char": str, + "Array": list, + "ArrayList": list, + "Set": set, + "HashMap": dict, + "Hashtable": dict, + "Queue": list, # this can be `queue.Queue` as well, for simplicity we check with list + "Stack": list, + "String": str, + "any": str, +} + +JS_TYPE_CONVERSION = { + "String": str, + "integer": int, + "float": float, + "Bigint": int, + "Boolean": bool, + "dict": dict, + "array": list, + "any": str, +} diff --git a/evalharness/viz/renderers/excel.py b/evalharness/viz/renderers/excel.py index 3ed5175..f77d851 100644 --- a/evalharness/viz/renderers/excel.py +++ b/evalharness/viz/renderers/excel.py @@ -11,7 +11,7 @@ Sheets: """ import json -from typing import List, Union +from typing import Dict, List, Union from ...eval.record import EvalReport from .. import register_renderer diff --git a/pyproject.toml b/pyproject.toml index 4bec278..06916f3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,11 +15,16 @@ dependencies = [ "pylatexenc", "numpy", # official DROP aligner "scipy", + "rich", + "tree_sitter>=0.21", # vendored BFCL official AST checker (python) + "tree-sitter-java>=0.21", # bfcl java categories + "tree-sitter-javascript>=0.21", # bfcl javascript categories ] [project.optional-dependencies] -bfcl = ["bfcl-eval", "soundfile"] # heavy: official bfcl ast_checker (pulls 5 cloud SDKs - # + qwen-agent); native scorer is the default +# none today: the official BFCL checker is vendored under +# evalharness/third_party/bfcl (Apache-2.0); heavy execution environments +# (humaneval/bigcodebench/swe) live in docker images, never in the venv [project.scripts] evalharness = "evalharness.cli:main"