From ea93602dfa0e1e80029aacd4e353ec07eebcc28e Mon Sep 17 00:00:00 2001 From: sora <2075279110@qq.com> Date: Thu, 10 Sep 2026 06:30:22 +0000 Subject: [PATCH] Unified, nicer result tables + conda env setup in README - cli: rich Run Summary table for multi-benchmark runs (green/red rows, fallback to aligned plain text); unified _fmt_score (fractions render as percentages everywhere -- was 1.0 in summary vs 100.0% in detail); fix the stray "summary csv -> None/viz/..." print without --out-dir; summary.md upgraded to a proper table with model/timestamp/ok-count header -- one table for a whole N-benchmark run - text renderer: single-bench headline deduped (dataset==recipe) and compacted to one facts line; adaptive metric-name column (long names no longer break alignment) - md_compare: auto-switches to one-row-per-benchmark when comparing different benchmarks with different metrics; same-bench model comparison gains baseline delta markers (+/- percentage points) - README: conda create/activate in the install block Co-Authored-By: Claude --- README.md | 16 ++--- evalharness/cli.py | 67 +++++++++++++++++---- evalharness/viz/renderers/text.py | 97 ++++++++++++++++++++++--------- 3 files changed, 129 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index c8a24b2..4fc3c9f 100644 --- a/README.md +++ b/README.md @@ -28,24 +28,17 @@ ## 安装 -要求 Python ≥ 3.10。 +要求 Python ≥ 3.10，一条命令装完即可跑全部 28 个 benchmark，无任何可选依赖 +（代码执行类需要宿主机有 Docker，镜像判分时自动拉取）。 ```bash git clone https://git.meta-stone.net/sora/EvalHarness.git cd EvalHarness +conda create -n evalharness python=3.10 -y +conda activate evalharness pip install . ``` -**一条命令装完即可跑全部 28 个 benchmark，没有任何可选依赖。** - -| 官方判分逻辑形态 | 涉及 benchmark | 位置 | -|---|---|---| -| 纯 Python 算法 | 数学、DROP、MCQ | 核心依赖（sympy/numpy/scipy） | -| BFCL 官方 AST 判定器 | bfcl_v3 | 已内置（`evalharness/third_party/bfcl/`，Apache-2.0） | -| 官方执行环境 | humaneval、bigcodebench、live_code_bench、swe_bench | Docker 镜像，判分时按需拉取 | - -另有本地引擎类（`tau2_bench`）使用官方 tau2 包（本地源码安装，无重依赖）。 - 离线验证安装（不需要模型、不联网）： ```bash @@ -54,7 +47,6 @@ evalharness eval run gsm8k --model mock:boxed --limit 8 # 数据 → prompt → 生成 → 判分 → 报告 全链路可用 ``` -代码执行类 benchmark 需要宿主机有 Docker。 ## 快速开始 diff --git a/evalharness/cli.py b/evalharness/cli.py index 84f75a9..1374a45 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -332,14 +332,7 @@ def _cmd_eval_run(args) -> int: 'failed', _time.time() - t0) if len(rows) > 1: - print(f'\n{"benchmark":<20} {"metric":<14} {"score":>8} {"n":>5} {"time":>9}') - print('-' * 62) - for r in rows: - val = 'ERR' if not r['ok'] else _f3(r['value']) - h = r.get('hours') or 0 - tdisp = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s" - print(f"{r['name']:<20} {r['metric']:<14} {val!s:>8} {r.get('n', '')!s:>5} {tdisp:>9}" - + (f" {r.get('err', '')}" if not r['ok'] else '')) + _print_summary_table(console, rows) ok = sum(1 for r in rows if r['ok']) print(f'\n{ok}/{len(rows)} ok' + (f' -> artifacts in {out_dir}/' if out_dir else '')) if out_dir: @@ -371,13 +364,65 @@ def _cmd_eval_run(args) -> int: 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] + [cats]) with open(f'{out_dir}/viz/summary.md', 'w', encoding='utf-8') as f: - f.write(f'# eval run summary\n\n| dataset | metric | value | secs |\n|---|---|---|---|\n') + import time as _tt + model_names = {r.get('model', '') for r in rows if r.get('model')} + head = f"# eval run summary\n\n- model: {', '.join(model_names) or '?'}\n" + head += f"- created: {_tt.strftime('%Y-%m-%d %H:%M:%S')}\n" + head += f"- benchmarks: {sum(1 for r in rows if r['ok'])}/{len(rows)} ok\n\n" + f.write(head) + f.write('| benchmark | metric | score | n | time | status |\n' + '|---|---|---:|---:|---:|---|\n') for r in rows: - f.write(f"| {r['name']} | {r['metric']} | {r['value']} | {r['secs']} |\n") - print(f'summary csv -> {out_dir}/viz/summary.csv') + v = _fmt_score(r.get('value')) + h = r.get('hours') or 0 + t = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s" + st = 'ok' if r['ok'] else f"failed: {r.get('err', '')[:60]}" + f.write(f"| {r['name']} | {r['metric']} | {v} | " + f"{r.get('n', '')} | {t} | {st} |\n") + if out_dir: + print(f'summary -> {out_dir}/viz/summary.md (+ summary.csv)') return 0 if all(r['ok'] for r in rows) else 1 +def _fmt_score(v): + """Unified score format: fractions render as percentages everywhere.""" + try: + v = float(v) + except (TypeError, ValueError): + return 'ERR' + return f'{v * 100:.1f}%' if 0.0 <= v <= 1.0 else f'{v:g}' + + +def _print_summary_table(console, rows): + """Rich multi-benchmark summary (falls back to aligned plain text).""" + if console is not None: + from rich.table import Table + + t = Table(title='Run Summary', header_style='bold cyan', + title_style='bold', expand=False) + for col, just in (('benchmark', 'left'), ('metric', 'left'), + ('score', 'right'), ('n', 'right'), ('time', 'right')): + t.add_column(col, justify=just) + for r in rows: + v = _fmt_score(r.get('value')) if r['ok'] else 'ERR' + h = r.get('hours') or 0 + tm = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s" + style = 'green' if r['ok'] else 'red' + t.add_row(r['name'], r['metric'], v, str(r.get('n', '')), tm, + style=style) + console.print(t) + return + print(f'\n{"benchmark":<20} {"metric":<16} {"score":>8} {"n":>6} {"time":>8}') + print('-' * 64) + for r in rows: + v = _fmt_score(r.get('value')) if r['ok'] else 'ERR' + h = r.get('hours') or 0 + tm = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s" + err = f" {r.get('err', '')}" if not r['ok'] else '' + print(f"{r['name']:<20} {r['metric']:<16} {v:>8} " + f"{str(r.get('n', '')):>6} {tm:>8}{err}") + + def _f3(v): try: return round(float(v), 4) diff --git a/evalharness/viz/renderers/text.py b/evalharness/viz/renderers/text.py index 5ffa417..41136ef 100644 --- a/evalharness/viz/renderers/text.py +++ b/evalharness/viz/renderers/text.py @@ -21,37 +21,42 @@ def text_table(target: Union[EvalReport, List[EvalReport]], opts: Dict) -> str: reports = target if isinstance(target, list) else [target] out: List[str] = [] for rep in reports: - head = f'{rep.dataset} [{rep.recipe}] model={rep.model or "?"} n={rep.num_samples}' - out.append('=' * max(len(head), 40)) + # headline: dedupe dataset/recipe when identical; join facts compactly + title = rep.dataset if rep.dataset == rep.recipe else f'{rep.dataset} [{rep.recipe}]' + facts = [f'model={rep.model or "?"}', f'n={rep.num_samples}'] + info = rep.metric_groups.get('run_info', {}) or {} + secs = sum(float((s.usage or {}).get('latency_s', 0) or 0) for s in rep.samples) + if secs >= 3600: + facts.append(f'time={secs / 3600:.2f}h') + elif secs: + facts.append(f'time={secs:.0f}s') + if info.get('gen_total_tokens'): + facts.append(f'tokens={info["gen_total_tokens"]}') + head = f'{title} · ' + ' '.join(facts) out.append(head) out.append('=' * max(len(head), 40)) - # run stats: duration, tokens, cost (from run_info + usage aggregates) - info = rep.metric_groups.get('run_info', {}) or {} - total_tokens = info.get('gen_total_tokens', 0) - tok_s = '' - if total_tokens: - tok_s = f' tokens={total_tokens}' - secs = 0.0 - for s in rep.samples: - secs += float((s.usage or {}).get('latency_s', 0) or 0) - dur = f' time={secs / 3600:.2f}h' if secs >= 3600 else (f' time={secs:.0f}s' if secs else '') - if dur or tok_s: - out.append(f'n_samples={rep.num_samples}{dur}{tok_s}') + if rep.num_failed_extractions: warn = (f'!! {rep.num_failed_extractions}/{rep.num_samples} extractions failed ' f'({_pct(rep.metrics.get("extraction_failure_rate", 0))}) -- check recipe/model fit') out.append(warn) - for metric, value in rep.metrics.items(): - if metric == 'extraction_failure_rate': - continue - out.append(f'{metric:<16} {_pct(value):>7} {_bars(value)}') + + metrics = [(m, v) for m, v in rep.metrics.items() + if m != 'extraction_failure_rate' and isinstance(v, (int, float))] + w = max([len(m) for m, _ in metrics] + [12]) # adaptive, long names survive + for metric, value in metrics: + out.append(f'{metric:<{w}} {_pct(value):>8} {_bars(value)}') + for group_name, groups in rep.metric_groups.items(): if group_name == 'run_info' or group_name.startswith('agg_error'): continue + numeric = {g: v for g, v in groups.items() if isinstance(v, (int, float))} + if not numeric: + continue + gw = max([len(str(g)) for g in numeric] + [12]) out.append(f'-- {group_name} ' + '-' * max(0, 30 - len(group_name))) - for g, v in groups.items(): - if isinstance(v, (int, float)): - out.append(f' {g:<28} {_pct(v):>7} {_bars(v, 20)}') + for g, v in numeric.items(): + out.append(f' {str(g):<{gw}} {_pct(v):>8} {_bars(v, 20)}') out.append('') return '\n'.join(out) @@ -79,22 +84,58 @@ def markdown(target: Union[EvalReport, List[EvalReport]], opts: Dict) -> str: @register_renderer('md_compare') def md_compare(target: List[EvalReport], opts: Dict) -> str: - """Side-by-side metric table for N reports (e.g. two models on one bench).""" + """Side-by-side metric table for N reports (models on one bench, or many + benches of one model -- the shape is the same: one row per benchmark). + The first report is the baseline; later columns get a delta marker.""" if not isinstance(target, list) or len(target) < 1: raise ValueError('md_compare needs a list of reports') + + # many DIFFERENT benchmarks, each with its own primary metric: a compact + # one-row-per-bench table beats a sparse metric-x-bench grid + datasets = {r.dataset for r in target} + primary_metrics = {next((m for m in r.metrics if m != 'extraction_failure_rate'), '') + for r in target} + if len(datasets) > 1 and len(target) == len(datasets) and len(primary_metrics) > 1: + out = ['# Comparison', '', '| benchmark | metric | score | n |', '|---|---|---:|---:|'] + best = max((next((v for m, v in r.metrics.items() + if m != 'extraction_failure_rate'), 0.0) for r in target)) + for r in target: + m = next((m for m in r.metrics if m != 'extraction_failure_rate'), '') + v = r.metrics.get(m, 0.0) + cell = f'**{_pct(v)}**' if v == best and len(target) > 1 else _pct(v) + out.append(f'| {r.dataset} | {m} | {cell} | {r.num_samples} |') + return '\n'.join(out + ['']) + metrics: List[str] = [] for rep in target: for m in rep.metrics: if m not in metrics and m != 'extraction_failure_rate': metrics.append(m) - cols = [f'{rep.dataset}/{rep.recipe}[{rep.model or "?"}]' for rep in target] - out = ['# Comparison', '', '| metric | ' + ' | '.join(cols) + ' |', + cols = [] + for rep in target: + # models on one bench -> show the model; many benches -> show the bench + same_dataset = len({r.dataset for r in target}) == 1 + name = rep.model or '?' if same_dataset else rep.dataset + cols.append(name) + out = ['# Comparison', '', + '| metric | ' + ' | '.join(cols) + ' |', '|---' * (len(cols) + 1) + '|'] for m in metrics: - cells = [_pct(rep.metrics.get(m, 0.0)) for rep in target] - best = max(rep.metrics.get(m, 0.0) for rep in target) - cells = [f'**{c}**' if rep.metrics.get(m, 0.0) == best and len(target) > 1 else c - for c, rep in zip(cells, target)] + cells = [] + base = target[0].metrics.get(m, 0.0) + for rep, col_i in zip(target, range(len(target))): + v = rep.metrics.get(m, 0.0) + cell = _pct(v) + best = max(r.metrics.get(m, 0.0) for r in target) + if v == best and len(target) > 1: + cell = f'**{cell}**' + if rep is not target[0] and isinstance(base, (int, float)): + d = v - base + if d > 0.0005: + cell += f' ▲{d * 100:+.1f}' + elif d < -0.0005: + cell += f' ▼{d * 100:+.1f}' + cells.append(cell) out.append(f'| {m} | ' + ' | '.join(cells) + ' |') out.append('') return '\n'.join(out)