diff --git a/README.md b/README.md index d4632b4..0d16609 100644 --- a/README.md +++ b/README.md @@ -127,9 +127,9 @@ evalharness viz show report.json --style errors # 失败样本下钻 ``` / ├── summary.xlsx 总表 Excel(Summary/Perf/Categories/Samples 四 sheet,主入口) -├── summary.md / .csv 同一张总表的 markdown / csv 版 +├── summary.csv 同一张总表的 csv 版 └── / 每个 benchmark 一个目录 - ├── report.json 完整报告(每样本预测、分数明细、token、轨迹;重判分用) + ├── report.jsonl 完整报告,流式行格式(首行报告头,之后每行一个样本;grep/tail 友好) └── detail.md 该 bench 的 markdown 详情 ``` diff --git a/evalharness/cli.py b/evalharness/cli.py index ad78878..6d31a38 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -524,7 +524,7 @@ def _cmd_eval_run(args) -> int: bench_dir = _P(out_dir) / name bench_dir.mkdir(parents=True, exist_ok=True) - report.save(str(bench_dir / 'report.json')) + report.save(str(bench_dir / 'report.jsonl')) with open(bench_dir / 'detail.md', 'w', encoding='utf-8') as f: f.write(render(report, style='md')) if args.verbose: @@ -618,22 +618,6 @@ def _cmd_eval_run(args) -> int: 'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s', 'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] + [cats]) - with open(f'{out_dir}/summary.md', 'w', encoding='utf-8') as f: - import time as _tt - model_names = {r.get('model', '') for r in rows if r.get('model')} - head = f"# eval run summary\n\n- model: {', '.join(model_names) or '?'}\n" - head += f"- created: {_tt.strftime('%Y-%m-%d %H:%M:%S')}\n" - head += f"- benchmarks: {sum(1 for r in rows if r['ok'])}/{len(rows)} ok\n\n" - f.write(head) - f.write('| benchmark | metric | score | n | time | status |\n' - '|---|---|---:|---:|---:|---|\n') - for r in rows: - v = _fmt_score(r.get('value')) - h = r.get('hours') or 0 - t = f'{h:.2f}h' if h >= 0.995 else f"{r.get('secs', 0):.0f}s" - st = 'ok' if r['ok'] else f"failed: {r.get('err', '')[:60]}" - f.write(f"| {r['name']} | {r['metric']} | {v} | " - f"{r.get('n', '')} | {t} | {st} |\n") # artifacts notice: tell the user where everything landed (or how to save); # rich terminals get clickable file:// links (iTerm2/kitty/WezTerm/WT...) @@ -649,12 +633,12 @@ def _cmd_eval_run(args) -> int: f'\n{mark} [bold]运行结束[/bold] · {ok_n}/{len(rows)} benchmarks ok\n' f' 结果 {ap}\n' f' ├─ summary.xlsx (excel 打开总表)\n' - f' ├─ summary.md / summary.csv\n' - f' └─ /report.json + detail.md (每 benchmark 一个目录)') + f' ├─ summary.csv\n' + f' └─ /report.jsonl + detail.md (每 benchmark 一个目录)') else: print(f'\n运行结束 · {ok_n}/{len(rows)} benchmarks ok\n' f' 结果 {ap}\n' - f' ├─ summary.xlsx\n └─ /report.json + detail.md') + f' ├─ summary.xlsx\n └─ /report.jsonl + detail.md') elif rows and rows[0]['ok'] and args.out: _notice('运行结束 · 结果已保存', args.out) print(f'{ok_n}/{len(rows)} benchmarks ok') @@ -828,8 +812,7 @@ def build_parser() -> argparse.ArgumentParser: help='first N samples PER subset/category (evalscope --limit semantics); ' 'composable with --limit (intersection)') p.add_argument('--out', help='save the EvalReport json here (single dataset)') - p.add_argument('--out-dir', help='save reports/.json + viz/.txt + summary.md ' - 'here (multi-dataset runs)') + p.add_argument('--out-dir', help='output directory: summary.xlsx/csv + one dir per benchmark') p.add_argument('--style', default='text', help='result render style (text/md/radar/errors)') p.add_argument('--verbose', action='store_true', help='print full render for every dataset') _add_override_flags(p) diff --git a/evalharness/eval/record.py b/evalharness/eval/record.py index 4adc651..62370cd 100644 --- a/evalharness/eval/record.py +++ b/evalharness/eval/record.py @@ -65,6 +65,18 @@ class EvalReport(BaseModel): def save(self, path) -> None: import json + if str(path).endswith('.jsonl'): + # streaming format: first line = report header, then one + # sample per line (grep/split/tail friendly) + head = self.model_dump(exclude={'samples'}) + head['type'] = 'report' + with open(path, 'w', encoding='utf-8') as f: + f.write(json.dumps(head, ensure_ascii=False) + '\n') + for smp in self.samples: + row = smp if isinstance(smp, dict) else smp.model_dump() + row['type'] = 'sample' + f.write(json.dumps(row, ensure_ascii=False) + '\n') + return with open(path, 'w', encoding='utf-8') as f: json.dump(self.model_dump(), f, ensure_ascii=False, indent=2) @@ -72,5 +84,20 @@ class EvalReport(BaseModel): def load(cls, path) -> 'EvalReport': import json + if str(path).endswith('.jsonl'): + head, samples = None, [] + with open(path, encoding='utf-8') as f: + for line in f: + line = line.strip() + if not line: + continue + row = json.loads(line) + if row.pop('type', '') == 'report' or head is None: + head = {k: v for k, v in row.items() if k != 'type'} + else: + samples.append({k: v for k, v in row.items() if k != 'type'}) + head = head or {} + head['samples'] = samples + return cls.model_validate(head) with open(path, encoding='utf-8') as f: return cls.model_validate(json.load(f))