!perf spec flag; perf_stats full dashboard columns (success_rate, latency/ttft/tpot mean+P90+P99, tokens mean/total, output TPS, request QPS); summary.csv perf columns aligned

evalharness/cli.py

@@ -205,15 +205,28 @@ def _cmd_eval_run(args) -> int:
             with open(f'{out_dir}/viz/summary.csv', 'w', newline='', encoding='utf-8') as f:
                 w = _csv.writer(f)
                 w.writerow(['benchmark', 'score', 'metric', 'num_samples',
-                            'time_h', 'time_s', 'extract_fail', 'categories'])
+                            'time_h', 'time_s', 'extract_fail',
+                            'success_rate', 'latency_mean_s', 'output_tps', 'request_qps',
+                            'input_tokens_mean', 'output_tokens_mean', 'total_tokens',
+                            'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s',
+                            'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s',
+                            'categories'])
                 for r in rows:
+                    perf = (r.get('groups') or {}).get('perf') or {}
                     cats = '; '.join(f'{g}={_f3(v)}'
                                      for gname, gv in (r.get('groups') or {}).items()
+                                     if gname != 'perf'
                                      for g, v in (gv or {}).items()
                                      if isinstance(v, (int, float)))[:2000]
                     w.writerow([r['name'], _f3(r.get('value')), r['metric'], r.get('n', ''),
                                 r.get('hours', ''), r.get('secs', ''),
-                                r.get('extract_fail', 0), cats])
+                                r.get('extract_fail', 0)] +
+                               [perf.get(k, '') for k in (
+                                   'success_rate', 'latency_mean_s', 'output_tps', 'request_qps',
+                                   'input_tokens_mean', 'output_tokens_mean', 'total_tokens',
+                                   'ttft_mean_s', 'ttft_p90_s', 'ttft_p99_s',
+                                   'tpot_mean_s', 'tpot_p90_s', 'tpot_p99_s')] +
+                               [cats])
             with open(f'{out_dir}/viz/summary.md', 'w', encoding='utf-8') as f:
                 f.write(f'# eval run summary\n\n| dataset | metric | value | secs |\n|---|---|---|---|\n')
                 for r in rows:

evalharness/eval/aggregator.py

@@ -135,19 +135,38 @@ def perf_stats(results: List[SampleResult], metric: str):
             if (r.usage or {}).get('ttft_s') is not None]
     itl = [float(r.usage['itl_mean_s']) for r in results
            if (r.usage or {}).get('itl_mean_s') is not None]
-    in_tok = sum(int((r.usage or {}).get('input_tokens', 0) or 0) for r in results)
+    n = len(lat)
-    out_tok = sum(int((r.usage or {}).get('output_tokens', 0) or 0) for r in results)
+    in_toks = [int((r.usage or {}).get('input_tokens', 0) or 0) for r in results]
+    out_toks = [int((r.usage or {}).get('output_tokens', 0) or 0) for r in results]
+    in_tok, out_tok = sum(in_toks), sum(out_toks)
     retried = sum(1 for r in results if (r.usage or {}).get('retries'))
+    ok = sum(1 for r in results
+             if (r.usage or {}).get('http_status') in (None, 200))  # None=unmeasured
     wall = sum(lat)
+    # TPOT per request: (latency - ttft) / max(output_tokens - 1, 1)
+    tpots = []
+    for r in results:
+        u = r.usage or {}
+        lt, tf, ot = u.get('latency_s'), u.get('ttft_s'), u.get('output_tokens')
+        if lt and tf is not None and ot and ot > 1:
+            tpots.append((lt - tf) / (ot - 1))
     out = {
-        'n_requests': len(lat),
+        'n_requests': n,
-        'latency_p50_s': _pct(lat, 50), 'latency_p95_s': _pct(lat, 95),
+        'success_rate': round(ok / n, 4) if n else None,
-        'latency_p99_s': _pct(lat, 99), 'latency_mean_s': round(statistics.mean(lat), 3) if lat else None,
+        'latency_mean_s': round(statistics.mean(lat), 3) if lat else None,
-        'ttft_p50_s': _pct(ttft, 50), 'ttft_p95_s': _pct(ttft, 95),
+        'latency_p50_s': _pct(lat, 50), 'latency_p90_s': _pct(lat, 90),
-        'itl_mean_s': round(statistics.mean(itl), 4) if itl else None,
+        'latency_p95_s': _pct(lat, 95), 'latency_p99_s': _pct(lat, 99),
-        'input_tokens': in_tok, 'output_tokens': out_tok,
+        'ttft_mean_s': round(statistics.mean(ttft), 3) if ttft else None,
-        'retry_rate': round(retried / len(lat), 3) if lat else None,
+        'ttft_p90_s': _pct(ttft, 90), 'ttft_p99_s': _pct(ttft, 99),
+        'tpot_mean_s': round(statistics.mean(tpots), 4) if tpots else None,
+        'tpot_p90_s': _pct(tpots, 90), 'tpot_p99_s': _pct(tpots, 99),
+        'input_tokens_mean': round(statistics.mean(in_toks), 1) if in_toks else 0,
+        'output_tokens_mean': round(statistics.mean(out_toks), 1) if out_toks else 0,
+        'total_tokens': in_tok + out_tok,
+        'output_tps': round(out_tok / wall, 2) if wall else None,    # tokens/s
+        'request_qps': round(n / wall, 4) if wall else None,        # req/s
         'wall_latency_s': round(wall, 1),
+        'retry_rate': round(retried / n, 3) if n else None,
     }
     return out
 

evalharness/model/runner.py

@@ -405,7 +405,7 @@ def _make_adapter(spec: str) -> ModelAdapter:
         return _CACHE[cache_key]
     opts = {}
     while True:
-        for f in ('!nothink', '!textools'):
+        for f in ('!nothink', '!textools', '!perf'):
             if spec.endswith(f):
                 spec = spec[:-len(f)]
                 opts[f] = True
@@ -437,6 +437,8 @@ def _make_adapter(spec: str) -> ModelAdapter:
             a.extra['no_think'] = True
         if opts.get('!textools'):
             a.extra['tools_mode'] = 'text'
+        if opts.get('!perf'):
+            a.extra['collect_perf'] = True
     _CACHE[cache_key] = adapter
     return adapter