Fail-fast endpoint probe (English, full curl hint, red panel on failure); slim run narration (drop duplicate generating/materializing lines); probe success line

Co-Authored-By: Claude <noreply@anthropic.com>

evalharness/cli.py

@@ -546,6 +546,13 @@ def _cmd_eval_run(args) -> int:
                          'secs': round(_time.time() - t0, 1), 'ok': False,
                          'err': f'{type(e).__name__}: {str(e)[:100]}'})
             print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr)
+            if console is not None:
+                from rich.panel import Panel
+
+                console.print(Panel(
+                    f'{type(e).__name__}: {e}',
+                    title=f'[bold red]✗ {name} FAILED[/bold red]',
+                    border_style='red', expand=False))
             _print_benchmark_result(console, i + 1, total_runs, name,
                                     'failed', _time.time() - t0)
 

evalharness/model/runner.py

@@ -497,6 +497,54 @@ def _progress(progress: bool, done: int, total: int, t0: float, usage: Usage) ->
         print(f'  [{done}/{total}] {rate:.1f} samples/s tokens={usage.total_tokens}', flush=True)
 
 
+
+_PROBED_SPECS = set()
+
+
+async def _probe_model(adapter, model_spec: str) -> None:
+    """Fail fast on an unreachable model endpoint.
+
+    One 1-token request before any dataset work: a wrong api-url/model
+    name surfaces in seconds (with a clear fix hint) instead of the
+    multi-minute retry ladder. Cached per spec so multi-benchmark runs
+    probe only once. Mock adapters are exempt.
+    """
+    if getattr(adapter, 'name', '') == 'mock':
+        return
+    members = getattr(adapter, 'adapters', [adapter])
+    if model_spec in _PROBED_SPECS:
+        return
+    bad = []
+    for a in members:
+        try:
+            out = await asyncio.wait_for(
+                a.generate([ChatMessage(role='user', content='ping')],
+                           max_tokens=1, temperature=0.0),
+                timeout=30)
+            if out is None or (not out.text and not out.tool_calls):
+                raise RuntimeError('empty response')
+        except Exception as e:
+            bad.append(f'{a.api_base}: {type(e).__name__} {str(e)[:80]}')
+    if bad and len(bad) == len(members):
+        import json as _json
+
+        model_name = getattr(members[0], 'model', '') or '<model>'
+        body = _json.dumps({'model': model_name,
+                            'messages': [{'role': 'user', 'content': 'ping'}],
+                            'max_tokens': 1})
+        curl = f'curl -m 5 {members[0].api_base}/chat/completions -H "Content-Type: application/json" -d {body!r}'
+        raise RuntimeError(
+            'Model endpoint unreachable -- aborted before running any samples.\n'
+            f'  endpoint: {members[0].api_base}\n'
+            f'  reason:   {bad[0]}\n'
+            'Fix: check that --api-url points to a running OpenAI-compatible server\n'
+            '  and --model matches the served model name. Verify manually:\n'
+            f'  {curl}')
+    if members and not bad:
+        print(f'· model endpoint ok ({len(members)} instance(s), '
+              f'{members[0].api_base})', flush=True)
+    _PROBED_SPECS.add(model_spec)
+
 async def run_eval(
     dataset: Union[Dataset, List[Sample]],
     model_spec: str,
@@ -541,6 +589,7 @@ async def run_eval(
     if few_shot_num < 0:
         few_shot_num = (spec.few_shot_num if spec is not None else 0)
     adapter = _make_adapter(model_spec, api_key=api_key)
+    await _probe_model(adapter, model_spec if isinstance(model_spec, str) else repr(adapter))
     # reports carry a string model label: pre-built adapter objects need one
     model_spec = model_spec if isinstance(model_spec, str) \
         else (getattr(model_spec, 'model', '') or repr(model_spec))
@@ -557,11 +606,8 @@ async def run_eval(
                                 scorers={'acc': {'name': 'exact', 'mode': 'raw'}})
     # materialize in a worker thread: hub downloads here are synchronous
     # (requests/ssl) and would otherwise stall the whole event loop
-    if status_callback:
-        status_callback('materializing dataset: cache check/download and parsing')
     raw_samples = await asyncio.to_thread(lambda: list(dataset))
-    if status_callback:
+
-        status_callback(f'dataset materialized: {len(raw_samples)} samples')
     if limit:
         raw_samples = raw_samples[:limit]
     # generate_predictions applies the SAME deterministic limiting internally;
@@ -569,7 +615,7 @@ async def run_eval(
     # list (positional pairing) instead of relying on in-place aliasing.
     samples = _apply_limits(list(raw_samples), limit, limit_per_task,
                              shuffle=not no_shuffle)  # MUST mirror generate_predictions
-    if progress:
+    if progress and not status_callback:
         mode = f'agent env={env}' if env else 'single-turn'
         print(f'generating: {adapter} on {len(samples)} samples '
               f'({mode}, concurrency={concurrency})', flush=True)