diff --git a/evalharness/cli.py b/evalharness/cli.py index 1c45d18..4a13b19 100644 --- a/evalharness/cli.py +++ b/evalharness/cli.py @@ -546,6 +546,13 @@ def _cmd_eval_run(args) -> int: 'secs': round(_time.time() - t0, 1), 'ok': False, 'err': f'{type(e).__name__}: {str(e)[:100]}'}) print(f'{name}: FAILED {type(e).__name__}: {str(e)[:160]}', file=sys.stderr) + if console is not None: + from rich.panel import Panel + + console.print(Panel( + f'{type(e).__name__}: {e}', + title=f'[bold red]✗ {name} FAILED[/bold red]', + border_style='red', expand=False)) _print_benchmark_result(console, i + 1, total_runs, name, 'failed', _time.time() - t0) diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 16fbaa6..9789728 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -497,6 +497,54 @@ def _progress(progress: bool, done: int, total: int, t0: float, usage: Usage) -> print(f' [{done}/{total}] {rate:.1f} samples/s tokens={usage.total_tokens}', flush=True) + +_PROBED_SPECS = set() + + +async def _probe_model(adapter, model_spec: str) -> None: + """Fail fast on an unreachable model endpoint. + + One 1-token request before any dataset work: a wrong api-url/model + name surfaces in seconds (with a clear fix hint) instead of the + multi-minute retry ladder. Cached per spec so multi-benchmark runs + probe only once. Mock adapters are exempt. + """ + if getattr(adapter, 'name', '') == 'mock': + return + members = getattr(adapter, 'adapters', [adapter]) + if model_spec in _PROBED_SPECS: + return + bad = [] + for a in members: + try: + out = await asyncio.wait_for( + a.generate([ChatMessage(role='user', content='ping')], + max_tokens=1, temperature=0.0), + timeout=30) + if out is None or (not out.text and not out.tool_calls): + raise RuntimeError('empty response') + except Exception as e: + bad.append(f'{a.api_base}: {type(e).__name__} {str(e)[:80]}') + if bad and len(bad) == len(members): + import json as _json + + model_name = getattr(members[0], 'model', '') or '' + body = _json.dumps({'model': model_name, + 'messages': [{'role': 'user', 'content': 'ping'}], + 'max_tokens': 1}) + curl = f'curl -m 5 {members[0].api_base}/chat/completions -H "Content-Type: application/json" -d {body!r}' + raise RuntimeError( + 'Model endpoint unreachable -- aborted before running any samples.\n' + f' endpoint: {members[0].api_base}\n' + f' reason: {bad[0]}\n' + 'Fix: check that --api-url points to a running OpenAI-compatible server\n' + ' and --model matches the served model name. Verify manually:\n' + f' {curl}') + if members and not bad: + print(f'· model endpoint ok ({len(members)} instance(s), ' + f'{members[0].api_base})', flush=True) + _PROBED_SPECS.add(model_spec) + async def run_eval( dataset: Union[Dataset, List[Sample]], model_spec: str, @@ -541,6 +589,7 @@ async def run_eval( if few_shot_num < 0: few_shot_num = (spec.few_shot_num if spec is not None else 0) adapter = _make_adapter(model_spec, api_key=api_key) + await _probe_model(adapter, model_spec if isinstance(model_spec, str) else repr(adapter)) # reports carry a string model label: pre-built adapter objects need one model_spec = model_spec if isinstance(model_spec, str) \ else (getattr(model_spec, 'model', '') or repr(model_spec)) @@ -557,11 +606,8 @@ async def run_eval( scorers={'acc': {'name': 'exact', 'mode': 'raw'}}) # materialize in a worker thread: hub downloads here are synchronous # (requests/ssl) and would otherwise stall the whole event loop - if status_callback: - status_callback('materializing dataset: cache check/download and parsing') raw_samples = await asyncio.to_thread(lambda: list(dataset)) - if status_callback: - status_callback(f'dataset materialized: {len(raw_samples)} samples') + if limit: raw_samples = raw_samples[:limit] # generate_predictions applies the SAME deterministic limiting internally; @@ -569,7 +615,7 @@ async def run_eval( # list (positional pairing) instead of relying on in-place aliasing. samples = _apply_limits(list(raw_samples), limit, limit_per_task, shuffle=not no_shuffle) # MUST mirror generate_predictions - if progress: + if progress and not status_callback: mode = f'agent env={env}' if env else 'single-turn' print(f'generating: {adapter} on {len(samples)} samples ' f'({mode}, concurrency={concurrency})', flush=True)