"""Multi-endpoint load balancing: one logical model, N local ports. from evalharness.model.pool import PooledAdapter from evalharness.model.adapter import resolve_adapter base = resolve_adapter('openai/http://127.0.0.1:8123/v1?Qwen3-8B') pool = PooledAdapter([resolve_adapter(f'openai/http://127.0.0.1:{p}/v1?Qwen3-8B') for p in range(8123, 8131)]) out = await pool.generate(...) # round-robin over instances Traffic management: round-robin keeps per-endpoint traffic even; backends that fail repeatedly enter a cool-down window and are skipped until it expires, so one sick endpoint cannot absorb its share of the load. """ import asyncio import contextlib import itertools import time from typing import Dict, List, Optional from ..data.sample import ChatMessage from .adapter import ModelAdapter from .output import ModelOutput, Usage class PooledAdapter(ModelAdapter): """Round-robin over N equivalent backend instances with health cooling.""" name = 'pool' COOLDOWN_S = 60.0 # a backend that failed EVERY attempt rests this long COOLDOWN_AFTER = 2 # consecutive full-pass failures before cooling def __init__(self, adapters: List[ModelAdapter]): if not adapters: raise ValueError('PooledAdapter needs at least one backend') super().__init__(model=adapters[0].model, api_base=adapters[0].api_base) self.adapters = adapters self._cycle = itertools.cycle(range(len(adapters))) self.usage = Usage() # request outcome counters (success rate accounting) self.stats = {'requests': 0, 'ok': 0, 'failed': 0, 'retried': 0} # per-backend health: consecutive_failures, cooling_until, per-endpoint counts self._health = [{ 'fails': 0, 'until': 0.0, 'ok': 0, 'req': 0} for _ in adapters] # adaptive per-endpoint concurrency gates (AIMD over /metrics signals) self._gates = [AdaptiveGate(a) for a in adapters] def _next(self) -> ModelAdapter: """Round-robin, skipping endpoints inside their cool-down window.""" n = len(self.adapters) now = time.time() for _ in range(n): i = next(self._cycle) h = self._health[i] if h['until'] <= now or all(x['until'] <= now for x in self._health): self._health[i]['req'] += 1 return self.adapters[i] # everything cooling: take the next anyway (better to try than stall) i = next(self._cycle) self._health[i]['req'] += 1 return self.adapters[i] def _mark(self, adapter: ModelAdapter, ok: bool) -> None: try: i = self.adapters.index(adapter) except ValueError: return h = self._health[i] if ok: h['fails'] = 0 h['until'] = 0.0 h['ok'] += 1 else: h['fails'] += 1 if h['fails'] >= self.COOLDOWN_AFTER: h['until'] = time.time() + self.COOLDOWN_S h['fails'] = 0 def request_stats(self) -> Dict[str, float]: """Success-rate + per-endpoint traffic view (load-balance audit).""" n = self.stats['requests'] out = { 'requests': n, 'success_rate': self.stats['ok'] / n if n else 0.0, 'retry_rate': self.stats['retried'] / n if n else 0.0, 'failure_rate': self.stats['failed'] / n if n else 0.0, } for i, (a, h) in enumerate(zip(self.adapters, self._health)): tag = a.api_base.rsplit('//', 1)[-1].replace('/', '_') out[f'ep{i}_{tag}_reqs'] = h['req'] out[f'ep{i}_{tag}_ok'] = h['ok'] if i < len(self._gates): for k, v in self._gates[i].report().items(): out[f'ep{i}_{tag}_{k}'] = v return out async def generate(self, messages: List[ChatMessage], tools: Optional[list] = None, **kw) -> ModelOutput: last_exc = None self.stats['requests'] += 1 for _ in range(len(self.adapters)): # try each instance once adapter = self._next() try: i = self.adapters.index(adapter) await self._gates[i].acquire() try: out = await adapter.generate(messages, tools=tools, **kw) finally: self._gates[i].release(True) self.usage = self.usage + out.usage self.stats['ok'] += 1 self._mark(adapter, True) if out.usage.retries: self.stats['retried'] += 1 return out except Exception as e: # dead/overloaded instance -> next last_exc = e self._mark(adapter, False) with contextlib.suppress(ValueError): self._gates[self.adapters.index(adapter)].release(False) # 4xx (e.g. 400 overloaded) still worth trying ANOTHER instance: # one backend's state can differ from the rest continue self.stats['failed'] += 1 raise last_exc async def close(self) -> None: for a in self.adapters: await a.close() for g in self._gates: g.stop() class AdaptiveGate: """Per-endpoint adaptive concurrency limiter (AIMD + server signals). Goal: keep the backend SATURATED (high XPU util / throughput) without pushing it over the cliff (500s / child crashes). Signals: - server /metrics: num_queue_reqs > 0 means WE are pushing too hard for the current mix; idle (no queue, low running) means room to grow - request failures: multiplicative decrease (survive first) Control law (classic AIMD): +1 concurrency per probe interval when the endpoint looks underfed -1 when the server reports a queue (gentle) x0.7 on any failed request (fast backoff), floor at LO Purely additive to PooledAdapter: one gate per backend, no caller change. """ LO = 2 # never go below: progress beats perfection HI = 96 # sane ceiling for one endpoint PROBE_S = 5.0 # metrics probe interval def __init__(self, adapter: ModelAdapter): self.adapter = adapter self.limit = 8.0 # float for smooth x0.7; compare with int() self._inflight = 0 self._cond: Optional[asyncio.Condition] = None self._task: Optional[asyncio.Task] = None self._stopped = False self.stats = {'probe': 0, 'ramp': 0, 'hold_queue': 0, 'backoff_fail': 0, 'backoff_queue': 0} # ---- gate semantics ---- async def acquire(self) -> None: if self._cond is None: # lazy init in the running loop self._cond = asyncio.Condition() self._task = asyncio.get_event_loop().create_task(self._probe_loop()) while self._inflight >= max(1, int(self.limit)): await self._cond.acquire() try: await self._cond.wait() finally: self._cond.release() self._inflight += 1 def release(self, ok: bool) -> None: self._inflight = max(0, self._inflight - 1) if not ok: # multiplicative decrease -- survival first before = self.limit self.limit = max(self.LO, self.limit * 0.7) if before != self.limit: self.stats['backoff_fail'] += 1 self._wake() def _wake(self) -> None: if self._cond is not None: # fire-and-forget notify (loop may not be ours -- best effort) try: fut = asyncio.ensure_future(self._notify_all()) fut.add_done_callback(lambda f: None) except RuntimeError: pass async def _notify_all(self) -> None: async with self._cond: self._cond.notify_all() # ---- server-signal probe ---- async def _probe_once(self) -> None: import urllib.request url = f'{self.adapter.api_base.rstrip("/")}/metrics' try: with urllib.request.urlopen(url, timeout=4) as resp: text = resp.read().decode('utf-8', 'ignore') except Exception: return # no metrics (or busy): hold current limit running = queue = None for line in text.splitlines(): if line.startswith('sglang:num_running_reqs'): running = float(line.rsplit(' ', 1)[-1]) elif line.startswith('sglang:num_queue_reqs'): queue = float(line.rsplit(' ', 1)[-1]) self.stats['probe'] += 1 if queue is None and running is None: return if queue is not None and queue >= 2: # server is queuing OUR excess: gentle additive decrease self.limit = max(self.LO, self.limit - 1) self.stats['backoff_queue'] += 1 elif (queue or 0) == 0 and (running is None or running < max(2, int(self.limit))): # underfed: no queue and running below our own cap -> ramp up self.limit = min(self.HI, self.limit + 1) self.stats['ramp'] += 1 else: self.stats['hold_queue'] += 1 self._wake() async def _probe_loop(self) -> None: import contextlib while not self._stopped: with contextlib.suppress(Exception): await self._probe_once() await asyncio.sleep(self.PROBE_S) def stop(self) -> None: self._stopped = True if self._task is not None: self._task.cancel() def report(self) -> Dict[str, float]: return {'limit': max(1, int(self.limit)), 'inflight': self._inflight, **{f'gate_{k}': v for k, v in self.stats.items()}} def pooled(specs: List[str]) -> PooledAdapter: """['openai/http://127.0.0.1:8123/v1?M', ...] -> PooledAdapter.""" from .adapter import resolve_adapter return PooledAdapter([resolve_adapter(s) for s in specs])