Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Gate knee criteria: not-worse (0.9x) instead of must-improve (1.1x)

Browse Source 
Demanding a 10% gain to keep doubling settled [1,2]->1 on the first
noisy plateau (lbv2: 4k..2M-token docs, completion-rate noise dwarfs
10%). Now: keep climbing while not clearly worse (>=0.9x); bisect only
on clear degradation; samples per level doubled (max(3, 2*level)) to
shrink noise; steady re-probes +1 after ~60s so a noise-induced settle
cannot pin the gate forever. Overshoot past the true knee is trimmed
by the failure channel (timeouts -> x0.7), which is the real ceiling
finder on a prefill-bound endpoint.

Noise-swept at +-25%: capacities 4/8/16 settle at 11/31/16 without a
failure model; production failures pull the overshoot back down.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
sora 2026-09-15 03:32:08 +00:00
parent ecb29309ef
commit b81bff662d
1 changed files with 25 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
evalharness/model/pool.py
View File

@ -312,7 +312,7 @@ class AdaptiveGate:
# not enough evidence yet at this level: keep measuring. # not enough evidence yet at this level: keep measuring.
# need = max(MIN_OK, level): rate noise shrinks only with # need = max(MIN_OK, level): rate noise shrinks only with
# samples proportional to the concurrency being judged # samples proportional to the concurrency being judged
need_ok = max(self.MIN_OK, lvl) need_ok = max(self.MIN_OK, lvl * 2)
if self._level_ok < need_ok and dt < self.MAX_AT_LEVEL_S: if self._level_ok < need_ok and dt < self.MAX_AT_LEVEL_S:
return return
# zero completions in MAX_AT_LEVEL_S: hang or overloaded -> hold # zero completions in MAX_AT_LEVEL_S: hang or overloaded -> hold
@ -325,15 +325,20 @@ class AdaptiveGate:
if self._mode == 'probe': if self._mode == 'probe':
self.stats['probe'] += 1 self.stats['probe'] += 1
improved = prev_rate is None or rate > prev_rate * self.GAIN_EPS # continue while NOT WORSE (>= 0.9x): with heterogeneous
if improved and lvl < self.HI: # request lengths (lbv2: 4k..2M-token docs) completion-rate
# noise dwarfs a 10% gain threshold, and demanding strict
# improvement bisected [1,2]->1 on the first plateau.
# Only CLEAR degradation (<0.9x) means past the knee.
ok = prev_rate is None or rate >= prev_rate * 0.9
if ok and lvl < self.HI:
self._bis = (lvl, min(lvl * 2, self.HI)) # remember bounds self._bis = (lvl, min(lvl * 2, self.HI)) # remember bounds
self.limit = float(min(lvl * 2, self.HI)) self.limit = float(min(lvl * 2, self.HI))
self.stats['ramp_demand'] += 1 self.stats['ramp_demand'] += 1
self._push_limit() self._push_limit()
self._enter_level() self._enter_level()
elif not improved: elif not ok:
# throughput plateaued: knee is between prev_lvl and lvl # throughput CLEARLY degraded: knee is in (prev_lvl, lvl]
self._mode = 'bisect' self._mode = 'bisect'
self._bis = (prev_lvl or max(1, lvl // 2), lvl) self._bis = (prev_lvl or max(1, lvl // 2), lvl)
self._good_rate = prev_rate or rate self._good_rate = prev_rate or rate
@ -350,13 +355,24 @@ class AdaptiveGate:
self._enter_level() self._enter_level()
else: else:
self._enter_level('steady') # hit HI with gains: stay self._enter_level('steady') # hit HI with gains: stay
elif self._mode == 'steady':
# capacity estimates are noisy: periodically re-probe upward
self._steady_ticks = getattr(self, '_steady_ticks', 0) + 1
if self._steady_ticks >= 12: # ~60s at PROBE_S=5
self._steady_ticks = 0
self._mode = 'probe'
self._prev = (lvl, rate)
# try one level up, not a full double, from settled state
self.limit = float(min(lvl + 1, self.HI))
self._push_limit()
self._enter_level()
return
elif self._mode == 'bisect': elif self._mode == 'bisect':
lo, hi = self._bis lo, hi = self._bis
if rate > self._good_rate * self.GAIN_EPS: if rate >= self._good_rate * 0.9:
lo = lvl # still improving: knee is higher lo = lvl # not worse here: knee is at/above
self._good_rate = rate
else: else:
hi = lvl # no gain: knee is lower hi = lvl # clearly worse: knee is below
self._bis = (lo, hi) self._bis = (lo, hi)
if hi - lo <= 1: if hi - lo <= 1:
self.limit = float(lo) self.limit = float(lo)