From d830e08923e7ed89f386160e054198d540e777fc Mon Sep 17 00:00:00 2001 From: sora Date: Tue, 25 Aug 2026 02:26:26 +0000 Subject: [PATCH] Drop logprob scoring path entirely; MCQ = strict-letter generation (evalscope parity 'ANSWER: X'), few-shot defaults stay; fix output.py duplication --- evalharness/eval/runner.py | 12 ---- evalharness/model/adapter.py | 134 ----------------------------------- evalharness/model/output.py | 23 ------ evalharness/model/runner.py | 34 ++------- 4 files changed, 5 insertions(+), 198 deletions(-) diff --git a/evalharness/eval/runner.py b/evalharness/eval/runner.py index bd7827f..ebaa0ee 100644 --- a/evalharness/eval/runner.py +++ b/evalharness/eval/runner.py @@ -84,18 +84,6 @@ def evaluate( result.env_state = pred['env_state'] if isinstance(pred, dict) and pred.get('usage'): result.usage = pred['usage'] - if isinstance(pred, dict) and 'logprob_best' in pred: - letters = 'ABCDEFGH' - tgt = str(sample.target) - result.extracted_prediction = letters[pred['logprob_best']] \ - if 0 <= pred['logprob_best'] < len(letters) else '' - result.extraction_ok = bool(result.extracted_prediction) - result.extraction_note = 'logprob_argmax' - result.score_details['acc_norm'] = { - 'pred': letters[pred['logprob_best_norm']] - if 0 <= pred['logprob_best_norm'] < len(letters) else '', - 'correct_norm': int(letters[pred['logprob_best_norm']] == tgt) if pred.get('logprob_best_norm', -1) >= 0 else 0, - } try: if bp is not None and sample.sandbox and sample.sandbox.image: bp.ensure(sample.sandbox.image) # wait only if this one still pulling diff --git a/evalharness/model/adapter.py b/evalharness/model/adapter.py index 3410f52..c49d7c2 100644 --- a/evalharness/model/adapter.py +++ b/evalharness/model/adapter.py @@ -53,140 +53,6 @@ class ModelAdapter: ) -> ModelOutput: raise NotImplementedError - async def score_choices( - self, - prompt: str, - choices: List[str], - ) -> 'ModelOutput': - """Paper-faithful MCQ scoring (P1): logprob of each choice continuing - the prompt. Strategy per backend capability: - - completions_echo -- /completions echo+prompt_logprobs (vllm/sglang/ - ollama); true continuation logprob, the lm-eh paper path - chat_first_token -- /chat/completions max_tokens=1 + top_logprobs; the - NEXT-token distribution encodes each choice; works on - every OpenAI-protocol API (cloud included); scores - choices as first-token probabilities - - Selected via extra['logprob_strategy'] ('auto' default). - """ - strategy = self.extra.get('logprob_strategy', 'auto') - if strategy in ('auto', 'completions_echo'): - try: - return await self._score_choices_echo(prompt, choices) - except Exception as e: - if strategy == 'completions_echo': - raise - self.extra['logprob_strategy'] = 'chat_first_token' # downgrade once - return await self._score_choices_first_token(prompt, choices) - - async def _score_choices_first_token(self, prompt: str, choices: List[str]): - """Approximate P1 via next-token top_logprobs (any chat API). - - The choices are shown in the prompt; we score P(first content token - of each choice | prompt) from the top_logprobs of a 1-token reply. - """ - from .output import ChoiceLogprob, ModelOutput - - letters = 'ABCDEFGHIJ' - opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(choices)) - asked = (f'{prompt}\n{opts}\n\n' - 'Answer with only the letter of the correct option.') - payload = {'model': self.model, 'messages': [{'role': 'user', 'content': asked}], - 'max_tokens': 1, 'temperature': 0.0, - 'logprobs': True, 'top_logprobs': 20} - headers = {'Content-Type': 'application/json'} - if self.api_key: - headers['Authorization'] = f'Bearer {self.api_key}' - data = await self._post(f'{self.api_base}/chat/completions', payload, headers) - import json as _json - - choice = (data.get('choices') or [{}])[0] - lp_wrapper = (choice.get('logprobs') or {}) - top = lp_wrapper.get('content') or lp_wrapper.get('top_logprobs') or [] - # top: list per generated token of {token: logprob} dicts - token_lps: Dict[str, float] = {} - first = top[0] if top else None - if isinstance(first, dict) and 'top_logprobs' in first: - # OpenAI/sglang chat shape: content[0] = {token, logprob, top_logprobs:[...]} - entries = {} - for e in (first.get('top_logprobs') or []): - tok, lp = e.get('token'), e.get('logprob') - if tok is not None and lp is not None: - entries[str(tok)] = lp - elif isinstance(first, dict): - entries = first # vllm-ish {token: lp} map - else: - entries = {} - for tok, lp in entries.items(): - if isinstance(lp, dict): - lp = lp.get('logprob') - if lp is None: - continue - try: - token_lps[str(tok).strip().upper()] = float(lp) - except (TypeError, ValueError): - continue - if not token_lps: - raise RuntimeError(f'no top_logprobs from {self.api_base}; P1 needs a ' - 'backend exposing logprobs (or use mcq_mode=generate)') - results = [] - for i in range(len(choices)): - letter = letters[i] - lp = token_lps.get(letter, -100.0) - results.append(ChoiceLogprob(text=choices[i], logprob=lp, - token_logprobs=[lp], num_tokens=1)) - usage = Usage(finish_reason=choice.get('finish_reason', '')) - return ModelOutput(model=self.model, choice_logprobs=results, usage=usage) - - async def _score_choices_echo(self, prompt: str, choices: List[str]): - from .output import ChoiceLogprob, ModelOutput - - results: List[ChoiceLogprob] = [] - for choice in choices: - full = f'{prompt} {choice}' - payload = {'model': self.model, 'prompt': full, 'max_tokens': 1, - 'echo': True, 'prompt_logprobs': 0, - 'temperature': 0.0} - headers = {'Content-Type': 'application/json'} - if self.api_key: - headers['Authorization'] = f'Bearer {self.api_key}' - data = await self._post_raw(f'{self.api_base}/completions', payload, headers) - # vllm/sglang shape: choices[0].prompt_logprobs: list[None|{token: lp}] - import json as _json - - ch = (data.get('choices') or [{}])[0] - pl = ch.get('prompt_logprobs') or [] - prompt_ids = ch.get('prompt_token_ids') or [] - # tokens belonging to the choice = the tail after the prompt part. - # echo mode returns the whole sequence; we approximate the choice - # token count via the tail offset encoded in logprob entries. - lps: List[float] = [] - for entry in pl: - if isinstance(entry, dict): - lp = next(iter(entry.values())) - if isinstance(lp, dict): - lp = lp.get('logprob', 0.0) - lps.append(float(lp)) - if not lps: - raise RuntimeError( - f'backend at {self.api_base} returned no prompt_logprobs; ' - 'P1 logprob scoring needs vllm/sglang-style completions-echo' - ) - # last len tokens we cannot know exactly; use the trailing segment - # matching the choice's char proportion as a robust approximation - n_prompt_chars = len(prompt) + 1 - frac = max(len(choice), 1) / max(len(full), 1) - n_choice = max(1, round(frac * len(lps))) - choice_lps = lps[-n_choice:] - results.append(ChoiceLogprob( - text=choice, logprob=sum(choice_lps), - token_logprobs=choice_lps, num_tokens=len(choice_lps))) - return ModelOutput(model=self.model, choice_logprobs=results) - - async def _post_raw(self, url: str, payload: Dict, headers: Dict) -> Dict[str, Any]: - return await self._post(url, payload, headers) - async def close(self) -> None: pass diff --git a/evalharness/model/output.py b/evalharness/model/output.py index 1948161..40fba6c 100644 --- a/evalharness/model/output.py +++ b/evalharness/model/output.py @@ -44,21 +44,6 @@ class Usage(BaseModel): ) -class ChoiceLogprob(BaseModel): - """Continuation scoring for one choice (paper-faithful MCQ evaluation).""" - - text: str = '' - logprob: float = 0.0 # total logprob of the choice tokens - token_logprobs: List[float] = Field(default_factory=list) - num_tokens: int = 0 - - @property - def logprob_per_char(self) -> float: - """lm-eval-harness acc_norm uses per-BYTE normalization; chars are the - practical proxy (identical ranking for ascii-dominant choices).""" - return self.logprob / max(len(self.text), 1) - - class ModelOutput(BaseModel): """What every adapter returns. text may be '' when the model only calls tools.""" @@ -68,15 +53,7 @@ class ModelOutput(BaseModel): raw: Optional[Dict[str, Any]] = None # provider response (audit/retry) model: str = '' created_at: str = Field(default_factory=lambda: time.strftime('%Y-%m-%d %H:%M:%S')) - choice_logprobs: Optional[List[ChoiceLogprob]] = None # P1 continuation scoring @property def is_tool_call(self) -> bool: return bool(self.tool_calls) - - def best_choice(self, normalized: bool = False) -> int: - """argmax over choice_logprobs (acc / acc_norm).""" - if not self.choice_logprobs: - return -1 - key = (lambda c: c.logprob_per_char if normalized else c.logprob) - return max(range(len(self.choice_logprobs)), key=lambda i: key(self.choice_logprobs[i])) diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index 8b82978..a098710 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -35,19 +35,16 @@ async def generate_predictions( max_turns: int = 8, max_input_chars: int = 0, attach_context_keys: tuple = ('passage', 'context'), - mcq_mode: str = 'auto', few_shot_num: int = 0, few_shot_samples: Optional[List[Sample]] = None, few_shot_text: Optional[str] = None, prompt_style: str = 'strict_letter', ) -> tuple: - """mcq_mode: 'logprob' (paper P1) -> adapter.score_choices per MCQ sample, - prediction = 'A'-'J' by argmax + acc_norm details in usage-like fields; - 'generate' (P3) -> normal chat generation with choices in prompt. - 'auto': logprob when the adapter exposes score_choices, else generate. + """Fan out model calls; returns (pred-dicts, total_usage). - few_shot: prepend N exemplars (question+answer lines) from few_shot_samples - (loaded by run_eval from DatasetSpec.few_shot_split). + MCQ samples are generated with the strict-letter contract ('ANSWER: X', + evalscope parity). few_shot: official exemplar text (few_shot_text) or + dev/train-split samples (few_shot_samples) are prepended. """ gen_kwargs = gen_kwargs or {} sem = asyncio.Semaphore(concurrency) @@ -102,27 +99,8 @@ async def generate_predictions( text = f'{head}\n\n...[truncated {len(text) - 2 * keep} chars]...\n\n{tail}' return text - use_logprob = mcq_mode == 'logprob' # 'auto'/'generate' -> P3 generation; - # paper-faithful P1 is explicit opt-in (mcq_mode='logprob') - async def one(sample: Sample) -> Dict[str, Any]: nonlocal done_count, total_usage - if use_logprob and sample.choices and not sample.tools: - letters = 'ABCDEFGH' - prompt = assemble(sample) - async with sem: - out = await adapter.score_choices(prompt, [str(c) for c in sample.choices]) - total_usage = total_usage + out.usage - best = out.best_choice(normalized=False) - best_norm = out.best_choice(normalized=True) - raw = f'logprob choice {letters[best] if best >= 0 else "?"} ' \ - f'(norm {letters[best_norm] if best_norm >= 0 else "?"})' - done_count += 1 - _progress(progress, done_count, len(samples), t0, total_usage) - return {'raw': raw, 'usage': out.usage.model_dump(), - 'logprob_best': best, 'logprob_best_norm': best_norm, - 'group_key': str(sample.metadata.get('test_category') - or sample.metadata.get('category') or sample.id or '')} if env_factory is not None: from ..agent import drive, trajectory_to_prediction @@ -190,7 +168,6 @@ async def run_eval( system: str = '', max_turns: int = 8, max_input_chars: int = 0, - mcq_mode: str = 'auto', few_shot_num: int = -1, prompt_style: str = 'strict_letter', ) -> EvalReport: @@ -199,7 +176,6 @@ async def run_eval( env: environment plugin name ('bfcl_mock') -> agent message pump per sample; omit for single-turn generation. - mcq_mode: 'logprob' = paper-faithful P1 scoring; 'generate' = P3 CoT; 'auto' = logprob when the adapter supports it. few_shot_num: -1 = the dataset's declared paper default (mmlu 5, bbh 3, gsm8k 4, ...); 0 = zero-shot; N = explicit override. @@ -267,7 +243,7 @@ async def run_eval( adapter, samples, concurrency, progress=progress, gen_kwargs=gen_kwargs, env_factory=env_factory, system=system, max_turns=max_turns, max_input_chars=max_input_chars, - mcq_mode=mcq_mode, few_shot_num=few_shot_num, + few_shot_num=few_shot_num, few_shot_samples=few_shot_samples, few_shot_text=few_shot_text, prompt_style=prompt_style) finally: