Drop logprob scoring path entirely; MCQ = strict-letter generation (evalscope parity 'ANSWER: X'), few-shot defaults stay; fix output.py duplication

evalharness/eval/runner.py

@@ -84,18 +84,6 @@ def evaluate(
             result.env_state = pred['env_state']
         if isinstance(pred, dict) and pred.get('usage'):
             result.usage = pred['usage']
-        if isinstance(pred, dict) and 'logprob_best' in pred:
-            letters = 'ABCDEFGH'
-            tgt = str(sample.target)
-            result.extracted_prediction = letters[pred['logprob_best']] \
-                if 0 <= pred['logprob_best'] < len(letters) else ''
-            result.extraction_ok = bool(result.extracted_prediction)
-            result.extraction_note = 'logprob_argmax'
-            result.score_details['acc_norm'] = {
-                'pred': letters[pred['logprob_best_norm']]
-                if 0 <= pred['logprob_best_norm'] < len(letters) else '',
-                'correct_norm': int(letters[pred['logprob_best_norm']] == tgt) if pred.get('logprob_best_norm', -1) >= 0 else 0,
-            }
         try:
             if bp is not None and sample.sandbox and sample.sandbox.image:
                 bp.ensure(sample.sandbox.image)  # wait only if this one still pulling

evalharness/model/adapter.py

@@ -53,140 +53,6 @@ class ModelAdapter:
     ) -> ModelOutput:
         raise NotImplementedError
 
-    async def score_choices(
-        self,
-        prompt: str,
-        choices: List[str],
-    ) -> 'ModelOutput':
-        """Paper-faithful MCQ scoring (P1): logprob of each choice continuing
-        the prompt. Strategy per backend capability:
-
-        completions_echo  -- /completions echo+prompt_logprobs (vllm/sglang/
-                            ollama); true continuation logprob, the lm-eh paper path
-        chat_first_token  -- /chat/completions max_tokens=1 + top_logprobs; the
-                            NEXT-token distribution encodes each choice; works on
-                            every OpenAI-protocol API (cloud included); scores
-                            choices as first-token probabilities
-
-        Selected via extra['logprob_strategy'] ('auto' default).
-        """
-        strategy = self.extra.get('logprob_strategy', 'auto')
-        if strategy in ('auto', 'completions_echo'):
-            try:
-                return await self._score_choices_echo(prompt, choices)
-            except Exception as e:
-                if strategy == 'completions_echo':
-                    raise
-                self.extra['logprob_strategy'] = 'chat_first_token'  # downgrade once
-        return await self._score_choices_first_token(prompt, choices)
-
-    async def _score_choices_first_token(self, prompt: str, choices: List[str]):
-        """Approximate P1 via next-token top_logprobs (any chat API).
-
-        The choices are shown in the prompt; we score P(first content token
-        of each choice | prompt) from the top_logprobs of a 1-token reply.
-        """
-        from .output import ChoiceLogprob, ModelOutput
-
-        letters = 'ABCDEFGHIJ'
-        opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(choices))
-        asked = (f'{prompt}\n{opts}\n\n'
-                 'Answer with only the letter of the correct option.')
-        payload = {'model': self.model, 'messages': [{'role': 'user', 'content': asked}],
-                   'max_tokens': 1, 'temperature': 0.0,
-                   'logprobs': True, 'top_logprobs': 20}
-        headers = {'Content-Type': 'application/json'}
-        if self.api_key:
-            headers['Authorization'] = f'Bearer {self.api_key}'
-        data = await self._post(f'{self.api_base}/chat/completions', payload, headers)
-        import json as _json
-
-        choice = (data.get('choices') or [{}])[0]
-        lp_wrapper = (choice.get('logprobs') or {})
-        top = lp_wrapper.get('content') or lp_wrapper.get('top_logprobs') or []
-        # top: list per generated token of {token: logprob} dicts
-        token_lps: Dict[str, float] = {}
-        first = top[0] if top else None
-        if isinstance(first, dict) and 'top_logprobs' in first:
-            # OpenAI/sglang chat shape: content[0] = {token, logprob, top_logprobs:[...]}
-            entries = {}
-            for e in (first.get('top_logprobs') or []):
-                tok, lp = e.get('token'), e.get('logprob')
-                if tok is not None and lp is not None:
-                    entries[str(tok)] = lp
-        elif isinstance(first, dict):
-            entries = first  # vllm-ish {token: lp} map
-        else:
-            entries = {}
-        for tok, lp in entries.items():
-            if isinstance(lp, dict):
-                lp = lp.get('logprob')
-            if lp is None:
-                continue
-            try:
-                token_lps[str(tok).strip().upper()] = float(lp)
-            except (TypeError, ValueError):
-                continue
-        if not token_lps:
-            raise RuntimeError(f'no top_logprobs from {self.api_base}; P1 needs a '
-                               'backend exposing logprobs (or use mcq_mode=generate)')
-        results = []
-        for i in range(len(choices)):
-            letter = letters[i]
-            lp = token_lps.get(letter, -100.0)
-            results.append(ChoiceLogprob(text=choices[i], logprob=lp,
-                                         token_logprobs=[lp], num_tokens=1))
-        usage = Usage(finish_reason=choice.get('finish_reason', ''))
-        return ModelOutput(model=self.model, choice_logprobs=results, usage=usage)
-
-    async def _score_choices_echo(self, prompt: str, choices: List[str]):
-        from .output import ChoiceLogprob, ModelOutput
-
-        results: List[ChoiceLogprob] = []
-        for choice in choices:
-            full = f'{prompt} {choice}'
-            payload = {'model': self.model, 'prompt': full, 'max_tokens': 1,
-                       'echo': True, 'prompt_logprobs': 0,
-                       'temperature': 0.0}
-            headers = {'Content-Type': 'application/json'}
-            if self.api_key:
-                headers['Authorization'] = f'Bearer {self.api_key}'
-            data = await self._post_raw(f'{self.api_base}/completions', payload, headers)
-            # vllm/sglang shape: choices[0].prompt_logprobs: list[None|{token: lp}]
-            import json as _json
-
-            ch = (data.get('choices') or [{}])[0]
-            pl = ch.get('prompt_logprobs') or []
-            prompt_ids = ch.get('prompt_token_ids') or []
-            # tokens belonging to the choice = the tail after the prompt part.
-            # echo mode returns the whole sequence; we approximate the choice
-            # token count via the tail offset encoded in logprob entries.
-            lps: List[float] = []
-            for entry in pl:
-                if isinstance(entry, dict):
-                    lp = next(iter(entry.values()))
-                    if isinstance(lp, dict):
-                        lp = lp.get('logprob', 0.0)
-                    lps.append(float(lp))
-            if not lps:
-                raise RuntimeError(
-                    f'backend at {self.api_base} returned no prompt_logprobs; '
-                    'P1 logprob scoring needs vllm/sglang-style completions-echo'
-                )
-            # last len tokens we cannot know exactly; use the trailing segment
-            # matching the choice's char proportion as a robust approximation
-            n_prompt_chars = len(prompt) + 1
-            frac = max(len(choice), 1) / max(len(full), 1)
-            n_choice = max(1, round(frac * len(lps)))
-            choice_lps = lps[-n_choice:]
-            results.append(ChoiceLogprob(
-                text=choice, logprob=sum(choice_lps),
-                token_logprobs=choice_lps, num_tokens=len(choice_lps)))
-        return ModelOutput(model=self.model, choice_logprobs=results)
-
-    async def _post_raw(self, url: str, payload: Dict, headers: Dict) -> Dict[str, Any]:
-        return await self._post(url, payload, headers)
-
     async def close(self) -> None:
         pass
 

evalharness/model/output.py

@@ -44,21 +44,6 @@ class Usage(BaseModel):
         )
 
 
-class ChoiceLogprob(BaseModel):
-    """Continuation scoring for one choice (paper-faithful MCQ evaluation)."""
-
-    text: str = ''
-    logprob: float = 0.0            # total logprob of the choice tokens
-    token_logprobs: List[float] = Field(default_factory=list)
-    num_tokens: int = 0
-
-    @property
-    def logprob_per_char(self) -> float:
-        """lm-eval-harness acc_norm uses per-BYTE normalization; chars are the
-        practical proxy (identical ranking for ascii-dominant choices)."""
-        return self.logprob / max(len(self.text), 1)
-
-
 class ModelOutput(BaseModel):
     """What every adapter returns. text may be '' when the model only calls tools."""
 
@@ -68,15 +53,7 @@ class ModelOutput(BaseModel):
     raw: Optional[Dict[str, Any]] = None    # provider response (audit/retry)
     model: str = ''
     created_at: str = Field(default_factory=lambda: time.strftime('%Y-%m-%d %H:%M:%S'))
-    choice_logprobs: Optional[List[ChoiceLogprob]] = None  # P1 continuation scoring
 
     @property
     def is_tool_call(self) -> bool:
         return bool(self.tool_calls)
-
-    def best_choice(self, normalized: bool = False) -> int:
-        """argmax over choice_logprobs (acc / acc_norm)."""
-        if not self.choice_logprobs:
-            return -1
-        key = (lambda c: c.logprob_per_char if normalized else c.logprob)
-        return max(range(len(self.choice_logprobs)), key=lambda i: key(self.choice_logprobs[i]))

evalharness/model/runner.py

@@ -35,19 +35,16 @@ async def generate_predictions(
     max_turns: int = 8,
     max_input_chars: int = 0,
     attach_context_keys: tuple = ('passage', 'context'),
-    mcq_mode: str = 'auto',
     few_shot_num: int = 0,
     few_shot_samples: Optional[List[Sample]] = None,
     few_shot_text: Optional[str] = None,
     prompt_style: str = 'strict_letter',
 ) -> tuple:
-    """mcq_mode: 'logprob' (paper P1) -> adapter.score_choices per MCQ sample,
+    """Fan out model calls; returns (pred-dicts, total_usage).
-    prediction = 'A'-'J' by argmax + acc_norm details in usage-like fields;
-    'generate' (P3) -> normal chat generation with choices in prompt.
-    'auto': logprob when the adapter exposes score_choices, else generate.
 
-    few_shot: prepend N exemplars (question+answer lines) from few_shot_samples
+    MCQ samples are generated with the strict-letter contract ('ANSWER: X',
-    (loaded by run_eval from DatasetSpec.few_shot_split).
+    evalscope parity). few_shot: official exemplar text (few_shot_text) or
+    dev/train-split samples (few_shot_samples) are prepended.
     """
     gen_kwargs = gen_kwargs or {}
     sem = asyncio.Semaphore(concurrency)
@@ -102,27 +99,8 @@ async def generate_predictions(
             text = f'{head}\n\n...[truncated {len(text) - 2 * keep} chars]...\n\n{tail}'
         return text
 
-    use_logprob = mcq_mode == 'logprob'  # 'auto'/'generate' -> P3 generation;
-    # paper-faithful P1 is explicit opt-in (mcq_mode='logprob')
-
     async def one(sample: Sample) -> Dict[str, Any]:
         nonlocal done_count, total_usage
-        if use_logprob and sample.choices and not sample.tools:
-            letters = 'ABCDEFGH'
-            prompt = assemble(sample)
-            async with sem:
-                out = await adapter.score_choices(prompt, [str(c) for c in sample.choices])
-            total_usage = total_usage + out.usage
-            best = out.best_choice(normalized=False)
-            best_norm = out.best_choice(normalized=True)
-            raw = f'logprob choice {letters[best] if best >= 0 else "?"} ' \
-                  f'(norm {letters[best_norm] if best_norm >= 0 else "?"})'
-            done_count += 1
-            _progress(progress, done_count, len(samples), t0, total_usage)
-            return {'raw': raw, 'usage': out.usage.model_dump(),
-                    'logprob_best': best, 'logprob_best_norm': best_norm,
-                    'group_key': str(sample.metadata.get('test_category')
-                                     or sample.metadata.get('category') or sample.id or '')}
         if env_factory is not None:
             from ..agent import drive, trajectory_to_prediction
 
@@ -190,7 +168,6 @@ async def run_eval(
     system: str = '',
     max_turns: int = 8,
     max_input_chars: int = 0,
-    mcq_mode: str = 'auto',
     few_shot_num: int = -1,
     prompt_style: str = 'strict_letter',
 ) -> EvalReport:
@@ -199,7 +176,6 @@ async def run_eval(
 
     env: environment plugin name ('bfcl_mock') -> agent message pump per
     sample; omit for single-turn generation.
-    mcq_mode: 'logprob' = paper-faithful P1 scoring; 'generate' = P3 CoT;
     'auto' = logprob when the adapter supports it.
     few_shot_num: -1 = the dataset's declared paper default (mmlu 5, bbh 3,
     gsm8k 4, ...); 0 = zero-shot; N = explicit override.
@@ -267,7 +243,7 @@ async def run_eval(
             adapter, samples, concurrency, progress=progress,
             gen_kwargs=gen_kwargs, env_factory=env_factory,
             system=system, max_turns=max_turns, max_input_chars=max_input_chars,
-            mcq_mode=mcq_mode, few_shot_num=few_shot_num,
+            few_shot_num=few_shot_num,
             few_shot_samples=few_shot_samples, few_shot_text=few_shot_text,
             prompt_style=prompt_style)
     finally: