Meta-Eval/EvalHarness
3
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Drop logprob scoring path entirely; MCQ = strict-letter generation (evalscope parity 'ANSWER: X'), few-shot defaults stay; fix output.py duplication

Browse Source
This commit is contained in:
sora 2026-08-25 02:26:26 +00:00
parent a8d3400ed5
commit d830e08923
4 changed files with 5 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
evalharness/eval/runner.py
View File

@ -84,18 +84,6 @@ def evaluate(
result.env_state = pred['env_state']
if isinstance(pred, dict) and pred.get('usage'):
result.usage = pred['usage']
if isinstance(pred, dict) and 'logprob_best' in pred:
letters = 'ABCDEFGH'
tgt = str(sample.target)
result.extracted_prediction = letters[pred['logprob_best']] \
if 0 <= pred['logprob_best'] < len(letters) else ''
result.extraction_ok = bool(result.extracted_prediction)
result.extraction_note = 'logprob_argmax'
result.score_details['acc_norm'] = {
'pred': letters[pred['logprob_best_norm']]
if 0 <= pred['logprob_best_norm'] < len(letters) else '',
'correct_norm': int(letters[pred['logprob_best_norm']] == tgt) if pred.get('logprob_best_norm', -1) >= 0 else 0,
}
try:
if bp is not None and sample.sandbox and sample.sandbox.image:
bp.ensure(sample.sandbox.image) # wait only if this one still pulling

134
evalharness/model/adapter.py
View File

@ -53,140 +53,6 @@ class ModelAdapter:
) -> ModelOutput:
raise NotImplementedError
async def score_choices(
self,
prompt: str,
choices: List[str],
) -> 'ModelOutput':
"""Paper-faithful MCQ scoring (P1): logprob of each choice continuing
the prompt. Strategy per backend capability:
completions_echo -- /completions echo+prompt_logprobs (vllm/sglang/
ollama); true continuation logprob, the lm-eh paper path
chat_first_token -- /chat/completions max_tokens=1 + top_logprobs; the
NEXT-token distribution encodes each choice; works on
every OpenAI-protocol API (cloud included); scores
choices as first-token probabilities
Selected via extra['logprob_strategy'] ('auto' default).
"""
strategy = self.extra.get('logprob_strategy', 'auto')
if strategy in ('auto', 'completions_echo'):
try:
return await self._score_choices_echo(prompt, choices)
except Exception as e:
if strategy == 'completions_echo':
raise
self.extra['logprob_strategy'] = 'chat_first_token' # downgrade once
return await self._score_choices_first_token(prompt, choices)
async def _score_choices_first_token(self, prompt: str, choices: List[str]):
"""Approximate P1 via next-token top_logprobs (any chat API).
The choices are shown in the prompt; we score P(first content token
of each choice | prompt) from the top_logprobs of a 1-token reply.
"""
from .output import ChoiceLogprob, ModelOutput
letters = 'ABCDEFGHIJ'
opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(choices))
asked = (f'{prompt}\n{opts}\n\n'
'Answer with only the letter of the correct option.')
payload = {'model': self.model, 'messages': [{'role': 'user', 'content': asked}],
'max_tokens': 1, 'temperature': 0.0,
'logprobs': True, 'top_logprobs': 20}
headers = {'Content-Type': 'application/json'}
if self.api_key:
headers['Authorization'] = f'Bearer {self.api_key}'
data = await self._post(f'{self.api_base}/chat/completions', payload, headers)
import json as _json
choice = (data.get('choices') or [{}])[0]
lp_wrapper = (choice.get('logprobs') or {})
top = lp_wrapper.get('content') or lp_wrapper.get('top_logprobs') or []
# top: list per generated token of {token: logprob} dicts
token_lps: Dict[str, float] = {}
first = top[0] if top else None
if isinstance(first, dict) and 'top_logprobs' in first:
# OpenAI/sglang chat shape: content[0] = {token, logprob, top_logprobs:[...]}
entries = {}
for e in (first.get('top_logprobs') or []):
tok, lp = e.get('token'), e.get('logprob')
if tok is not None and lp is not None:
entries[str(tok)] = lp
elif isinstance(first, dict):
entries = first # vllm-ish {token: lp} map
else:
entries = {}
for tok, lp in entries.items():
if isinstance(lp, dict):
lp = lp.get('logprob')
if lp is None:
continue
try:
token_lps[str(tok).strip().upper()] = float(lp)
except (TypeError, ValueError):
continue
if not token_lps:
raise RuntimeError(f'no top_logprobs from {self.api_base}; P1 needs a '
'backend exposing logprobs (or use mcq_mode=generate)')
results = []
for i in range(len(choices)):
letter = letters[i]
lp = token_lps.get(letter, -100.0)
results.append(ChoiceLogprob(text=choices[i], logprob=lp,
token_logprobs=[lp], num_tokens=1))
usage = Usage(finish_reason=choice.get('finish_reason', ''))
return ModelOutput(model=self.model, choice_logprobs=results, usage=usage)
async def _score_choices_echo(self, prompt: str, choices: List[str]):
from .output import ChoiceLogprob, ModelOutput
results: List[ChoiceLogprob] = []
for choice in choices:
full = f'{prompt} {choice}'
payload = {'model': self.model, 'prompt': full, 'max_tokens': 1,
'echo': True, 'prompt_logprobs': 0,
'temperature': 0.0}
headers = {'Content-Type': 'application/json'}
if self.api_key:
headers['Authorization'] = f'Bearer {self.api_key}'
data = await self._post_raw(f'{self.api_base}/completions', payload, headers)
# vllm/sglang shape: choices[0].prompt_logprobs: list[None|{token: lp}]
import json as _json
ch = (data.get('choices') or [{}])[0]
pl = ch.get('prompt_logprobs') or []
prompt_ids = ch.get('prompt_token_ids') or []
# tokens belonging to the choice = the tail after the prompt part.
# echo mode returns the whole sequence; we approximate the choice
# token count via the tail offset encoded in logprob entries.
lps: List[float] = []
for entry in pl:
if isinstance(entry, dict):
lp = next(iter(entry.values()))
if isinstance(lp, dict):
lp = lp.get('logprob', 0.0)
lps.append(float(lp))
if not lps:
raise RuntimeError(
f'backend at {self.api_base} returned no prompt_logprobs; '
'P1 logprob scoring needs vllm/sglang-style completions-echo'
)
# last len tokens we cannot know exactly; use the trailing segment
# matching the choice's char proportion as a robust approximation
n_prompt_chars = len(prompt) + 1
frac = max(len(choice), 1) / max(len(full), 1)
n_choice = max(1, round(frac * len(lps)))
choice_lps = lps[-n_choice:]
results.append(ChoiceLogprob(
text=choice, logprob=sum(choice_lps),
token_logprobs=choice_lps, num_tokens=len(choice_lps)))
return ModelOutput(model=self.model, choice_logprobs=results)
async def _post_raw(self, url: str, payload: Dict, headers: Dict) -> Dict[str, Any]:
return await self._post(url, payload, headers)
async def close(self) -> None:
pass

23
evalharness/model/output.py
View File

@ -44,21 +44,6 @@ class Usage(BaseModel):
)
class ChoiceLogprob(BaseModel):
"""Continuation scoring for one choice (paper-faithful MCQ evaluation)."""
text: str = ''
logprob: float = 0.0 # total logprob of the choice tokens
token_logprobs: List[float] = Field(default_factory=list)
num_tokens: int = 0
@property
def logprob_per_char(self) -> float:
"""lm-eval-harness acc_norm uses per-BYTE normalization; chars are the
practical proxy (identical ranking for ascii-dominant choices)."""
return self.logprob / max(len(self.text), 1)
class ModelOutput(BaseModel):
"""What every adapter returns. text may be '' when the model only calls tools."""
@ -68,15 +53,7 @@ class ModelOutput(BaseModel):
raw: Optional[Dict[str, Any]] = None # provider response (audit/retry)
model: str = ''
created_at: str = Field(default_factory=lambda: time.strftime('%Y-%m-%d %H:%M:%S'))
choice_logprobs: Optional[List[ChoiceLogprob]] = None # P1 continuation scoring
@property
def is_tool_call(self) -> bool:
return bool(self.tool_calls)
def best_choice(self, normalized: bool = False) -> int:
"""argmax over choice_logprobs (acc / acc_norm)."""
if not self.choice_logprobs:
return -1
key = (lambda c: c.logprob_per_char if normalized else c.logprob)
return max(range(len(self.choice_logprobs)), key=lambda i: key(self.choice_logprobs[i]))

34
evalharness/model/runner.py
View File

@ -35,19 +35,16 @@ async def generate_predictions(
max_turns: int = 8,
max_input_chars: int = 0,
attach_context_keys: tuple = ('passage', 'context'),
mcq_mode: str = 'auto',
few_shot_num: int = 0,
few_shot_samples: Optional[List[Sample]] = None,
few_shot_text: Optional[str] = None,
prompt_style: str = 'strict_letter',
) -> tuple:
"""mcq_mode: 'logprob' (paper P1) -> adapter.score_choices per MCQ sample,
prediction = 'A'-'J' by argmax + acc_norm details in usage-like fields;
'generate' (P3) -> normal chat generation with choices in prompt.
'auto': logprob when the adapter exposes score_choices, else generate.
"""Fan out model calls; returns (pred-dicts, total_usage).
few_shot: prepend N exemplars (question+answer lines) from few_shot_samples
(loaded by run_eval from DatasetSpec.few_shot_split).
MCQ samples are generated with the strict-letter contract ('ANSWER: X',
evalscope parity). few_shot: official exemplar text (few_shot_text) or
dev/train-split samples (few_shot_samples) are prepended.
"""
gen_kwargs = gen_kwargs or {}
sem = asyncio.Semaphore(concurrency)
@ -102,27 +99,8 @@ async def generate_predictions(
text = f'{head}\n\n...[truncated {len(text) - 2 * keep} chars]...\n\n{tail}'
return text
use_logprob = mcq_mode == 'logprob' # 'auto'/'generate' -> P3 generation;
# paper-faithful P1 is explicit opt-in (mcq_mode='logprob')
async def one(sample: Sample) -> Dict[str, Any]:
nonlocal done_count, total_usage
if use_logprob and sample.choices and not sample.tools:
letters = 'ABCDEFGH'
prompt = assemble(sample)
async with sem:
out = await adapter.score_choices(prompt, [str(c) for c in sample.choices])
total_usage = total_usage + out.usage
best = out.best_choice(normalized=False)
best_norm = out.best_choice(normalized=True)
raw = f'logprob choice {letters[best] if best >= 0 else "?"} ' \
f'(norm {letters[best_norm] if best_norm >= 0 else "?"})'
done_count += 1
_progress(progress, done_count, len(samples), t0, total_usage)
return {'raw': raw, 'usage': out.usage.model_dump(),
'logprob_best': best, 'logprob_best_norm': best_norm,
'group_key': str(sample.metadata.get('test_category')
or sample.metadata.get('category') or sample.id or '')}
if env_factory is not None:
from ..agent import drive, trajectory_to_prediction
@ -190,7 +168,6 @@ async def run_eval(
system: str = '',
max_turns: int = 8,
max_input_chars: int = 0,
mcq_mode: str = 'auto',
few_shot_num: int = -1,
prompt_style: str = 'strict_letter',
) -> EvalReport:
@ -199,7 +176,6 @@ async def run_eval(
env: environment plugin name ('bfcl_mock') -> agent message pump per
sample; omit for single-turn generation.
mcq_mode: 'logprob' = paper-faithful P1 scoring; 'generate' = P3 CoT;
'auto' = logprob when the adapter supports it.
few_shot_num: -1 = the dataset's declared paper default (mmlu 5, bbh 3,
gsm8k 4, ...); 0 = zero-shot; N = explicit override.
@ -267,7 +243,7 @@ async def run_eval(
adapter, samples, concurrency, progress=progress,
gen_kwargs=gen_kwargs, env_factory=env_factory,
system=system, max_turns=max_turns, max_input_chars=max_input_chars,
mcq_mode=mcq_mode, few_shot_num=few_shot_num,
few_shot_num=few_shot_num,
few_shot_samples=few_shot_samples, few_shot_text=few_shot_text,
prompt_style=prompt_style)
finally: