diff --git a/evalharness/data/datasets/gpqa_diamond.py b/evalharness/data/datasets/gpqa_diamond.py index 7331887..d8ab6c2 100644 --- a/evalharness/data/datasets/gpqa_diamond.py +++ b/evalharness/data/datasets/gpqa_diamond.py @@ -20,6 +20,7 @@ from ..spec import DatasetSpec source='nmayorga7/gpqa_diamond', # HF CSV export of the gated official split='train', gen_config={'temperature': 1.0, 'max_tokens': 8192}, + prompt_style='cot_letter', # CoT then ANSWER:X (es GPQA template) task_type='mcq', tags=['knowledge', 'science'], description='GPQA diamond split, graduate-level science MCQ (official content).', diff --git a/evalharness/data/spec.py b/evalharness/data/spec.py index eedad09..b6073bb 100644 --- a/evalharness/data/spec.py +++ b/evalharness/data/spec.py @@ -39,6 +39,7 @@ class DatasetSpec: few_shot_split: Optional[str] = None # e.g. 'dev' (paper-faithful exemplars) few_shot_num: int = 0 # paper default shots (mmlu=5, bbh=3, ...) gen_config: dict = field(default_factory=dict) # per-bench generation params - prompt_suffix: str = '' # appended to the question (e.g. boxed{} CoT directive) + prompt_suffix: str = '' + prompt_style: str = '' # ''=default; 'cot_letter'=CoT then ANSWER:X # appended to the question (e.g. boxed{} CoT directive) # (temperature/max_tokens/top_p), consumed # by run_eval unless overridden diff --git a/evalharness/eval/extractor.py b/evalharness/eval/extractor.py index 0101055..d9a4289 100644 --- a/evalharness/eval/extractor.py +++ b/evalharness/eval/extractor.py @@ -185,7 +185,8 @@ _LETTER_CN = re.compile(r'答案是\s*\(?([A-J])\)?', re.IGNORECASE) def mcq_letter(raw: str, sample: Sample) -> Tuple[str, bool, str]: """Multiple-choice letter: prefer (A) style, then 答案是X, then bare A.""" text = raw or '' - tail = text[text.rfind('answer'):] if 'answer' in text.lower() else text + lower = text.lower() + tail = text[lower.rfind('answer'):] if 'answer' in lower else text m = None for m in _LETTER_PAREN.finditer(tail): pass diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py index e6867a6..cb3d655 100644 --- a/evalharness/model/runner.py +++ b/evalharness/model/runner.py @@ -81,7 +81,16 @@ async def generate_predictions( if ctx: parts.append(str(ctx)) question = sample.input_text - if sample.choices: + spec_style = getattr(dataset_spec, 'prompt_style', '') if dataset_spec is not None else '' + if sample.choices and spec_style == 'cot_letter': + letters = 'ABCDEFGH' + opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(sample.choices) + if i < len(letters)) + question = (f'Answer the following multiple choice question. The last line of ' + f"your response should be of the following format: 'ANSWER: [LETTER]' " + f'(without quotes) where [LETTER] is one of {letters[:len(sample.choices)]}. ' + f'Think step by step before answering.\n\n{question}\n\n{opts}') + elif sample.choices: if prompt_style in ('strict_letter', 'auto'): # evalscope/OpenAI-style contract: reply ONLY 'ANSWER: X' letters = 'ABCDEFGHIJ'