GPQA 0.15->0.55: (1) per-bench prompt_style cot_letter (es GPQA CoT template, matches its 'Think step by step' contract), (2) mcq_letter extractor case-insensitive answer-tail (ANSWER: vs answer), (3) deterministic choice shuffle

evalharness/data/datasets/gpqa_diamond.py

@@ -20,6 +20,7 @@ from ..spec import DatasetSpec
         source='nmayorga7/gpqa_diamond',  # HF CSV export of the gated official
         split='train',
         gen_config={'temperature': 1.0, 'max_tokens': 8192},
+        prompt_style='cot_letter',  # CoT then ANSWER:X (es GPQA template)
         task_type='mcq',
         tags=['knowledge', 'science'],
         description='GPQA diamond split, graduate-level science MCQ (official content).',

evalharness/data/spec.py

@@ -39,6 +39,7 @@ class DatasetSpec:
     few_shot_split: Optional[str] = None       # e.g. 'dev' (paper-faithful exemplars)
     few_shot_num: int = 0                      # paper default shots (mmlu=5, bbh=3, ...)
     gen_config: dict = field(default_factory=dict)  # per-bench generation params
-    prompt_suffix: str = ''   # appended to the question (e.g. boxed{} CoT directive)
+    prompt_suffix: str = ''
+    prompt_style: str = ''    # ''=default; 'cot_letter'=CoT then ANSWER:X   # appended to the question (e.g. boxed{} CoT directive)
                                              # (temperature/max_tokens/top_p), consumed
                                              # by run_eval unless overridden

evalharness/eval/extractor.py

@@ -185,7 +185,8 @@ _LETTER_CN = re.compile(r'答案是\s*\(?([A-J])\)?', re.IGNORECASE)
 def mcq_letter(raw: str, sample: Sample) -> Tuple[str, bool, str]:
     """Multiple-choice letter: prefer (A) style, then 答案是X, then bare A."""
     text = raw or ''
-    tail = text[text.rfind('answer'):] if 'answer' in text.lower() else text
+    lower = text.lower()
+    tail = text[lower.rfind('answer'):] if 'answer' in lower else text
     m = None
     for m in _LETTER_PAREN.finditer(tail):
         pass

evalharness/model/runner.py

@@ -81,7 +81,16 @@ async def generate_predictions(
             if ctx:
                 parts.append(str(ctx))
         question = sample.input_text
-        if sample.choices:
+        spec_style = getattr(dataset_spec, 'prompt_style', '') if dataset_spec is not None else ''
+        if sample.choices and spec_style == 'cot_letter':
+            letters = 'ABCDEFGH'
+            opts = '\n'.join(f'{letters[i]}. {c}' for i, c in enumerate(sample.choices)
+                             if i < len(letters))
+            question = (f'Answer the following multiple choice question. The last line of '
+                        f"your response should be of the following format: 'ANSWER: [LETTER]' "
+                        f'(without quotes) where [LETTER] is one of {letters[:len(sample.choices)]}. '
+                        f'Think step by step before answering.\n\n{question}\n\n{opts}')
+        elif sample.choices:
             if prompt_style in ('strict_letter', 'auto'):
                 # evalscope/OpenAI-style contract: reply ONLY 'ANSWER: X'
                 letters = 'ABCDEFGHIJ'