From 60b2bfc0dc06ccdc33ae1a07749b36f0826314da Mon Sep 17 00:00:00 2001
From: sora <Wanght@meta-stone.com>
Date: Fri, 28 Aug 2026 16:24:11 +0000
Subject: [PATCH] =?UTF-8?q?GPQA=20deterministic=20per-question=20choice=20?=
 =?UTF-8?q?shuffle=20(es-adapter=20parity:=20sha256(question)=20seed,=20po?=
 =?UTF-8?q?sition-bias=20protection)=20=E2=80=94=20was=20fixed-A;=20lb2=20?=
 =?UTF-8?q?subset=20grouping=20aligned=20to=20es=20(length=20field,=2060?=
 =?UTF-8?q?=20samples)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 evalharness/data/datasets/gpqa_diamond.py | 21 +++++++++++++++------
 evalharness/data/datasets/longbench_v2.py |  1 +
 evalharness/model/runner.py               |  4 +++-
 3 files changed, 19 insertions(+), 7 deletions(-)

diff --git a/evalharness/data/datasets/gpqa_diamond.py b/evalharness/data/datasets/gpqa_diamond.py
index 7d6f8fe..7331887 100644
--- a/evalharness/data/datasets/gpqa_diamond.py
+++ b/evalharness/data/datasets/gpqa_diamond.py
@@ -28,17 +28,26 @@ from ..spec import DatasetSpec
 )
 def gpqa_diamond():
     def to_sample(record: dict) -> Sample:
+        # position-bias protection, ported from the es adapter: deterministic
+        # per-question shuffle (seed = sha256(question)) keeps reruns identical
+        import hashlib
+        import random as _rnd
+
         choices = [
-            record['Correct Answer'],
-            record['Incorrect Answer 1'],
-            record['Incorrect Answer 2'],
-            record['Incorrect Answer 3'],
+            str(record['Incorrect Answer 1'] or '').strip(),
+            str(record['Incorrect Answer 2'] or '').strip(),
+            str(record['Incorrect Answer 3'] or '').strip(),
+            str(record['Correct Answer'] or '').strip(),
         ]
+        seed = int.from_bytes(
+            hashlib.sha256(str(record['Question']).strip().encode('utf-8')).digest()[:8], 'big')
+        _rnd.Random(seed).shuffle(choices)
+        target = 'ABCD'[choices.index(str(record['Correct Answer'] or '').strip())]
         return Sample(
             input=record['Question'],
             choices=choices,
-            target='A',  # correct answer is first; shuffle at eval time
-            metadata={'subdomain': record.get('Subdomain'), 'unshuffled': True},
+            target=target,
+            metadata={'subdomain': record.get('Subdomain')},
         )
 
     return to_sample
diff --git a/evalharness/data/datasets/longbench_v2.py b/evalharness/data/datasets/longbench_v2.py
index 27c7c1b..2b5466c 100644
--- a/evalharness/data/datasets/longbench_v2.py
+++ b/evalharness/data/datasets/longbench_v2.py
@@ -27,6 +27,7 @@ def longbench_v2():
                 'sub_domain': record.get('sub_domain'),
                 'difficulty': record.get('difficulty'),
                 'length': record.get('length'),
+                'subset': record.get('length'),  # official subsets: short/medium/long
             },
         )
 
diff --git a/evalharness/model/runner.py b/evalharness/model/runner.py
index 4e51fea..e6867a6 100644
--- a/evalharness/model/runner.py
+++ b/evalharness/model/runner.py
@@ -271,9 +271,11 @@ def _apply_limits(samples: List[Sample], total: Optional[int],
         seen: Dict[str, int] = {}
         out = []
         for s in samples:
-            key = str((s.metadata or {}).get('category')
+            key = str((s.metadata or {}).get('subset')
+                      or (s.metadata or {}).get('category')
                       or (s.metadata or {}).get('subject')
                       or (s.metadata or {}).get('test_category')
+                      or (s.metadata or {}).get('length')
                       or getattr(getattr(dataset, 'spec', None), 'subset', 'default'))
             if seen.get(key, 0) < per_task:
                 seen[key] = seen.get(key, 0) + 1