GPQA deterministic per-question choice shuffle (es-adapter parity: sha256(question) seed, position-bias protection) — was fixed-A; lb2 subset grouping aligned to es (length field, 60 samples)

evalharness/data/datasets/gpqa_diamond.py

@@ -28,17 +28,26 @@ from ..spec import DatasetSpec
 )
 def gpqa_diamond():
     def to_sample(record: dict) -> Sample:
+        # position-bias protection, ported from the es adapter: deterministic
+        # per-question shuffle (seed = sha256(question)) keeps reruns identical
+        import hashlib
+        import random as _rnd
+
         choices = [
-            record['Correct Answer'],
-            record['Incorrect Answer 1'],
-            record['Incorrect Answer 2'],
-            record['Incorrect Answer 3'],
+            str(record['Incorrect Answer 1'] or '').strip(),
+            str(record['Incorrect Answer 2'] or '').strip(),
+            str(record['Incorrect Answer 3'] or '').strip(),
+            str(record['Correct Answer'] or '').strip(),
         ]
+        seed = int.from_bytes(
+            hashlib.sha256(str(record['Question']).strip().encode('utf-8')).digest()[:8], 'big')
+        _rnd.Random(seed).shuffle(choices)
+        target = 'ABCD'[choices.index(str(record['Correct Answer'] or '').strip())]
         return Sample(
             input=record['Question'],
             choices=choices,
-            target='A',  # correct answer is first; shuffle at eval time
-            metadata={'subdomain': record.get('Subdomain'), 'unshuffled': True},
+            target=target,
+            metadata={'subdomain': record.get('Subdomain')},
         )
 
     return to_sample

evalharness/data/datasets/longbench_v2.py

@@ -27,6 +27,7 @@ def longbench_v2():
                 'sub_domain': record.get('sub_domain'),
                 'difficulty': record.get('difficulty'),
                 'length': record.get('length'),
+                'subset': record.get('length'),  # official subsets: short/medium/long
             },
         )
 

evalharness/model/runner.py

@@ -271,9 +271,11 @@ def _apply_limits(samples: List[Sample], total: Optional[int],
         seen: Dict[str, int] = {}
         out = []
         for s in samples:
-            key = str((s.metadata or {}).get('category')
+            key = str((s.metadata or {}).get('subset')
+                      or (s.metadata or {}).get('category')
                       or (s.metadata or {}).get('subject')
                       or (s.metadata or {}).get('test_category')
+                      or (s.metadata or {}).get('length')
                       or getattr(getattr(dataset, 'spec', None), 'subset', 'default'))
             if seen.get(key, 0) < per_task:
                 seen[key] = seen.get(key, 0) + 1