370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
37 lines
1.4 KiB
Python
37 lines
1.4 KiB
Python
"""LongBench v2 (official source: THUDM/LongBench-v2)."""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='longbench_v2',
|
|
source='THUDM/LongBench-v2', # official: https://huggingface.co/datasets/THUDM/LongBench-v2
|
|
split='train', # the dataset ships a single split
|
|
params={'filter_column': 'length'},
|
|
prompt_style='lb2_es', # es <text> wrapper + CoT contract # subset selects length: short/medium/long
|
|
task_type='mcq',
|
|
tags=['long_context'],
|
|
description='LongBench v2 long-context MCQ (official). Context kept in metadata.',
|
|
)
|
|
)
|
|
def longbench_v2():
|
|
def to_sample(record: dict) -> Sample:
|
|
return Sample(
|
|
input=record['question'],
|
|
choices=[record['choice_A'], record['choice_B'], record['choice_C'], record['choice_D']],
|
|
target=str(record['answer']).strip(),
|
|
metadata={
|
|
'context': record['context'], # the long document; eval-time prompt assembly
|
|
'domain': record.get('domain'),
|
|
'sub_domain': record.get('sub_domain'),
|
|
'difficulty': record.get('difficulty'),
|
|
'length': record.get('length'),
|
|
'subset': record.get('length'), # official subsets: short/medium/long
|
|
},
|
|
)
|
|
|
|
return to_sample
|