38 lines
1.1 KiB
Python
38 lines
1.1 KiB
Python
"""HLE - Humanity's Last Exam.
|
|
|
|
Official source cais/hle is gated on HF (needs auth); we load the ModelScope
|
|
mirror of the identical data via the native raw-file downloader.
|
|
"""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='hle',
|
|
source='cais/hle', # ModelScope mirror of the gated HF original
|
|
split='test',
|
|
task_type='qa',
|
|
tags=['knowledge', 'frontier'],
|
|
description="Humanity's Last Exam. Some samples carry an image field (multimodal).",
|
|
params={'hub': 'modelscope'},
|
|
)
|
|
)
|
|
def hle():
|
|
def to_sample(record: dict) -> Sample:
|
|
return Sample(
|
|
input=record['question'],
|
|
target=str(record['answer']).strip(),
|
|
metadata={
|
|
'id': record.get('id'),
|
|
'answer_type': record.get('answer_type'),
|
|
'category': record.get('category'),
|
|
'raw_subject': record.get('raw_subject'),
|
|
'has_image': bool(record.get('image')),
|
|
},
|
|
)
|
|
|
|
return to_sample
|