30 lines
844 B
Python
30 lines
844 B
Python
"""SimpleQA (OpenAI; official release is the CSV in github.com/openai/simple-evals).
|
|
|
|
The HF id below is the standard community mirror of that CSV.
|
|
"""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='simple_qa',
|
|
source='basicv8vc/SimpleQA', # mirror of the official openai/simple-evals CSV
|
|
split='test',
|
|
task_type='qa',
|
|
tags=['factuality'],
|
|
description='SimpleQA factuality benchmark (OpenAI, community mirror of official CSV).',
|
|
)
|
|
)
|
|
def simple_qa():
|
|
def to_sample(record: dict) -> Sample:
|
|
return Sample(
|
|
input=record['problem'],
|
|
target=str(record['answer']).strip(),
|
|
metadata={'metadata': record.get('metadata')},
|
|
)
|
|
|
|
return to_sample
|