46 lines
1.7 KiB
Python
46 lines
1.7 KiB
Python
"""TriviaQA (official source: mandarjoshi/trivia_qa, rc.wikipedia config).
|
|
|
|
rc.wikipedia = reading-comprehension WITH the Wikipedia evidence document
|
|
(open-book, aligned with evalscope's default); rc.nocontext is the
|
|
closed-book variant (pass --subset rc.nocontext).
|
|
"""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='trivia_qa',
|
|
source='mandarjoshi/trivia_qa', # official: https://huggingface.co/datasets/mandarjoshi/trivia_qa
|
|
subset='rc.wikipedia', # open-book (evalscope parity); --subset rc.nocontext for closed
|
|
split='validation',
|
|
task_type='qa',
|
|
tags=['knowledge', 'openqa'],
|
|
description='TriviaQA with Wikipedia evidence (open-book); any alias counts.',
|
|
)
|
|
)
|
|
def trivia_qa():
|
|
def to_sample(record: dict) -> Sample:
|
|
answer = record['answer'] # {'value': ..., 'aliases': [...], ...}
|
|
targets = [answer['value']] + list(answer.get('aliases') or [])
|
|
# open-book: the Wikipedia evidence document (runner prepends it via
|
|
# metadata['context'] when assembling the prompt)
|
|
wiki = ''
|
|
entity = record.get('entity_pages') or {}
|
|
for doc in (entity.get('wiki_content') or [])[:1]:
|
|
wiki = doc or ''
|
|
break
|
|
search = record.get('search_results') or {}
|
|
if not wiki:
|
|
wiki = '\n'.join((search.get('search_context') or [])[:2])
|
|
return Sample(
|
|
input=record['question'],
|
|
target=targets, # multi-target: any alias counts
|
|
metadata={'question_id': record.get('question_id'),
|
|
'context': wiki or None},
|
|
)
|
|
|
|
return to_sample
|