"""TriviaQA (official source: mandarjoshi/trivia_qa, rc.wikipedia config). rc.wikipedia = reading-comprehension WITH the Wikipedia evidence document (open-book, aligned with evalscope's default); rc.nocontext is the closed-book variant (pass --subset rc.nocontext). """ from ..sample import Sample from ..registry import register_dataset from ..spec import DatasetSpec @register_dataset( DatasetSpec( name='trivia_qa', source='mandarjoshi/trivia_qa', # official: https://huggingface.co/datasets/mandarjoshi/trivia_qa subset='rc.wikipedia', # open-book (evalscope parity); --subset rc.nocontext for closed split='validation', task_type='qa', tags=['knowledge', 'openqa'], description='TriviaQA with Wikipedia evidence (open-book); any alias counts.', ) ) def trivia_qa(): def to_sample(record: dict) -> Sample: answer = record['answer'] # {'value': ..., 'aliases': [...], ...} targets = [answer['value']] + list(answer.get('aliases') or []) # open-book: the Wikipedia evidence document (runner prepends it via # metadata['context'] when assembling the prompt) wiki = '' entity = record.get('entity_pages') or {} for doc in (entity.get('wiki_content') or [])[:1]: wiki = doc or '' break search = record.get('search_results') or {} if not wiki: wiki = '\n'.join((search.get('search_context') or [])[:2]) return Sample( input=record['question'], target=targets, # multi-target: any alias counts metadata={'question_id': record.get('question_id'), 'context': wiki or None}, ) return to_sample