64 lines
2.5 KiB
Python
64 lines
2.5 KiB
Python
"""BFCL v3 (Berkeley Function Calling Leaderboard).
|
|
|
|
Official release: github.com/gorilla-llm/Berkeley-Function-Calling-Leaderboard.
|
|
We load the ModelScope mirror (AI-ModelScope/bfcl_v3, parquet) with columns
|
|
``id / turns / tools / test_category / ground_truth``. Filter by
|
|
``metadata.test_category`` at eval time (simple / irrelevance / multi_turn /
|
|
parallel / java / javascript / ...).
|
|
"""
|
|
|
|
import json
|
|
|
|
from ..sample import ChatMessage, Sample, ToolInfo
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='bfcl_v3',
|
|
source='llamastack/bfcl_v3', # HF conversion of the official GitHub data
|
|
split='train', # the conversion ships a single split
|
|
task_type='fc',
|
|
tags=['function_calling', 'tool_use'],
|
|
description='BFCL v3 function calling (official content, HF conversion).',
|
|
params={'hub': 'hf_raw'},
|
|
)
|
|
)
|
|
def bfcl_v3():
|
|
def _parse(v):
|
|
if isinstance(v, str):
|
|
try:
|
|
return json.loads(v)
|
|
except (ValueError, TypeError):
|
|
return v
|
|
return v
|
|
|
|
def to_sample(record: dict) -> Sample:
|
|
messages = []
|
|
for turn in _parse(record.get('turns')) or []:
|
|
messages.extend(turn if isinstance(turn, list) else [turn])
|
|
tools = []
|
|
for t in _parse(record.get('tools')) or []:
|
|
spec = t.get('function') if isinstance(t, dict) and 'function' in t else t
|
|
if isinstance(spec, dict) and spec.get('name'):
|
|
tools.append(ToolInfo(name=spec['name'], description=spec.get('description'),
|
|
parameters=spec.get('parameters') or {}))
|
|
return Sample(
|
|
input=[ChatMessage(role=m.get('role', 'user'), content=m['content'] if isinstance(m.get('content'), str)
|
|
else json.dumps(m['content'], ensure_ascii=False))
|
|
for m in messages if isinstance(m, dict)] or record.get('id', ''),
|
|
target=json.dumps(_parse(record.get('ground_truth')), ensure_ascii=False)
|
|
if record.get('ground_truth') is not None else '',
|
|
tools=tools or None,
|
|
metadata={
|
|
'id': record.get('id'),
|
|
'test_category': record.get('test_category'),
|
|
'multi_turn': record.get('multi_turn'),
|
|
'language': record.get('language'),
|
|
'functions': _parse(record.get('functions')),
|
|
},
|
|
)
|
|
|
|
return to_sample
|