370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
36 lines
1.4 KiB
Python
36 lines
1.4 KiB
Python
"""general_fc: simple function-calling demo set (native to evalscope, its official home)."""
|
|
|
|
import json
|
|
|
|
from ..sample import ChatMessage, Sample, ToolInfo
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='general_fc',
|
|
source='evalscope/GeneralFunctionCall-Test', # evalscope-native release (ModelScope)
|
|
split='test',
|
|
gen_config={'temperature': 0.0, 'max_tokens': 4096},
|
|
task_type='fc',
|
|
tags=['function_calling'],
|
|
description='Minimal function-calling test set (evalscope-native).',
|
|
params={'hub': 'modelscope'},
|
|
)
|
|
)
|
|
def general_fc():
|
|
def to_sample(record: dict) -> Sample:
|
|
messages = json.loads(record['messages']) if isinstance(record['messages'], str) else record['messages']
|
|
tools = json.loads(record['tools']) if isinstance(record['tools'], str) else record.get('tools')
|
|
chat = [ChatMessage(role=m.get('role', 'user'), content=m['content'] if isinstance(m.get('content'), str)
|
|
else json.dumps(m['content'], ensure_ascii=False)) for m in messages]
|
|
return Sample(
|
|
input=chat,
|
|
target=str(record.get('should_call_tool', '')),
|
|
tools=[ToolInfo(**t['function']) if isinstance(t, dict) and 'function' in t else ToolInfo(name=str(t))
|
|
for t in (tools or [])] or None,
|
|
)
|
|
|
|
return to_sample
|