370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
52 lines
2.1 KiB
Python
52 lines
2.1 KiB
Python
"""tau2-bench (Sierra). Official release: github.com/sierra-research/tau2-bench.
|
|
|
|
We load the ModelScope mirror of the official task files
|
|
(evalscope/tau2-bench-data, same repo layout: tau2/domains/<domain>/tasks.json).
|
|
Each record is a full task: agent purpose + user scenario + evaluation criteria.
|
|
"""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
_DOMAINS = ('airline', 'retail', 'telecom', 'mock')
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='tau2_bench',
|
|
source='HuggingFaceH4/tau2-bench-data', # HF mirror of the official GitHub data
|
|
subset='airline', # or retail / telecom / mock; override with --subset
|
|
split='test',
|
|
gen_config={'temperature': 0.0, 'max_tokens': 16384},
|
|
task_type='agent',
|
|
tags=['agent', 'tool_use', 'dialog'],
|
|
description='tau2-bench agent-tool-dialog tasks (official content, HF mirror).',
|
|
params={'hub': 'hf_raw', 'hf_files': ['domains/{subset}/tasks.json']},
|
|
)
|
|
)
|
|
def tau2_bench():
|
|
def to_sample(record: dict) -> Sample:
|
|
desc = record.get('description') or {}
|
|
scenario = record.get('user_scenario') or {}
|
|
instructions = (scenario.get('instructions') or {}).get('task_instructions')
|
|
return Sample(
|
|
input=desc.get('purpose') or record.get('id', ''),
|
|
target='',
|
|
metadata={
|
|
'id': record.get('id'),
|
|
'domain': record.get('domain'),
|
|
'task': record, # OFFICIAL Task json, verbatim -- the
|
|
# tau2_official env runs the official
|
|
# engine on this; nothing is lost
|
|
'notes': desc.get('notes'),
|
|
'task_instructions': instructions,
|
|
'user_scenario': scenario,
|
|
'initial_state': record.get('initial_state'),
|
|
'evaluation_criteria': record.get('evaluation_criteria'),
|
|
'annotations': record.get('annotations'),
|
|
},
|
|
)
|
|
|
|
return to_sample
|