370953729b
- perf_stats aggregator lives in eval/, not model/: the import failed silently and EVERY perf column was empty (not just ttft). Now warns on stderr instead of swallowing. - repeats > 1 get their own checkpoint key (:rep2, :rep3, ...): repeat 2 previously restored repeat 1's predictions and finished instantly with identical scores. rep1 keeps the legacy key (existing checkpoints still resume). - repeats summary: report the MEAN score and aggregate time/tokens over ALL runs (was: last run only). - README: six-benchmark command as the primary example. Co-Authored-By: Claude <noreply@anthropic.com>
42 lines
1.7 KiB
Python
42 lines
1.7 KiB
Python
"""LiveCodeBench code generation lite (official source: livecodebench/code_generation_lite)."""
|
|
|
|
from ..sample import Sample
|
|
from ..registry import register_dataset
|
|
from ..spec import DatasetSpec
|
|
|
|
|
|
@register_dataset(
|
|
DatasetSpec(
|
|
name='live_code_bench',
|
|
source='sam-paech/livecodebench-code_generation_lite', # HF parquet conversion; the official
|
|
# livecodebench/code_generation_lite is script-based (unloadable by datasets>=5)
|
|
subset='release_latest', # or release_v1..v6
|
|
split='test',
|
|
prompt_style='lcb_es', # official LCB code contract (system+format)
|
|
gen_config={'temperature': 1.0, 'max_tokens': 32768},
|
|
task_type='coding',
|
|
tags=['code'],
|
|
description='LiveCodeBench (lite) contest problems; version tags via DatasetSpec.version.',
|
|
params={'hub': 'hf_raw'},
|
|
)
|
|
)
|
|
def live_code_bench():
|
|
def to_sample(record: dict) -> Sample:
|
|
return Sample(
|
|
input=record['question_content'],
|
|
target='', # judged by running hidden test cases, no reference text
|
|
metadata={
|
|
'question_id': record['question_id'],
|
|
'contest_id': record.get('contest_id'),
|
|
'contest_date': record.get('contest_date'),
|
|
'platform': record.get('platform'),
|
|
'difficulty': record.get('difficulty'),
|
|
'starter_code': record.get('starter_code'),
|
|
'public_test_cases': record.get('public_test_cases'),
|
|
'private_test_cases': record.get('private_test_cases'),
|
|
'raw_metadata': record.get('metadata'), # fn_name (func_name) lives here
|
|
},
|
|
)
|
|
|
|
return to_sample
|