diff --git a/evalharness/config/gen_profiles.yaml b/evalharness/config/gen_profiles.yaml new file mode 100644 index 0000000..bb1fe27 --- /dev/null +++ b/evalharness/config/gen_profiles.yaml @@ -0,0 +1,31 @@ +# Generation-parameter profiles (YAML). +# Usage: evalharness eval run --model ... --profile +# Resolution: DatasetSpec.gen_config < profile.default < profile. < gen_kwargs + +dp4-nothink: + default: {temperature: 0.0, max_tokens: 32768, top_p: 1.0} + +qwen3-es-parity: + default: {temperature: 0.0, max_tokens: 32768} + simple_qa: {max_tokens: 1024} + hle: {max_tokens: 8192} + gpqa_diamond: {temperature: 1.0, max_tokens: 8192} + aime24: {temperature: 1.0, max_tokens: 8192} + aime25: {temperature: 1.0, max_tokens: 8192} + aime26: {temperature: 1.0, max_tokens: 8192} + hmmt26: {temperature: 1.0, max_tokens: 8192} + imo_answerbench: {temperature: 1.0, max_tokens: 8192} + +glm53-nothink: + default: {temperature: 0.0, max_tokens: 8192, top_p: 1.0} + aime24: {temperature: 1.0, max_tokens: 8192} + aime25: {temperature: 1.0, max_tokens: 8192} + aime26: {temperature: 1.0, max_tokens: 8192} + hmmt26: {temperature: 1.0, max_tokens: 8192} + imo_answerbench: {temperature: 1.0, max_tokens: 8192} + gpqa_diamond: {temperature: 1.0, max_tokens: 8192} + humaneval: {temperature: 1.0, max_tokens: 32768} + live_code_bench: {temperature: 1.0, max_tokens: 32768} + +t1-short: + default: {temperature: 1.0, max_tokens: 8192, top_p: 1.0} diff --git a/evalharness/model/gen_profiles.py b/evalharness/model/gen_profiles.py index f41dda8..262cba6 100644 --- a/evalharness/model/gen_profiles.py +++ b/evalharness/model/gen_profiles.py @@ -71,6 +71,7 @@ def _candidate_yaml_paths(): if env: paths.append(Path(env)) paths.append(Path('gen_profiles.yaml')) + paths.append(Path(__file__).parent.parent / 'config' / 'gen_profiles.yaml') paths.append(Path.home() / '.config' / 'evalharness' / 'gen_profiles.yaml') return paths