diff --git a/MARKER_TEST b/MARKER_TEST new file mode 100644 index 0000000..f1824ac --- /dev/null +++ b/MARKER_TEST @@ -0,0 +1 @@ +MARKER_FROM_HOST diff --git a/README.md b/README.md index 3ab3027..d4511fd 100644 --- a/README.md +++ b/README.md @@ -129,18 +129,15 @@ docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz # 5. 准备代码执行 sandbox 镜像（humaneval / bigcodebench 必须） # 从 ModelScope 下载预打包的 sandbox 镜像（避免 DockerHub 限流/超时） python3 -c " -from modelscope_hub import HubApi -api = HubApi() -api.download_file( - repo_id='SoraAmami/evalscope-sandbox-images', - repo_type='dataset', - path='bigcodebench-sandbox.tar.gz', +from modelscope.hub.file_download import model_file_download +model_file_download( + model_id='SoraAmami/evalscope-sandbox-images', + file_path='bigcodebench-sandbox.tar.gz', local_dir='/data1/sora/evalscope/docker/sandbox_images' ) -api.download_file( - repo_id='SoraAmami/evalscope-sandbox-images', - repo_type='dataset', - path='python-3.11-slim.tar.gz', +model_file_download( + model_id='SoraAmami/evalscope-sandbox-images', + file_path='python-3.11-slim.tar.gz', local_dir='/data1/sora/evalscope/docker/sandbox_images' ) " @@ -205,18 +202,15 @@ docker load -i /data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz # 7. 准备代码执行 sandbox 镜像 # 从 ModelScope 下载预打包的 sandbox 镜像（避免 DockerHub 限流/超时） python3 -c " -from modelscope_hub import HubApi -api = HubApi() -api.download_file( - repo_id='SoraAmami/evalscope-sandbox-images', - repo_type='dataset', - path='bigcodebench-sandbox.tar.gz', +from modelscope.hub.file_download import model_file_download +model_file_download( + model_id='SoraAmami/evalscope-sandbox-images', + file_path='bigcodebench-sandbox.tar.gz', local_dir='/data1/sora/evalscope/docker/sandbox_images' ) -api.download_file( - repo_id='SoraAmami/evalscope-sandbox-images', - repo_type='dataset', - path='python-3.11-slim.tar.gz', +model_file_download( + model_id='SoraAmami/evalscope-sandbox-images', + file_path='python-3.11-slim.tar.gz', local_dir='/data1/sora/evalscope/docker/sandbox_images' ) " @@ -276,15 +270,15 @@ docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3. ### 4.2 `official` 官方对比套件 -对齐腾讯入库标准对比表核心子集： +当前默认配置： | 类型 | Benchmark | |------|-----------| | multi-run | `aime25`, `aime26`, `live_code_bench` | -| single-run | `hle`, `mmlu_pro`, `gpqa_diamond`, `longbench_v2` | +| single-run | `hle`, `mmlu_pro`, `gpqa_diamond`, `longbench_v2`, `swe_bench_verified` | | agent | `tau2_bench` | -> `swe_bench_verified` 默认注释掉，如需启用可修改 `bash/run.py` 的 `official` suite。 +> **注意**：`hle` 和 `tau2_bench` 需要 LLM-as-judge（默认 `DeepSeek/DeepSeek-V4-Pro` @ Vectron）。如果裁判 API 余额/配额不足，会报 `402`/`429`，需要充值或换用其他 judge 模型。 预计时间：视模型速度和是否启用 swe 而定，通常 **数小时到数十小时**。 @@ -314,6 +308,35 @@ docker images | grep -E 'evalscope-complete-py312|bigcodebench-sandbox|python:3. | `group2` | - | `hle`, `mmlu_pro`, `trivia_qa` | - | ~26h | | `group3` | `gpqa_diamond` | `openai_mrcr`, `longbench_v2`, `bfcl_v3`, `mmlu`, `cmmlu`, `bbh`, `simple_qa` | `tau2_bench`, `general_fc` | ~27h | +### 4.6 多机同时跑 `official` 套件示例 + +`official` 套件可在多台机器上并行执行，每台调用各自的本地模型服务： + +```bash +# 01 跑 swe_bench_verified +python bash/run.py \ + --api-url http://174.1.51.1:30000/v1 \ + --datasets swe_bench_verified \ + --folder-name DP4-flash-int8-thinking-add0 \ + --thinking --max-tokens-add 0 + +# 02 跑 mmlu_pro / gpqa_diamond / longbench_v2 +python bash/run.py \ + --api-url http://174.1.51.2:30000/v1 \ + --datasets mmlu_pro,gpqa_diamond,longbench_v2 \ + --folder-name DP4-flash-int8-thinking-add0 \ + --thinking --max-tokens-add 0 + +# 04 跑 hle（需要可用 judge API，否则会失败） +python bash/run.py \ + --api-url http://174.1.51.4:30000/v1 \ + --datasets hle \ + --folder-name DP4-flash-int8-thinking-add0 \ + --thinking --max-tokens-add 0 +``` + +跑完后所有机器的 `output/DP4-flash-int8-thinking-add0/` 可合并到同一目录，再运行 `bash/collect_results.py` 生成汇总表。 + --- ## 6. Multi-run 配置 @@ -395,7 +418,7 @@ for f in sorted(glob.glob('output/{folder_name}/*/seed_*/reports/*/*.json')): conda create -n evalscope python=3.12 -y conda activate evalscope cd /data1/sora/evalscope -pip install -e ./evalscope +pip install -e . ``` ### 7.2 运行单个 benchmark diff --git a/bash/run.py b/bash/run.py index 6fc8479..61c1715 100644 --- a/bash/run.py +++ b/bash/run.py @@ -652,7 +652,18 @@ def main(): truncation_tokens = args.truncation_tokens _patch_adapters_for_truncation(args.tokenizer_path, truncation_tokens) - dataset_configs = load_dataset_configs(args.config) + # Auto-select thinking config if --thinking is enabled and user did not + # explicitly provide a different --config. + config_path = args.config + if enable_thinking and args.config == DEFAULT_CONFIG: + thinking_config = str(PROJECT_ROOT / 'config' / 'dpv4-int8_thinking.yaml') + if Path(thinking_config).exists(): + config_path = thinking_config + print(f'Auto-selected thinking config: {config_path}') + else: + print(f'WARNING: thinking config not found at {thinking_config}, falling back to {args.config}') + + dataset_configs = load_dataset_configs(config_path) # Resolve top-level output folder name if args.folder_name: diff --git a/config/dpv4-int8_thinking.yaml b/config/dpv4-int8_thinking.yaml new file mode 100644 index 0000000..4570bf4 --- /dev/null +++ b/config/dpv4-int8_thinking.yaml @@ -0,0 +1,205 @@ +gpqa_diamond: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +hle: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +aime24: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +aime25: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 65536 +mmlu_pro: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +simple_qa: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +arc: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +bbh: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +live_code_bench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 65536 + dataset_args: + subset_list: + - release_v6 +aime26: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 65536 +hmmt26: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +imo_answerbench: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +super_gpqa: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +drop: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +hellaswag: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +mmlu: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +openai_mrcr: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +bigcodebench: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +humaneval: + generation_config: + temperature: 1.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +gsm8k: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +competition_math: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +cmmlu: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +trivia_qa: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +winogrande: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 +longbench_v2: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 8192 + dataset_args: + subset_list: + - short + - medium + - long +tau2_bench: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 16384 + dataset_args: + extra_params: + user_model: deepseek-v4-pro + api_key: sk-9ed86ef546ca47e3afa7c3b014dea268 + api_base: https://api.deepseek.com/v1 + generation_config: + temperature: 0.0 + max_tokens: 4096 + agent_config: + mode: native + strategy: react + max_steps: 50 +general_fc: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 4096 +bfcl_v3: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 4096 + parallel_tool_calls: true + dataset_args: + extra_params: + is_fc_model: true + underscore_to_dot: true +swe_bench_verified: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 +swe_bench_pro: + generation_config: + temperature: 0.0 + top_p: 1.0 + stream: true + max_tokens: 32768 \ No newline at end of file diff --git a/evalscope/evalscope/benchmarks/swe_bench/utils.py b/evalscope/evalscope/benchmarks/swe_bench/utils.py index 4e49a76..eab9689 100644 --- a/evalscope/evalscope/benchmarks/swe_bench/utils.py +++ b/evalscope/evalscope/benchmarks/swe_bench/utils.py @@ -83,6 +83,8 @@ def eval_instance( # Build + start instance container (instance image should already be built) container = None + client = None + instance_id = None eval_completed = False report = {} try: @@ -198,15 +200,20 @@ def eval_instance( f"Result for {instance_id}: resolved: {report[instance_id]['resolved']}") eval_completed = True except Exception as e: - error_msg = (f'Error in evaluating model for {instance_id}: {e}\n' + display_id = instance_id or 'unknown' + error_msg = (f'Error in evaluating model for {display_id}: {e}\n' f'{traceback.format_exc()}') logger.error(error_msg) report['error'] = error_msg finally: # Remove instance container - cleanup_container(client, container, logger) + if client is not None: + cleanup_container(client, container, logger) + resolved = False + if instance_id is not None: + resolved = report.get(instance_id, {}).get('resolved', False) return { 'completed': eval_completed, - 'resolved': report.get(instance_id, {}).get('resolved', False), + 'resolved': resolved, 'report': report, } diff --git a/scripts/copy_image_to_machine.sh b/scripts/copy_image_to_machine.sh new file mode 100755 index 0000000..c7e3469 --- /dev/null +++ b/scripts/copy_image_to_machine.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# ============================================================ +# 把本地新镜像 scp 到远程机器并加载 +# 用法: bash scripts/copy_image_to_machine.sh +# ============================================================ + +set -euo pipefail + +IP="${1:?请提供远程机器 IP}" +PASS="${SSHPASS:-sskj2025}" +LOCAL_TAR="/data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz" +REMOTE_TAR="/data1/sora/evalscope/docker/evalscope-complete-py312.tar.gz" + +echo "==> [$IP] 复制镜像 tar 包..." +sshpass -p "$PASS" scp -o StrictHostKeyChecking=no "$LOCAL_TAR" "root@$IP:$REMOTE_TAR" + +echo "==> [$IP] 加载 Docker 镜像..." +sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "root@$IP" "docker load -i $REMOTE_TAR" + +echo "==> [$IP] 验证 torch..." +sshpass -p "$PASS" ssh -o StrictHostKeyChecking=no "root@$IP" "docker run --rm evalscope-complete-py312:latest python -c 'import torch; print(\"torch\", torch.__version__)'" + +echo "==> [$IP] 完成" diff --git a/scripts/deploy_evalscope.sh b/scripts/deploy_evalscope.sh index fee5952..d589631 100755 --- a/scripts/deploy_evalscope.sh +++ b/scripts/deploy_evalscope.sh @@ -237,14 +237,11 @@ download_sandbox_image_from_modelscope() { return 0 fi echo "==> 从 ModelScope 下载 $file ..." - # 优先使用新版 modelscope_hub API（支持 dataset repo） python3 -c " -from modelscope_hub import HubApi -api = HubApi() -api.download_file( - repo_id='SoraAmami/evalscope-sandbox-images', - repo_type='dataset', - path='$file', +from modelscope.hub.file_download import model_file_download +model_file_download( + model_id='SoraAmami/evalscope-sandbox-images', + file_path='$file', local_dir='$SANDBOX_IMAGES_DIR' ) " && return 0