Add deployment and multi-machine benchmark scripts
- scripts/build_and_upload_docker.sh: sync context, build, save and upload evalscope-complete-py312 image to ModelScope. - scripts/deploy_remote_machine.sh: SSH to a fresh machine and run install.sh. - scripts/run_multi_machine.sh: distribute official-suite benchmarks across 3 machines (gpu048, gpu049, gpu051).
This commit is contained in:
parent
5f0b63bf0f
commit
5ab3f249c6
66
scripts/build_and_upload_docker.sh
Executable file
66
scripts/build_and_upload_docker.sh
Executable file
@ -0,0 +1,66 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# 构建并上传 evalscope-complete-py312 Docker 镜像
|
||||
# 用法: bash scripts/build_and_upload_docker.sh
|
||||
# ============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
DOCKER_CTX="$ROOT_DIR/tools/docker"
|
||||
IMAGE_TAG="evalscope-complete-py312:latest"
|
||||
OUTPUT_TAR="$ROOT_DIR/docker/evalscope-complete-py312.tar.gz"
|
||||
LOG_FILE="$ROOT_DIR/logs/build_docker_$(date +%Y%m%d_%H%M%S).log"
|
||||
|
||||
mkdir -p "$ROOT_DIR/logs"
|
||||
mkdir -p "$ROOT_DIR/docker"
|
||||
|
||||
exec > >(tee -a "$LOG_FILE")
|
||||
exec 2>&1
|
||||
|
||||
echo "==> 开始构建 Docker 镜像: $IMAGE_TAG"
|
||||
echo " 日志: $LOG_FILE"
|
||||
echo ""
|
||||
|
||||
# 1. 同步构建上下文
|
||||
echo "==> 1. 同步构建上下文到 $DOCKER_CTX"
|
||||
cd "$ROOT_DIR"
|
||||
rsync -av --delete --exclude='*.log' --exclude='__pycache__' --exclude='.git' \
|
||||
bash/ "$DOCKER_CTX/bash/"
|
||||
rsync -av --delete --exclude='*.log' --exclude='__pycache__' --exclude='.git' \
|
||||
evalscope/ "$DOCKER_CTX/evalscope/"
|
||||
rsync -av --delete --exclude='*.log' --exclude='__pycache__' --exclude='.git' \
|
||||
tools/tau2-bench/ "$DOCKER_CTX/tau2-bench/"
|
||||
|
||||
# 2. 构建镜像
|
||||
echo ""
|
||||
echo "==> 2. 构建 Docker 镜像"
|
||||
cd "$DOCKER_CTX"
|
||||
docker build -f Dockerfile.py312 -t "$IMAGE_TAG" .
|
||||
|
||||
# 3. 保存镜像
|
||||
echo ""
|
||||
echo "==> 3. 保存镜像到 $OUTPUT_TAR"
|
||||
docker save "$IMAGE_TAG" | gzip > "$OUTPUT_TAR"
|
||||
ls -lh "$OUTPUT_TAR"
|
||||
|
||||
# 4. 生成 md5
|
||||
echo ""
|
||||
echo "==> 4. 生成 md5"
|
||||
md5sum "$OUTPUT_TAR" > "$OUTPUT_TAR.md5"
|
||||
|
||||
# 5. 上传到 ModelScope
|
||||
echo ""
|
||||
echo "==> 5. 上传到 ModelScope"
|
||||
modelscope upload \
|
||||
--commit-message "Update evalscope-complete-py312 with torch (CPU) for tokenizer support" \
|
||||
SoraAmami/evalscope-docker \
|
||||
"$OUTPUT_TAR"
|
||||
|
||||
echo ""
|
||||
echo "========================================"
|
||||
echo "构建并上传完成!"
|
||||
echo "镜像: $IMAGE_TAG"
|
||||
echo "本地 tar: $OUTPUT_TAR"
|
||||
echo "日志: $LOG_FILE"
|
||||
echo "========================================"
|
||||
29
scripts/deploy_remote_machine.sh
Executable file
29
scripts/deploy_remote_machine.sh
Executable file
@ -0,0 +1,29 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# 在远程机器上部署 EvalScope
|
||||
# 用法: bash scripts/deploy_remote_machine.sh <IP> [BASE_DIR]
|
||||
# 示例: bash scripts/deploy_remote_machine.sh 174.1.51.3 /data1/sora
|
||||
# ============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
IP="${1:?请提供远程机器 IP}"
|
||||
BASE_DIR="${2:-/data1/sora}"
|
||||
PASS="${SSHPASS:-sskj2025}"
|
||||
REMOTE="root@$IP"
|
||||
|
||||
SSH="sshpass -p $PASS ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 $REMOTE"
|
||||
SCP="sshpass -p $PASS scp -o StrictHostKeyChecking=no -o ConnectTimeout=10"
|
||||
|
||||
echo "==> 部署到 $IP ..."
|
||||
|
||||
# 1. 确保基础目录存在
|
||||
$SSH "mkdir -p $BASE_DIR && which git docker || (apt-get update && apt-get install -y git docker.io)"
|
||||
|
||||
# 2. 克隆/更新代码
|
||||
$SSH "cd $BASE_DIR && if [ -d evalscope ]; then cd evalscope && git pull; else git clone https://git.meta-stone.net/sora/evalstone.git evalscope; fi"
|
||||
|
||||
# 3. 运行部署脚本(下载数据、Docker 镜像、配置环境)
|
||||
$SSH "cd $BASE_DIR/evalscope && bash install.sh $BASE_DIR"
|
||||
|
||||
echo "==> $IP 部署完成"
|
||||
55
scripts/run_multi_machine.sh
Executable file
55
scripts/run_multi_machine.sh
Executable file
@ -0,0 +1,55 @@
|
||||
#!/bin/bash
|
||||
# ============================================================
|
||||
# 多机器 benchmark 分发脚本
|
||||
# 根据当前进度把 official suite 拆到 3 台机器跑
|
||||
# 用法: bash scripts/run_multi_machine.sh
|
||||
# ============================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
PASS="${SSHPASS:-sskj2025}"
|
||||
SSH="sshpass -p $PASS ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10"
|
||||
|
||||
# 机器配置
|
||||
MACHINES=(
|
||||
"174.1.51.1:/data1/sora" # gpu048 / P800-01
|
||||
"174.1.51.2:/data1/sora" # gpu049 / P800-02
|
||||
"174.1.51.4:/data1/sora" # gpu051 / P800-04
|
||||
)
|
||||
|
||||
COMMON_ARGS="--model DeepSeek-V4-Flash-Int8 --api-url http://localhost:30000/v1 --dataset-dir /data1/sora/evalscope --output-dir /data1/sora/evalscope/output --batch-size 4 --thinking --limit none --seed 42"
|
||||
|
||||
# 分发方案:尽量让 3 台机器同时跑完
|
||||
# 估算:aime25/aime26 12 run,live_code_bench 5 run,其余 1 run
|
||||
# Machine 1: 推理/数学 multi-run(约 30h)
|
||||
M1_BENCHES="aime25,aime26,live_code_bench"
|
||||
M1_FOLDER="DP4-flash-int8-thinking-m1"
|
||||
|
||||
# Machine 2: 知识与长上下文(约 18h)
|
||||
M2_BENCHES="hle,mmlu_pro,gpqa_diamond,longbench_v2"
|
||||
M2_FOLDER="DP4-flash-int8-thinking-m2"
|
||||
|
||||
# Machine 3: SWE + Agent(约 30-50h,取决于 SWE 镜像)
|
||||
M3_BENCHES="swe_bench_verified,tau2_bench"
|
||||
M3_FOLDER="DP4-flash-int8-thinking-m3"
|
||||
|
||||
run_on_machine() {
|
||||
local ip_base="$1"
|
||||
local ip="${ip_base%%:*}"
|
||||
local base_dir="${ip_base##*:}"
|
||||
local datasets="$2"
|
||||
local folder="$3"
|
||||
local cmd="cd $base_dir/evalscope && nohup python bash/run.py --datasets $datasets --folder-name $folder $COMMON_ARGS > $base_dir/evalscope/logs/run_${folder}.log 2>&1 &"
|
||||
|
||||
echo "==> 在 $ip 启动: $folder ($datasets)"
|
||||
$SSH root@$ip "$cmd"
|
||||
}
|
||||
|
||||
run_on_machine "${MACHINES[0]}" "$M1_BENCHES" "$M1_FOLDER"
|
||||
run_on_machine "${MACHINES[1]}" "$M2_BENCHES" "$M2_FOLDER"
|
||||
run_on_machine "${MACHINES[2]}" "$M3_BENCHES" "$M3_FOLDER"
|
||||
|
||||
echo ""
|
||||
echo "==================================================="
|
||||
echo "已分发到 3 台机器,各自日志在 logs/run_*.log"
|
||||
echo "==================================================="
|
||||
Loading…
x
Reference in New Issue
Block a user