65 lines
2.7 KiB
Bash
65 lines
2.7 KiB
Bash
#!/bin/bash
|
||
# Sanitizer 竞态触发 + 崩溃收割（60.8, deploy_ppmtp_sanitize.sh 配套）
|
||
# 自带健康等待；健康后打 bench cc16 剖面杀手负载（16x16384->512 random-ids,
|
||
# flush-cache 队列churn——09-06/07 档案中唯一可靠的竞态触发器）；
|
||
# 容器死亡时收割全量 docker logs 到 /root/san_crash_s<seed>.log。
|
||
# 用法: nohup bash killer_cc16.sh [seed] > /root/killer_s<seed>.log 2>&1 &
|
||
SEED=${1:-6401}
|
||
URL=http://127.0.0.1:30000
|
||
|
||
echo "killer: waiting for health (up to 3600s)..."
|
||
HEALTHY=0
|
||
for i in $(seq 1 360); do
|
||
code=$(curl -s -o /dev/null -m5 -w '%{http_code}' $URL/health 2>/dev/null)
|
||
if [ "$code" = "200" ]; then HEALTHY=1; break; fi
|
||
if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$'; then
|
||
echo "CONTAINER-DIED-BEFORE-HEALTH at $(date +%T)"
|
||
docker logs glm53-nvfp4 > /root/san_crash_prehealth_s${SEED}.log 2>&1
|
||
echo "log saved: /root/san_crash_prehealth_s${SEED}.log"
|
||
exit 2
|
||
fi
|
||
sleep 10
|
||
done
|
||
[ $HEALTHY -eq 1 ] || { echo "HEALTH-TIMEOUT"; exit 4; }
|
||
echo "healthy at $(date +%T)"
|
||
|
||
docker exec glm53-nvfp4 python3 -m sglang.bench_serving \
|
||
--backend sglang --host 127.0.0.1 --port 30000 \
|
||
--dataset-name random-ids --tokenizer /data/hf_models/GLM-5.3-NVFP4 \
|
||
--num-prompts 16 --random-input-len 16384 --random-output-len 512 \
|
||
--random-range-ratio 1.0 --max-concurrency 16 \
|
||
--temperature 0.0 --flush-cache --warmup-requests 1 --seed ${SEED} \
|
||
--output-file /data/hf_models/bs_results/san_cc16_s${SEED}.json \
|
||
> /root/san_bench_s${SEED}.log 2>&1 &
|
||
BPID=$!
|
||
echo "bench pid=$BPID started $(date +%T)"
|
||
|
||
BADH=0
|
||
while kill -0 $BPID 2>/dev/null; do
|
||
sleep 20
|
||
if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$'; then
|
||
echo "CONTAINER-DIED at $(date +%T)"
|
||
docker logs glm53-nvfp4 > /root/san_crash_s${SEED}.log 2>&1
|
||
echo "full log saved: /root/san_crash_s${SEED}.log ($(wc -l < /root/san_crash_s${SEED}.log) lines)"
|
||
grep -nE "=========|Invalid|invalid|illegal|CUDA_ERROR|out of bounds|Error Summ" /root/san_crash_s${SEED}.log | head -40
|
||
exit 2
|
||
fi
|
||
code=$(curl -s -o /dev/null -m5 -w '%{http_code}' $URL/health 2>/dev/null)
|
||
if [ "$code" != "200" ]; then
|
||
BADH=$((BADH+1))
|
||
if [ $BADH -ge 3 ]; then
|
||
echo "HEALTH-DEAD at $(date +%T)"
|
||
docker logs glm53-nvfp4 > /root/san_crash_s${SEED}.log 2>&1
|
||
echo "full log saved: /root/san_crash_s${SEED}.log ($(wc -l < /root/san_crash_s${SEED}.log) lines)"
|
||
grep -nE "=========|Invalid|invalid|illegal|CUDA_ERROR|out of bounds|Error Summ" /root/san_crash_s${SEED}.log | head -40
|
||
exit 3
|
||
fi
|
||
else
|
||
BADH=0
|
||
fi
|
||
done
|
||
wait $BPID; RC=$?
|
||
echo "BENCH-FINISHED rc=$RC $(date +%T) —— 竞态未触发（负载全程存活）"
|
||
tail -8 /root/san_bench_s${SEED}.log
|
||
exit $RC
|