#!/bin/bash
# Sanitizer 竞态触发 + 崩溃收割（60.8, deploy_ppmtp_sanitize.sh 配套）
# 自带健康等待；健康后打 bench cc16 剖面杀手负载（16x16384->512 random-ids,
# flush-cache 队列churn——09-06/07 档案中唯一可靠的竞态触发器）；
# 容器死亡时收割全量 docker logs 到 /root/san_crash_s<seed>.log。
# 用法: nohup bash killer_cc16.sh [seed] > /root/killer_s<seed>.log 2>&1 &
SEED=${1:-6401}
URL=http://127.0.0.1:30000

echo "killer: waiting for health (up to 3600s)..."
HEALTHY=0
for i in $(seq 1 360); do
  code=$(curl -s -o /dev/null -m5 -w '%{http_code}' $URL/health 2>/dev/null)
  if [ "$code" = "200" ]; then HEALTHY=1; break; fi
  if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$'; then
    echo "CONTAINER-DIED-BEFORE-HEALTH at $(date +%T)"
    docker logs glm53-nvfp4 > /root/san_crash_prehealth_s${SEED}.log 2>&1
    echo "log saved: /root/san_crash_prehealth_s${SEED}.log"
    exit 2
  fi
  sleep 10
done
[ $HEALTHY -eq 1 ] || { echo "HEALTH-TIMEOUT"; exit 4; }
echo "healthy at $(date +%T)"

docker exec glm53-nvfp4 python3 -m sglang.bench_serving \
  --backend sglang --host 127.0.0.1 --port 30000 \
  --dataset-name random-ids --tokenizer /data/hf_models/GLM-5.3-NVFP4 \
  --num-prompts 16 --random-input-len 16384 --random-output-len 512 \
  --random-range-ratio 1.0 --max-concurrency 16 \
  --temperature 0.0 --flush-cache --warmup-requests 1 --seed ${SEED} \
  --output-file /data/hf_models/bs_results/san_cc16_s${SEED}.json \
  > /root/san_bench_s${SEED}.log 2>&1 &
BPID=$!
echo "bench pid=$BPID started $(date +%T)"

BADH=0
while kill -0 $BPID 2>/dev/null; do
  sleep 20
  if ! docker ps --format '{{.Names}}' 2>/dev/null | grep -q '^glm53-nvfp4$'; then
    echo "CONTAINER-DIED at $(date +%T)"
    docker logs glm53-nvfp4 > /root/san_crash_s${SEED}.log 2>&1
    echo "full log saved: /root/san_crash_s${SEED}.log ($(wc -l < /root/san_crash_s${SEED}.log) lines)"
    grep -nE "=========|Invalid|invalid|illegal|CUDA_ERROR|out of bounds|Error Summ" /root/san_crash_s${SEED}.log | head -40
    exit 2
  fi
  code=$(curl -s -o /dev/null -m5 -w '%{http_code}' $URL/health 2>/dev/null)
  if [ "$code" != "200" ]; then
    BADH=$((BADH+1))
    if [ $BADH -ge 3 ]; then
      echo "HEALTH-DEAD at $(date +%T)"
      docker logs glm53-nvfp4 > /root/san_crash_s${SEED}.log 2>&1
      echo "full log saved: /root/san_crash_s${SEED}.log ($(wc -l < /root/san_crash_s${SEED}.log) lines)"
      grep -nE "=========|Invalid|invalid|illegal|CUDA_ERROR|out of bounds|Error Summ" /root/san_crash_s${SEED}.log | head -40
      exit 3
    fi
  else
    BADH=0
  fi
done
wait $BPID; RC=$?
echo "BENCH-FINISHED rc=$RC $(date +%T) —— 竞态未触发（负载全程存活）"
tail -8 /root/san_bench_s${SEED}.log
exit $RC