[BugFix] align Phase 3 captures with active decode

README.md

@@ -1,5 +1,9 @@
 # sskj — 多平台大模型推理性能基准测试项目
 
+> **更新（2026-07-31 19:02:00 CST）**
+>
+> Phase 3 Nsight smoke 已在 Head/Worker 各生成一份可解析报告；首份时间线确认 NCCL AllReduce、Sparse MLA、CUTLASS GEMM 与 MoE kernel 均可见。根据 smoke 修正正式 capture：Decode 对照和混合负载必须等服务实际进入 `Decode batch` 后才触发；每段等待本节点全部 8 个 rank 完成，而非任一 rank；双节点近同步停止以减少 Gloo 断链噪声。`nsys stats` 改为可写临时 SQLite、保留 Kernel/API/NVTX 高价值汇总并删除临时库，结果与报告数量均 fail-closed。
+>
 > **更新（2026-07-31 18:53:00 CST）**
 >
 > Phase 3 PyTorch Profiler 首轮 smoke 已证明双节点 16 个 rank 均可完成 trace 写盘，Head/Worker 分别生成 8 份、约 1.5/1.4 GB 压缩 trace。修复当前 OpenSSH 不接受 `scp remote:/path/.` 导致 Worker 结果未回收的问题，改为 SSH tar 流式传输；正式入口不再忽略回收失败，并新增至少 16 份 PyTorch rank trace、至少 2 份 Nsight 节点报告的结果门禁。

experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/.gitignore

@@ -0,0 +1,3 @@
+results/
+runtime/
+*.log

experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh

@@ -278,16 +278,22 @@ start_service() {
   return 1
 }
 
-stop_service_node() {
+request_stop_node() {
   local node="$1"
   local container_name="$2"
-  local role="$3"
   if [[ "${DRY_RUN}" == "1" ]]; then
     log "[DRY] ${node}: docker stop --time ${NSYS_EXPORT_TIMEOUT_S} ${container_name}"
     return 0
   fi
   run_on_node "${node}" \
     "docker stop --time '${NSYS_EXPORT_TIMEOUT_S}' '${container_name}' >/dev/null 2>&1 || true"
+}
+
+collect_service_node() {
+  local node="$1"
+  local container_name="$2"
+  local role="$3"
+  [[ "${DRY_RUN}" == "1" ]] && return 0
   run_on_node "${node}" "docker logs '${container_name}' 2>&1" \
     > "${SERVICE_DIR}/${role}_server.log" 2>&1 || true
   run_on_node "${node}" "docker inspect '${container_name}' 2>/dev/null" \
@@ -305,12 +311,24 @@ fetch_worker_profiles() {
 }
 
 stop_service() {
-  stop_service_node "${HEAD_NODE}" "${HEAD_CONTAINER}" head
+  if [[ "${DRY_RUN}" == "1" ]]; then
-  stop_service_node "${WORKER_NODE}" "${WORKER_CONTAINER}" worker
+    request_stop_node "${HEAD_NODE}" "${HEAD_CONTAINER}"
+    request_stop_node "${WORKER_NODE}" "${WORKER_CONTAINER}"
+    return 0
+  fi
+  request_stop_node "${HEAD_NODE}" "${HEAD_CONTAINER}" &
+  local head_stop_pid=$!
+  request_stop_node "${WORKER_NODE}" "${WORKER_CONTAINER}" &
+  local worker_stop_pid=$!
+  wait "${head_stop_pid}"
+  wait "${worker_stop_pid}"
+  collect_service_node "${HEAD_NODE}" "${HEAD_CONTAINER}" head
+  collect_service_node "${WORKER_NODE}" "${WORKER_CONTAINER}" worker
   fetch_worker_profiles
 }
 
 validate_torch_traces() {
+  [[ "${DRY_RUN}" == "1" ]] && return 0
   local count
   count="$(find "${RESULT_DIR}/profiles" -type f -name '*.trace.json.gz' | wc -l)"
   if (( count < TP_SIZE )); then
@@ -321,10 +339,12 @@ validate_torch_traces() {
 }
 
 validate_nsys_reports() {
-  local count
+  [[ "${DRY_RUN}" == "1" ]] && return 0
+  local count expected
   count="$(find "${RESULT_DIR}/profiles" -type f -name '*.nsys-rep' | wc -l)"
-  if (( count < NNODES )); then
+  expected=$(( NNODES * CAPTURE_RANGES ))
-    log "ERROR: expected at least ${NNODES} Nsight reports, found ${count}"
+  if (( count < expected )); then
+    log "ERROR: expected at least ${expected} Nsight reports, found ${count}"
     return 1
   fi
   log "Nsight report validation passed: ${count} reports"
@@ -366,9 +386,10 @@ start_profile() {
 
 wait_for_profile_stop() {
   local before="$1"
+  local expected=$(( TP_SIZE / NNODES ))
   local waited=0
   while (( waited < PROFILE_STOP_WAIT_S )); do
-    if (( $(profile_done_count) > before )); then
+    if (( $(profile_done_count) >= before + expected )); then
       return 0
     fi
     sleep 2
@@ -378,6 +399,26 @@ wait_for_profile_stop() {
   return 1
 }
 
+server_decode_count() {
+  run_on_node "${HEAD_NODE}" "docker logs '${HEAD_CONTAINER}' 2>&1" \
+    | grep -c 'Decode batch' || true
+}
+
+wait_for_decode_activity() {
+  local before="$1"
+  local bench_pid="$2"
+  local waited=0
+  while (( waited < 180 )); do
+    if (( $(server_decode_count) > before )); then
+      return 0
+    fi
+    kill -0 "${bench_pid}" 2>/dev/null || return 1
+    sleep 1
+    ((waited+=1))
+  done
+  return 1
+}
+
 prepare_bench_command() {
   local output_file="$1"
   local isl="$2"
@@ -455,6 +496,47 @@ run_profiled_case() {
   [[ "${DRY_RUN}" == "1" ]] || wait_for_profile_stop "${before}"
 }
 
+run_profiled_decode_case() {
+  local case_id="$1"
+  local steps="$2"
+  local isl="$3"
+  local osl="$4"
+  local concurrency="$5"
+  local prompts="$6"
+  local seed="$7"
+  local case_path="${CASE_DIR}/${case_id}"
+  local decode_before=0
+  local profile_before=0
+  flush_cache
+  mkdir -p "${case_path}"
+  prepare_bench_command "${case_path}/bench.jsonl" \
+    "${isl}" "${osl}" "${concurrency}" "${prompts}" "${seed}"
+  print_command timeout --signal=TERM --kill-after=30s \
+    "${PROFILE_TIMEOUT_S}s" "${BENCH_CMD[@]}" > "${case_path}/bench_cmd.txt"
+  if [[ "${DRY_RUN}" == "1" ]]; then
+    log "[DRY] start decode workload, wait for Decode batch, then start Nsight range"
+    printf '[DRY] '
+    cat "${case_path}/bench_cmd.txt"
+    start_profile "${case_id}" '["CUDA_PROFILER"]' "${steps}"
+    return 0
+  fi
+  decode_before="$(server_decode_count)"
+  timeout --signal=TERM --kill-after=30s "${PROFILE_TIMEOUT_S}s" \
+    "${BENCH_CMD[@]}" > "${case_path}/bench.log" 2>&1 &
+  local bench_pid=$!
+  wait_for_bench_main "${case_path}/bench.log" "${bench_pid}" \
+    && wait_for_decode_activity "${decode_before}" "${bench_pid}" || {
+      kill "${bench_pid}" 2>/dev/null || true
+      wait "${bench_pid}" 2>/dev/null || true
+      log "ERROR: ${case_id} did not reach active Decode"
+      return 1
+    }
+  profile_before="$(profile_done_count)"
+  start_profile "${case_id}" '["CUDA_PROFILER"]' "${steps}"
+  wait_for_profile_stop "${profile_before}"
+  wait "${bench_pid}"
+}
+
 wait_for_bench_main() {
   local bench_log="$1"
   local bench_pid="$2"
@@ -474,6 +556,7 @@ run_mixed_profile() {
   local injection_id="${case_id}/prefill_injection"
   local background_path="${CASE_DIR}/${background_id}"
   local before=0
+  local decode_before=0
   flush_cache
   mkdir -p "${background_path}"
   prepare_bench_command "${background_path}/bench.jsonl" \
@@ -481,16 +564,18 @@ run_mixed_profile() {
   print_command timeout --signal=TERM --kill-after=30s \
     "${PROFILE_TIMEOUT_S}s" "${BENCH_CMD[@]}" > "${background_path}/bench_cmd.txt"
   if [[ "${DRY_RUN}" == "1" ]]; then
-    log "[DRY] start decode background, wait for main run, start profile, inject 128K prefill"
+    log "[DRY] start decode background, wait for Decode batch, start profile, inject 128K prefill"
     start_profile "${case_id}" '["CUDA_PROFILER"]' "${NSYS_MIXED_STEPS}"
     run_bench "${injection_id}" "${MIXED_INJECTION_ISL}" \
       "${MIXED_INJECTION_OSL}" 1 1 5202
     return 0
   fi
+  decode_before="$(server_decode_count)"
   timeout --signal=TERM --kill-after=30s "${PROFILE_TIMEOUT_S}s" \
     "${BENCH_CMD[@]}" > "${background_path}/bench.log" 2>&1 &
   local background_pid=$!
-  wait_for_bench_main "${background_path}/bench.log" "${background_pid}" || {
+  wait_for_bench_main "${background_path}/bench.log" "${background_pid}" \
+    && wait_for_decode_activity "${decode_before}" "${background_pid}" || {
     kill "${background_pid}" 2>/dev/null || true
     wait "${background_pid}" 2>/dev/null || true
     log "ERROR: mixed decode background did not enter main benchmark"
@@ -537,10 +622,16 @@ generate_nsys_stats() {
     relative="${report#${RESULT_DIR}/}"
     output="${report%.nsys-rep}.stats.txt"
     docker run --rm \
-      -v "${RESULT_DIR}:/results:ro" \
+      -v "${RESULT_DIR}:/results" \
       --entrypoint nsys "${DOCKER_IMAGE}" stats \
-      --report cuda_gpu_kern_sum,cuda_api_sum,cuda_kern_exec_sum,nvtx_sum \
+      --force-export=true \
-      "/results/${relative}" > "${output}" 2>&1 || true
+      --report cuda_gpu_kern_sum,cuda_api_sum,nvtx_sum \
+      "/results/${relative}" > "${output}" 2>&1
+    grep -q 'CUDA GPU Kernel Summary' "${output}" || {
+      log "ERROR: nsys stats missing CUDA kernel summary for ${relative}"
+      return 1
+    }
+    rm -f "${report%.nsys-rep}.sqlite"
   done < <(find "${RESULT_DIR}/profiles" -type f -name '*.nsys-rep' | sort)
 }
 
@@ -586,7 +677,7 @@ run_all() {
   CLEANUP_ON_EXIT=1
   start_service
   write_manifest RUNNING
-  run_profiled_case decode_control '["CUDA_PROFILER"]' \
+  run_profiled_decode_case decode_control \
     "${NSYS_CONTROL_STEPS}" "${CONTROL_ISL}" "${CONTROL_OSL}" \
     "${CONTROL_CONCURRENCY}" "${CONTROL_PROMPTS}" 5301
   run_mixed_profile