From 405608ad23ad331df2322510dd3eb71f67ef116c Mon Sep 17 00:00:00 2001
From: Zhiyi Hong <2497491955@qq.com>
Date: Fri, 31 Jul 2026 18:53:19 +0800
Subject: [PATCH] [BugFix] make Phase 3 artifact collection fail closed

---
 README.md                                     |  4 +++
 .../run_timeline_profiling.sh                 | 30 +++++++++++++++++--
 2 files changed, 31 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 5f5f031..8ed8c4d 100644
--- a/README.md
+++ b/README.md
@@ -1,5 +1,9 @@
 # sskj — 多平台大模型推理性能基准测试项目
 
+> **更新（2026-07-31 18:53:00 CST）**
+>
+> Phase 3 PyTorch Profiler 首轮 smoke 已证明双节点 16 个 rank 均可完成 trace 写盘，Head/Worker 分别生成 8 份、约 1.5/1.4 GB 压缩 trace。修复当前 OpenSSH 不接受 `scp remote:/path/.` 导致 Worker 结果未回收的问题，改为 SSH tar 流式传输；正式入口不再忽略回收失败，并新增至少 16 份 PyTorch rank trace、至少 2 份 Nsight 节点报告的结果门禁。
+>
 > **更新（2026-07-31 18:43:00 CST）**
 >
 > 为 Phase 2 实验档案第 11 节的 GPU/DCGM、CPU/进程/NUMA、双 Rail RDMA、PCIe/NCCL 四组结果补充服务器证据路径。每组同时标明结构化汇总 CSV、Head/Worker 原始采样日志、通信原始输出及实际命令文件，便于从结论直接追溯最终 Run `dsv4pro-phase2-20260731-163620` 的证据。
diff --git a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh
index f5c9d0f..e8611a3 100644
--- a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh
+++ b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh
@@ -298,15 +298,36 @@ stop_service_node() {
 fetch_worker_profiles() {
   [[ "${DRY_RUN}" == "1" ]] && return 0
   mkdir -p "${RESULT_DIR}/profiles/worker"
-  scp -q -r "${WORKER_NODE}:${WORKER_PROFILE_HOST}/." \
-    "${RESULT_DIR}/profiles/worker/"
+  ssh -o BatchMode=yes -o StrictHostKeyChecking=no "${WORKER_NODE}" \
+    "tar -C '${WORKER_PROFILE_HOST}' -cf - ." \
+    | tar -C "${RESULT_DIR}/profiles/worker" -xf -
   run_on_node "${WORKER_NODE}" "rm -rf '${WORKER_RUN_DIR}'"
 }
 
 stop_service() {
   stop_service_node "${HEAD_NODE}" "${HEAD_CONTAINER}" head
   stop_service_node "${WORKER_NODE}" "${WORKER_CONTAINER}" worker
-  fetch_worker_profiles || true
+  fetch_worker_profiles
+}
+
+validate_torch_traces() {
+  local count
+  count="$(find "${RESULT_DIR}/profiles" -type f -name '*.trace.json.gz' | wc -l)"
+  if (( count < TP_SIZE )); then
+    log "ERROR: expected at least ${TP_SIZE} rank traces, found ${count}"
+    return 1
+  fi
+  log "PyTorch trace validation passed: ${count} rank traces"
+}
+
+validate_nsys_reports() {
+  local count
+  count="$(find "${RESULT_DIR}/profiles" -type f -name '*.nsys-rep' | wc -l)"
+  if (( count < NNODES )); then
+    log "ERROR: expected at least ${NNODES} Nsight reports, found ${count}"
+    return 1
+  fi
+  log "Nsight report validation passed: ${count} reports"
 }
 
 flush_cache() {
@@ -539,6 +560,7 @@ run_torch_smoke() {
     "${TORCH_SMOKE_STEPS}" "${CONTROL_ISL}" 128 8 8 5101
   stop_service
   CLEANUP_ON_EXIT=0
+  validate_torch_traces
   write_manifest COMPLETED
 }
 
@@ -553,6 +575,7 @@ run_nsys_smoke() {
     "${CONTROL_CONCURRENCY}" "${CONTROL_PROMPTS}" 5102
   stop_service
   CLEANUP_ON_EXIT=0
+  validate_nsys_reports
   generate_nsys_stats
   write_manifest COMPLETED
 }
@@ -571,6 +594,7 @@ run_all() {
     "${NSYS_PREFILL_STEPS}" "${PREFILL_ISL}" "${PREFILL_OSL}" 1 1 5303
   stop_service
   CLEANUP_ON_EXIT=0
+  validate_nsys_reports
   generate_nsys_stats
   write_manifest COMPLETED
 }