qqtang/sskj
1
3
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[BugFix] make Phase 3 artifact collection fail closed

Browse Source
This commit is contained in:
Zhiyi Hong 2026-07-31 18:53:19 +08:00
parent 771b868757
commit 405608ad23
2 changed files with 31 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@ -1,5 +1,9 @@
# sskj — 多平台大模型推理性能基准测试项目 # sskj — 多平台大模型推理性能基准测试项目
> **更新（2026-07-31 18:53:00 CST）**
>
> Phase 3 PyTorch Profiler 首轮 smoke 已证明双节点 16 个 rank 均可完成 trace 写盘，Head/Worker 分别生成 8 份、约 1.5/1.4 GB 压缩 trace。修复当前 OpenSSH 不接受 `scp remote:/path/.` 导致 Worker 结果未回收的问题，改为 SSH tar 流式传输；正式入口不再忽略回收失败，并新增至少 16 份 PyTorch rank trace、至少 2 份 Nsight 节点报告的结果门禁。
>
> **更新（2026-07-31 18:43:00 CST）** > **更新（2026-07-31 18:43:00 CST）**
> >
> 为 Phase 2 实验档案第 11 节的 GPU/DCGM、CPU/进程/NUMA、双 Rail RDMA、PCIe/NCCL 四组结果补充服务器证据路径。每组同时标明结构化汇总 CSV、Head/Worker 原始采样日志、通信原始输出及实际命令文件，便于从结论直接追溯最终 Run `dsv4pro-phase2-20260731-163620` 的证据。 > 为 Phase 2 实验档案第 11 节的 GPU/DCGM、CPU/进程/NUMA、双 Rail RDMA、PCIe/NCCL 四组结果补充服务器证据路径。每组同时标明结构化汇总 CSV、Head/Worker 原始采样日志、通信原始输出及实际命令文件，便于从结论直接追溯最终 Run `dsv4pro-phase2-20260731-163620` 的证据。

30
experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh
View File

@ -298,15 +298,36 @@ stop_service_node() {
fetch_worker_profiles() { fetch_worker_profiles() {
[[ "${DRY_RUN}" == "1" ]] && return 0 [[ "${DRY_RUN}" == "1" ]] && return 0
mkdir -p "${RESULT_DIR}/profiles/worker" mkdir -p "${RESULT_DIR}/profiles/worker"
scp -q -r "${WORKER_NODE}:${WORKER_PROFILE_HOST}/." \ ssh -o BatchMode=yes -o StrictHostKeyChecking=no "${WORKER_NODE}" \
"${RESULT_DIR}/profiles/worker/" "tar -C '${WORKER_PROFILE_HOST}' -cf - ." \
| tar -C "${RESULT_DIR}/profiles/worker" -xf -
run_on_node "${WORKER_NODE}" "rm -rf '${WORKER_RUN_DIR}'" run_on_node "${WORKER_NODE}" "rm -rf '${WORKER_RUN_DIR}'"
} }
stop_service() { stop_service() {
stop_service_node "${HEAD_NODE}" "${HEAD_CONTAINER}" head stop_service_node "${HEAD_NODE}" "${HEAD_CONTAINER}" head
stop_service_node "${WORKER_NODE}" "${WORKER_CONTAINER}" worker stop_service_node "${WORKER_NODE}" "${WORKER_CONTAINER}" worker
fetch_worker_profiles || true fetch_worker_profiles
}
validate_torch_traces() {
local count
count="$(find "${RESULT_DIR}/profiles" -type f -name '*.trace.json.gz' | wc -l)"
if (( count < TP_SIZE )); then
log "ERROR: expected at least ${TP_SIZE} rank traces, found ${count}"
return 1
fi
log "PyTorch trace validation passed: ${count} rank traces"
}
validate_nsys_reports() {
local count
count="$(find "${RESULT_DIR}/profiles" -type f -name '*.nsys-rep' | wc -l)"
if (( count < NNODES )); then
log "ERROR: expected at least ${NNODES} Nsight reports, found ${count}"
return 1
fi
log "Nsight report validation passed: ${count} reports"
} }
flush_cache() { flush_cache() {
@ -539,6 +560,7 @@ run_torch_smoke() {
"${TORCH_SMOKE_STEPS}" "${CONTROL_ISL}" 128 8 8 5101 "${TORCH_SMOKE_STEPS}" "${CONTROL_ISL}" 128 8 8 5101
stop_service stop_service
CLEANUP_ON_EXIT=0 CLEANUP_ON_EXIT=0
validate_torch_traces
write_manifest COMPLETED write_manifest COMPLETED
} }
@ -553,6 +575,7 @@ run_nsys_smoke() {
"${CONTROL_CONCURRENCY}" "${CONTROL_PROMPTS}" 5102 "${CONTROL_CONCURRENCY}" "${CONTROL_PROMPTS}" 5102
stop_service stop_service
CLEANUP_ON_EXIT=0 CLEANUP_ON_EXIT=0
validate_nsys_reports
generate_nsys_stats generate_nsys_stats
write_manifest COMPLETED write_manifest COMPLETED
} }
@ -571,6 +594,7 @@ run_all() {
"${NSYS_PREFILL_STEPS}" "${PREFILL_ISL}" "${PREFILL_OSL}" 1 1 5303 "${NSYS_PREFILL_STEPS}" "${PREFILL_ISL}" "${PREFILL_OSL}" 1 1 5303
stop_service stop_service
CLEANUP_ON_EXIT=0 CLEANUP_ON_EXIT=0
validate_nsys_reports
generate_nsys_stats generate_nsys_stats
write_manifest COMPLETED write_manifest COMPLETED
} }