diff --git a/README.md b/README.md index 8ed8c4d..56ef9b4 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # sskj — 多平台大模型推理性能基准测试项目 +> **更新(2026-07-31 19:02:00 CST)** +> +> Phase 3 Nsight smoke 已在 Head/Worker 各生成一份可解析报告;首份时间线确认 NCCL AllReduce、Sparse MLA、CUTLASS GEMM 与 MoE kernel 均可见。根据 smoke 修正正式 capture:Decode 对照和混合负载必须等服务实际进入 `Decode batch` 后才触发;每段等待本节点全部 8 个 rank 完成,而非任一 rank;双节点近同步停止以减少 Gloo 断链噪声。`nsys stats` 改为可写临时 SQLite、保留 Kernel/API/NVTX 高价值汇总并删除临时库,结果与报告数量均 fail-closed。 +> > **更新(2026-07-31 18:53:00 CST)** > > Phase 3 PyTorch Profiler 首轮 smoke 已证明双节点 16 个 rank 均可完成 trace 写盘,Head/Worker 分别生成 8 份、约 1.5/1.4 GB 压缩 trace。修复当前 OpenSSH 不接受 `scp remote:/path/.` 导致 Worker 结果未回收的问题,改为 SSH tar 流式传输;正式入口不再忽略回收失败,并新增至少 16 份 PyTorch rank trace、至少 2 份 Nsight 节点报告的结果门禁。 diff --git a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/.gitignore b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/.gitignore new file mode 100644 index 0000000..ecba709 --- /dev/null +++ b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/.gitignore @@ -0,0 +1,3 @@ +results/ +runtime/ +*.log diff --git a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh index e8611a3..9eb7b62 100644 --- a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh +++ b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh @@ -278,16 +278,22 @@ start_service() { return 1 } -stop_service_node() { +request_stop_node() { local node="$1" local container_name="$2" - local role="$3" if [[ "${DRY_RUN}" == "1" ]]; then log "[DRY] ${node}: docker stop --time ${NSYS_EXPORT_TIMEOUT_S} ${container_name}" return 0 fi run_on_node "${node}" \ "docker stop --time '${NSYS_EXPORT_TIMEOUT_S}' '${container_name}' >/dev/null 2>&1 || true" +} + +collect_service_node() { + local node="$1" + local container_name="$2" + local role="$3" + [[ "${DRY_RUN}" == "1" ]] && return 0 run_on_node "${node}" "docker logs '${container_name}' 2>&1" \ > "${SERVICE_DIR}/${role}_server.log" 2>&1 || true run_on_node "${node}" "docker inspect '${container_name}' 2>/dev/null" \ @@ -305,12 +311,24 @@ fetch_worker_profiles() { } stop_service() { - stop_service_node "${HEAD_NODE}" "${HEAD_CONTAINER}" head - stop_service_node "${WORKER_NODE}" "${WORKER_CONTAINER}" worker + if [[ "${DRY_RUN}" == "1" ]]; then + request_stop_node "${HEAD_NODE}" "${HEAD_CONTAINER}" + request_stop_node "${WORKER_NODE}" "${WORKER_CONTAINER}" + return 0 + fi + request_stop_node "${HEAD_NODE}" "${HEAD_CONTAINER}" & + local head_stop_pid=$! + request_stop_node "${WORKER_NODE}" "${WORKER_CONTAINER}" & + local worker_stop_pid=$! + wait "${head_stop_pid}" + wait "${worker_stop_pid}" + collect_service_node "${HEAD_NODE}" "${HEAD_CONTAINER}" head + collect_service_node "${WORKER_NODE}" "${WORKER_CONTAINER}" worker fetch_worker_profiles } validate_torch_traces() { + [[ "${DRY_RUN}" == "1" ]] && return 0 local count count="$(find "${RESULT_DIR}/profiles" -type f -name '*.trace.json.gz' | wc -l)" if (( count < TP_SIZE )); then @@ -321,10 +339,12 @@ validate_torch_traces() { } validate_nsys_reports() { - local count + [[ "${DRY_RUN}" == "1" ]] && return 0 + local count expected count="$(find "${RESULT_DIR}/profiles" -type f -name '*.nsys-rep' | wc -l)" - if (( count < NNODES )); then - log "ERROR: expected at least ${NNODES} Nsight reports, found ${count}" + expected=$(( NNODES * CAPTURE_RANGES )) + if (( count < expected )); then + log "ERROR: expected at least ${expected} Nsight reports, found ${count}" return 1 fi log "Nsight report validation passed: ${count} reports" @@ -366,9 +386,10 @@ start_profile() { wait_for_profile_stop() { local before="$1" + local expected=$(( TP_SIZE / NNODES )) local waited=0 while (( waited < PROFILE_STOP_WAIT_S )); do - if (( $(profile_done_count) > before )); then + if (( $(profile_done_count) >= before + expected )); then return 0 fi sleep 2 @@ -378,6 +399,26 @@ wait_for_profile_stop() { return 1 } +server_decode_count() { + run_on_node "${HEAD_NODE}" "docker logs '${HEAD_CONTAINER}' 2>&1" \ + | grep -c 'Decode batch' || true +} + +wait_for_decode_activity() { + local before="$1" + local bench_pid="$2" + local waited=0 + while (( waited < 180 )); do + if (( $(server_decode_count) > before )); then + return 0 + fi + kill -0 "${bench_pid}" 2>/dev/null || return 1 + sleep 1 + ((waited+=1)) + done + return 1 +} + prepare_bench_command() { local output_file="$1" local isl="$2" @@ -455,6 +496,47 @@ run_profiled_case() { [[ "${DRY_RUN}" == "1" ]] || wait_for_profile_stop "${before}" } +run_profiled_decode_case() { + local case_id="$1" + local steps="$2" + local isl="$3" + local osl="$4" + local concurrency="$5" + local prompts="$6" + local seed="$7" + local case_path="${CASE_DIR}/${case_id}" + local decode_before=0 + local profile_before=0 + flush_cache + mkdir -p "${case_path}" + prepare_bench_command "${case_path}/bench.jsonl" \ + "${isl}" "${osl}" "${concurrency}" "${prompts}" "${seed}" + print_command timeout --signal=TERM --kill-after=30s \ + "${PROFILE_TIMEOUT_S}s" "${BENCH_CMD[@]}" > "${case_path}/bench_cmd.txt" + if [[ "${DRY_RUN}" == "1" ]]; then + log "[DRY] start decode workload, wait for Decode batch, then start Nsight range" + printf '[DRY] ' + cat "${case_path}/bench_cmd.txt" + start_profile "${case_id}" '["CUDA_PROFILER"]' "${steps}" + return 0 + fi + decode_before="$(server_decode_count)" + timeout --signal=TERM --kill-after=30s "${PROFILE_TIMEOUT_S}s" \ + "${BENCH_CMD[@]}" > "${case_path}/bench.log" 2>&1 & + local bench_pid=$! + wait_for_bench_main "${case_path}/bench.log" "${bench_pid}" \ + && wait_for_decode_activity "${decode_before}" "${bench_pid}" || { + kill "${bench_pid}" 2>/dev/null || true + wait "${bench_pid}" 2>/dev/null || true + log "ERROR: ${case_id} did not reach active Decode" + return 1 + } + profile_before="$(profile_done_count)" + start_profile "${case_id}" '["CUDA_PROFILER"]' "${steps}" + wait_for_profile_stop "${profile_before}" + wait "${bench_pid}" +} + wait_for_bench_main() { local bench_log="$1" local bench_pid="$2" @@ -474,6 +556,7 @@ run_mixed_profile() { local injection_id="${case_id}/prefill_injection" local background_path="${CASE_DIR}/${background_id}" local before=0 + local decode_before=0 flush_cache mkdir -p "${background_path}" prepare_bench_command "${background_path}/bench.jsonl" \ @@ -481,16 +564,18 @@ run_mixed_profile() { print_command timeout --signal=TERM --kill-after=30s \ "${PROFILE_TIMEOUT_S}s" "${BENCH_CMD[@]}" > "${background_path}/bench_cmd.txt" if [[ "${DRY_RUN}" == "1" ]]; then - log "[DRY] start decode background, wait for main run, start profile, inject 128K prefill" + log "[DRY] start decode background, wait for Decode batch, start profile, inject 128K prefill" start_profile "${case_id}" '["CUDA_PROFILER"]' "${NSYS_MIXED_STEPS}" run_bench "${injection_id}" "${MIXED_INJECTION_ISL}" \ "${MIXED_INJECTION_OSL}" 1 1 5202 return 0 fi + decode_before="$(server_decode_count)" timeout --signal=TERM --kill-after=30s "${PROFILE_TIMEOUT_S}s" \ "${BENCH_CMD[@]}" > "${background_path}/bench.log" 2>&1 & local background_pid=$! - wait_for_bench_main "${background_path}/bench.log" "${background_pid}" || { + wait_for_bench_main "${background_path}/bench.log" "${background_pid}" \ + && wait_for_decode_activity "${decode_before}" "${background_pid}" || { kill "${background_pid}" 2>/dev/null || true wait "${background_pid}" 2>/dev/null || true log "ERROR: mixed decode background did not enter main benchmark" @@ -537,10 +622,16 @@ generate_nsys_stats() { relative="${report#${RESULT_DIR}/}" output="${report%.nsys-rep}.stats.txt" docker run --rm \ - -v "${RESULT_DIR}:/results:ro" \ + -v "${RESULT_DIR}:/results" \ --entrypoint nsys "${DOCKER_IMAGE}" stats \ - --report cuda_gpu_kern_sum,cuda_api_sum,cuda_kern_exec_sum,nvtx_sum \ - "/results/${relative}" > "${output}" 2>&1 || true + --force-export=true \ + --report cuda_gpu_kern_sum,cuda_api_sum,nvtx_sum \ + "/results/${relative}" > "${output}" 2>&1 + grep -q 'CUDA GPU Kernel Summary' "${output}" || { + log "ERROR: nsys stats missing CUDA kernel summary for ${relative}" + return 1 + } + rm -f "${report%.nsys-rep}.sqlite" done < <(find "${RESULT_DIR}/profiles" -type f -name '*.nsys-rep' | sort) } @@ -586,7 +677,7 @@ run_all() { CLEANUP_ON_EXIT=1 start_service write_manifest RUNNING - run_profiled_case decode_control '["CUDA_PROFILER"]' \ + run_profiled_decode_case decode_control \ "${NSYS_CONTROL_STEPS}" "${CONTROL_ISL}" "${CONTROL_OSL}" \ "${CONTROL_CONCURRENCY}" "${CONTROL_PROMPTS}" 5301 run_mixed_profile