[BugFix] align Phase 3 captures with active decode
This commit is contained in:
Zhiyi Hong
parent
405608ad23
commit
119701a417
@ -1,5 +1,9 @@
|
||||
# sskj — 多平台大模型推理性能基准测试项目
|
||||
|
||||
> **更新（2026-07-31 19:02:00 CST）**
|
||||
>
|
||||
> Phase 3 Nsight smoke 已在 Head/Worker 各生成一份可解析报告；首份时间线确认 NCCL AllReduce、Sparse MLA、CUTLASS GEMM 与 MoE kernel 均可见。根据 smoke 修正正式 capture：Decode 对照和混合负载必须等服务实际进入 `Decode batch` 后才触发；每段等待本节点全部 8 个 rank 完成，而非任一 rank；双节点近同步停止以减少 Gloo 断链噪声。`nsys stats` 改为可写临时 SQLite、保留 Kernel/API/NVTX 高价值汇总并删除临时库，结果与报告数量均 fail-closed。
|
||||
>
|
||||
> **更新（2026-07-31 18:53:00 CST）**
|
||||
>
|
||||
> Phase 3 PyTorch Profiler 首轮 smoke 已证明双节点 16 个 rank 均可完成 trace 写盘，Head/Worker 分别生成 8 份、约 1.5/1.4 GB 压缩 trace。修复当前 OpenSSH 不接受 `scp remote:/path/.` 导致 Worker 结果未回收的问题，改为 SSH tar 流式传输；正式入口不再忽略回收失败，并新增至少 16 份 PyTorch rank trace、至少 2 份 Nsight 节点报告的结果门禁。
|
||||
|
||||
3
experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/.gitignore
vendored
Normal file
3
experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/.gitignore
vendored
Normal file
@ -0,0 +1,3 @@
|
||||
results/
|
||||
runtime/
|
||||
*.log
|
||||
@ -278,16 +278,22 @@ start_service() {
|
||||
return 1
|
||||
}
|
||||
|
||||
stop_service_node() {
|
||||
request_stop_node() {
|
||||
local node="$1"
|
||||
local container_name="$2"
|
||||
local role="$3"
|
||||
if [[ "${DRY_RUN}" == "1" ]]; then
|
||||
log "[DRY] ${node}: docker stop --time ${NSYS_EXPORT_TIMEOUT_S} ${container_name}"
|
||||
return 0
|
||||
fi
|
||||
run_on_node "${node}" \
|
||||
"docker stop --time '${NSYS_EXPORT_TIMEOUT_S}' '${container_name}' >/dev/null 2>&1 || true"
|
||||
}
|
||||
|
||||
collect_service_node() {
|
||||
local node="$1"
|
||||
local container_name="$2"
|
||||
local role="$3"
|
||||
[[ "${DRY_RUN}" == "1" ]] && return 0
|
||||
run_on_node "${node}" "docker logs '${container_name}' 2>&1" \
|
||||
> "${SERVICE_DIR}/${role}_server.log" 2>&1 || true
|
||||
run_on_node "${node}" "docker inspect '${container_name}' 2>/dev/null" \
|
||||
@ -305,12 +311,24 @@ fetch_worker_profiles() {
|
||||
}
|
||||
|
||||
stop_service() {
|
||||
stop_service_node "${HEAD_NODE}" "${HEAD_CONTAINER}" head
|
||||
stop_service_node "${WORKER_NODE}" "${WORKER_CONTAINER}" worker
|
||||
if [[ "${DRY_RUN}" == "1" ]]; then
|
||||
request_stop_node "${HEAD_NODE}" "${HEAD_CONTAINER}"
|
||||
request_stop_node "${WORKER_NODE}" "${WORKER_CONTAINER}"
|
||||
return 0
|
||||
fi
|
||||
request_stop_node "${HEAD_NODE}" "${HEAD_CONTAINER}" &
|
||||
local head_stop_pid=$!
|
||||
request_stop_node "${WORKER_NODE}" "${WORKER_CONTAINER}" &
|
||||
local worker_stop_pid=$!
|
||||
wait "${head_stop_pid}"
|
||||
wait "${worker_stop_pid}"
|
||||
collect_service_node "${HEAD_NODE}" "${HEAD_CONTAINER}" head
|
||||
collect_service_node "${WORKER_NODE}" "${WORKER_CONTAINER}" worker
|
||||
fetch_worker_profiles
|
||||
}
|
||||
|
||||
validate_torch_traces() {
|
||||
[[ "${DRY_RUN}" == "1" ]] && return 0
|
||||
local count
|
||||
count="$(find "${RESULT_DIR}/profiles" -type f -name '*.trace.json.gz' | wc -l)"
|
||||
if (( count < TP_SIZE )); then
|
||||
@ -321,10 +339,12 @@ validate_torch_traces() {
|
||||
}
|
||||
|
||||
validate_nsys_reports() {
|
||||
local count
|
||||
[[ "${DRY_RUN}" == "1" ]] && return 0
|
||||
local count expected
|
||||
count="$(find "${RESULT_DIR}/profiles" -type f -name '*.nsys-rep' | wc -l)"
|
||||
if (( count < NNODES )); then
|
||||
log "ERROR: expected at least ${NNODES} Nsight reports, found ${count}"
|
||||
expected=$(( NNODES * CAPTURE_RANGES ))
|
||||
if (( count < expected )); then
|
||||
log "ERROR: expected at least ${expected} Nsight reports, found ${count}"
|
||||
return 1
|
||||
fi
|
||||
log "Nsight report validation passed: ${count} reports"
|
||||
@ -366,9 +386,10 @@ start_profile() {
|
||||
|
||||
wait_for_profile_stop() {
|
||||
local before="$1"
|
||||
local expected=$(( TP_SIZE / NNODES ))
|
||||
local waited=0
|
||||
while (( waited < PROFILE_STOP_WAIT_S )); do
|
||||
if (( $(profile_done_count) > before )); then
|
||||
if (( $(profile_done_count) >= before + expected )); then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
@ -378,6 +399,26 @@ wait_for_profile_stop() {
|
||||
return 1
|
||||
}
|
||||
|
||||
server_decode_count() {
|
||||
run_on_node "${HEAD_NODE}" "docker logs '${HEAD_CONTAINER}' 2>&1" \
|
||||
| grep -c 'Decode batch' || true
|
||||
}
|
||||
|
||||
wait_for_decode_activity() {
|
||||
local before="$1"
|
||||
local bench_pid="$2"
|
||||
local waited=0
|
||||
while (( waited < 180 )); do
|
||||
if (( $(server_decode_count) > before )); then
|
||||
return 0
|
||||
fi
|
||||
kill -0 "${bench_pid}" 2>/dev/null || return 1
|
||||
sleep 1
|
||||
((waited+=1))
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
prepare_bench_command() {
|
||||
local output_file="$1"
|
||||
local isl="$2"
|
||||
@ -455,6 +496,47 @@ run_profiled_case() {
|
||||
[[ "${DRY_RUN}" == "1" ]] || wait_for_profile_stop "${before}"
|
||||
}
|
||||
|
||||
run_profiled_decode_case() {
|
||||
local case_id="$1"
|
||||
local steps="$2"
|
||||
local isl="$3"
|
||||
local osl="$4"
|
||||
local concurrency="$5"
|
||||
local prompts="$6"
|
||||
local seed="$7"
|
||||
local case_path="${CASE_DIR}/${case_id}"
|
||||
local decode_before=0
|
||||
local profile_before=0
|
||||
flush_cache
|
||||
mkdir -p "${case_path}"
|
||||
prepare_bench_command "${case_path}/bench.jsonl" \
|
||||
"${isl}" "${osl}" "${concurrency}" "${prompts}" "${seed}"
|
||||
print_command timeout --signal=TERM --kill-after=30s \
|
||||
"${PROFILE_TIMEOUT_S}s" "${BENCH_CMD[@]}" > "${case_path}/bench_cmd.txt"
|
||||
if [[ "${DRY_RUN}" == "1" ]]; then
|
||||
log "[DRY] start decode workload, wait for Decode batch, then start Nsight range"
|
||||
printf '[DRY] '
|
||||
cat "${case_path}/bench_cmd.txt"
|
||||
start_profile "${case_id}" '["CUDA_PROFILER"]' "${steps}"
|
||||
return 0
|
||||
fi
|
||||
decode_before="$(server_decode_count)"
|
||||
timeout --signal=TERM --kill-after=30s "${PROFILE_TIMEOUT_S}s" \
|
||||
"${BENCH_CMD[@]}" > "${case_path}/bench.log" 2>&1 &
|
||||
local bench_pid=$!
|
||||
wait_for_bench_main "${case_path}/bench.log" "${bench_pid}" \
|
||||
&& wait_for_decode_activity "${decode_before}" "${bench_pid}" || {
|
||||
kill "${bench_pid}" 2>/dev/null || true
|
||||
wait "${bench_pid}" 2>/dev/null || true
|
||||
log "ERROR: ${case_id} did not reach active Decode"
|
||||
return 1
|
||||
}
|
||||
profile_before="$(profile_done_count)"
|
||||
start_profile "${case_id}" '["CUDA_PROFILER"]' "${steps}"
|
||||
wait_for_profile_stop "${profile_before}"
|
||||
wait "${bench_pid}"
|
||||
}
|
||||
|
||||
wait_for_bench_main() {
|
||||
local bench_log="$1"
|
||||
local bench_pid="$2"
|
||||
@ -474,6 +556,7 @@ run_mixed_profile() {
|
||||
local injection_id="${case_id}/prefill_injection"
|
||||
local background_path="${CASE_DIR}/${background_id}"
|
||||
local before=0
|
||||
local decode_before=0
|
||||
flush_cache
|
||||
mkdir -p "${background_path}"
|
||||
prepare_bench_command "${background_path}/bench.jsonl" \
|
||||
@ -481,16 +564,18 @@ run_mixed_profile() {
|
||||
print_command timeout --signal=TERM --kill-after=30s \
|
||||
"${PROFILE_TIMEOUT_S}s" "${BENCH_CMD[@]}" > "${background_path}/bench_cmd.txt"
|
||||
if [[ "${DRY_RUN}" == "1" ]]; then
|
||||
log "[DRY] start decode background, wait for main run, start profile, inject 128K prefill"
|
||||
log "[DRY] start decode background, wait for Decode batch, start profile, inject 128K prefill"
|
||||
start_profile "${case_id}" '["CUDA_PROFILER"]' "${NSYS_MIXED_STEPS}"
|
||||
run_bench "${injection_id}" "${MIXED_INJECTION_ISL}" \
|
||||
"${MIXED_INJECTION_OSL}" 1 1 5202
|
||||
return 0
|
||||
fi
|
||||
decode_before="$(server_decode_count)"
|
||||
timeout --signal=TERM --kill-after=30s "${PROFILE_TIMEOUT_S}s" \
|
||||
"${BENCH_CMD[@]}" > "${background_path}/bench.log" 2>&1 &
|
||||
local background_pid=$!
|
||||
wait_for_bench_main "${background_path}/bench.log" "${background_pid}" || {
|
||||
wait_for_bench_main "${background_path}/bench.log" "${background_pid}" \
|
||||
&& wait_for_decode_activity "${decode_before}" "${background_pid}" || {
|
||||
kill "${background_pid}" 2>/dev/null || true
|
||||
wait "${background_pid}" 2>/dev/null || true
|
||||
log "ERROR: mixed decode background did not enter main benchmark"
|
||||
@ -537,10 +622,16 @@ generate_nsys_stats() {
|
||||
relative="${report#${RESULT_DIR}/}"
|
||||
output="${report%.nsys-rep}.stats.txt"
|
||||
docker run --rm \
|
||||
-v "${RESULT_DIR}:/results:ro" \
|
||||
-v "${RESULT_DIR}:/results" \
|
||||
--entrypoint nsys "${DOCKER_IMAGE}" stats \
|
||||
--report cuda_gpu_kern_sum,cuda_api_sum,cuda_kern_exec_sum,nvtx_sum \
|
||||
"/results/${relative}" > "${output}" 2>&1 || true
|
||||
--force-export=true \
|
||||
--report cuda_gpu_kern_sum,cuda_api_sum,nvtx_sum \
|
||||
"/results/${relative}" > "${output}" 2>&1
|
||||
grep -q 'CUDA GPU Kernel Summary' "${output}" || {
|
||||
log "ERROR: nsys stats missing CUDA kernel summary for ${relative}"
|
||||
return 1
|
||||
}
|
||||
rm -f "${report%.nsys-rep}.sqlite"
|
||||
done < <(find "${RESULT_DIR}/profiles" -type f -name '*.nsys-rep' | sort)
|
||||
}
|
||||
|
||||
@ -586,7 +677,7 @@ run_all() {
|
||||
CLEANUP_ON_EXIT=1
|
||||
start_service
|
||||
write_manifest RUNNING
|
||||
run_profiled_case decode_control '["CUDA_PROFILER"]' \
|
||||
run_profiled_decode_case decode_control \
|
||||
"${NSYS_CONTROL_STEPS}" "${CONTROL_ISL}" "${CONTROL_OSL}" \
|
||||
"${CONTROL_CONCURRENCY}" "${CONTROL_PROMPTS}" 5301
|
||||
run_mixed_profile
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user