diff --git a/README.md b/README.md index c3bc9bd..5c82256 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # sskj — 多平台大模型推理性能基准测试项目 +> **更新(2026-08-02 00:08:00 CST)** +> +> 修复 DeepSeek-V4-Pro 双机 Pro6000D SGLang Phase 3 混合 Capture 时序。首轮正式 Run 证明当前 SGLang `/start_profile` 请求会阻塞到捕获结束,旧实现因此在 Profile 完成后才发送 128K Prefill,第二份 Trace 未覆盖注入;随后混合背景触发默认 300 秒 Scheduler watchdog,第三段未执行。新实现异步触发 Profile,先捕获 2 个 Decode Step,再通过原生 `/generate` + `input_ids` 立即注入 128K 请求,避免 benchmark 客户端启动延迟;混合背景缩为单波 32 请求,Profiling 专用 watchdog 提升至 1800 秒,并让异常退出自动将 manifest 标记为 `FAILED`。 +> > **更新(2026-08-01 19:34:34 CST)** > > 完成 DeepSeek-V4-Pro 双机 Pro6000D SGLang Phase 3 正式时间线代码交付。正式 Control 与混合背景统一为 `1K -> 1K, C=32`;所有 `/start_profile` 请求显式记录 `start_step` 和 `num_steps`,Control 在活跃 Decode 后跳过 2 Step,混合场景先保留 2 个纯 Decode Step 再注入 `128K -> 1`,长 Prefill 从首个 Chunk 开始捕获。继续沿用已通过的双节点 PyTorch/Nsight smoke、16 Rank/6 份 Nsight 报告 fail-closed 门禁和单服务三段 Capture Range;正式结果尚未生成,因此不创建 `phase3_exp.html` 或 `phase3_code.html`。 diff --git a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/README.md b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/README.md index 84c6936..1eb8036 100644 --- a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/README.md +++ b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/README.md @@ -31,8 +31,10 @@ staging paths. No `source`, Conda activation, or command on Worker is required. 1. `decode_control`: `1K -> 1K`, `C=32`; trigger after Decode becomes active, skip 2 engine steps, then capture 16 steps. 2. `mixed_decode_with_128k_prefill`: active `1K -> 1K`, `C=32` decode; - capture 2 pure Decode steps before injecting one `128K -> 1` request, then - retain a 32-step range spanning both sides of the injection. + start `/start_profile` asynchronously, capture 2 pure Decode steps, then + inject one native `input_ids` based `128K -> 1` request without benchmark + client startup delay. The 32-step range therefore spans both sides of the + injection. 3. `long_prefill`: isolated `128K -> 1`, `C=1`. The service remains under one Nsight process for all three ranges, so it uses diff --git a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/config.env b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/config.env index 3278c53..a2f85c5 100644 --- a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/config.env +++ b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/config.env @@ -17,6 +17,7 @@ WORKER_STAGE_ROOT="${WORKER_STAGE_ROOT:-/data/hzy/dsv4pro_phase3_worker}" NSYS_TRACE="${NSYS_TRACE:-cuda,nvtx,nccl,osrt}" NSYS_CUDA_GRAPH_TRACE="${NSYS_CUDA_GRAPH_TRACE:-node}" NSYS_EXPORT_TIMEOUT_S="${NSYS_EXPORT_TIMEOUT_S:-300}" +PROFILE_WATCHDOG_TIMEOUT_S="${PROFILE_WATCHDOG_TIMEOUT_S:-1800}" NSYS_CONTROL_STEPS="${NSYS_CONTROL_STEPS:-16}" NSYS_MIXED_STEPS="${NSYS_MIXED_STEPS:-32}" NSYS_PREFILL_STEPS="${NSYS_PREFILL_STEPS:-16}" @@ -39,9 +40,10 @@ CONTROL_PROMPTS="${CONTROL_PROMPTS:-32}" MIXED_ISL="${MIXED_ISL:-1024}" MIXED_OSL="${MIXED_OSL:-1024}" MIXED_CONCURRENCY="${MIXED_CONCURRENCY:-32}" -MIXED_PROMPTS="${MIXED_PROMPTS:-64}" +MIXED_PROMPTS="${MIXED_PROMPTS:-32}" MIXED_INJECTION_ISL="${MIXED_INJECTION_ISL:-131072}" MIXED_INJECTION_OSL="${MIXED_INJECTION_OSL:-1}" +MIXED_INJECTION_TOKEN_ID="${MIXED_INJECTION_TOKEN_ID:-100}" PREFILL_ISL="${PREFILL_ISL:-131072}" PREFILL_OSL="${PREFILL_OSL:-1}" diff --git a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh index 46e9dd0..2bcb450 100644 --- a/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh +++ b/experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh @@ -209,6 +209,7 @@ build_server_command() { --mem-fraction-static "${MEM_FRACTION_STATIC}" --cuda-graph-max-bs-decode "${CUDA_GRAPH_MAX_BS_DECODE}" --max-running-requests "${MAX_RUNNING_REQUESTS}" + --watchdog-timeout "${PROFILE_WATCHDOG_TIMEOUT_S}" ) if [[ -n "${SGLANG_EXTRA_ARGS}" ]]; then local -a extra_args=() @@ -364,6 +365,11 @@ profile_done_count() { | grep -c 'Profiling done' || true } +profile_started_count() { + run_on_node "${HEAD_NODE}" "docker logs '${HEAD_CONTAINER}' 2>&1" \ + | grep -c 'Profiling starts' || true +} + start_profile() { local case_id="$1" local activities_json="$2" @@ -400,6 +406,23 @@ wait_for_profile_stop() { return 1 } +wait_for_profile_start() { + local before="$1" + local profile_request_pid="$2" + local expected=$(( TP_SIZE / NNODES )) + local waited=0 + while (( waited < PROFILE_STOP_WAIT_S )); do + if (( $(profile_started_count) >= before + expected )); then + return 0 + fi + kill -0 "${profile_request_pid}" 2>/dev/null || return 1 + sleep 1 + ((waited+=1)) + done + log "ERROR: profiler did not start within ${PROFILE_STOP_WAIT_S}s" + return 1 +} + server_decode_count() { run_on_node "${HEAD_NODE}" "docker logs '${HEAD_CONTAINER}' 2>&1" \ | grep -c 'Decode batch' || true @@ -487,6 +510,51 @@ run_bench() { "${BENCH_CMD[@]}" > "${case_path}/bench.log" 2>&1 } +prepare_native_injection_payload() { + local case_path="$1" + local payload_file="${case_path}/request.json" + mkdir -p "${case_path}" + python3 -c 'import json, sys +path, length, token_id, output_len = sys.argv[1:] +payload = { + "input_ids": [int(token_id)] * int(length), + "sampling_params": { + "max_new_tokens": int(output_len), + "temperature": 0, + "ignore_eos": True, + }, + "stream": False, +} +with open(path, "w", encoding="utf-8") as f: + json.dump(payload, f, separators=(",", ":")) +' "${payload_file}" "${MIXED_INJECTION_ISL}" \ + "${MIXED_INJECTION_TOKEN_ID}" "${MIXED_INJECTION_OSL}" +} + +run_native_injection() { + local case_id="$1" + local case_path="${CASE_DIR}/${case_id}" + local payload_file="${case_path}/request.json" + local response_file="${case_path}/response.json" + prepare_native_injection_payload "${case_path}" + print_command timeout --signal=TERM --kill-after=30s \ + "${PROFILE_TIMEOUT_S}s" curl --fail --silent --show-error \ + -X POST "http://${HEAD_IP}:${SGLANG_PORT}/generate" \ + -H 'Content-Type: application/json' --data-binary "@${payload_file}" \ + -o "${response_file}" > "${case_path}/curl_cmd.txt" + if [[ "${DRY_RUN}" == "1" ]]; then + printf '[DRY] ' + cat "${case_path}/curl_cmd.txt" + return 0 + fi + timeout --signal=TERM --kill-after=30s "${PROFILE_TIMEOUT_S}s" \ + curl --fail --silent --show-error \ + -X POST "http://${HEAD_IP}:${SGLANG_PORT}/generate" \ + -H 'Content-Type: application/json' --data-binary "@${payload_file}" \ + -o "${response_file}" + python3 -m json.tool "${response_file}" >/dev/null +} + run_profiled_case() { local case_id="$1" local activities_json="$2" @@ -568,6 +636,8 @@ run_mixed_profile() { local before=0 local decode_before=0 local mixed_baseline_before=0 + local profile_started_before=0 + local profile_request_pid=0 flush_cache mkdir -p "${background_path}" prepare_bench_command "${background_path}/bench.jsonl" \ @@ -575,12 +645,11 @@ run_mixed_profile() { print_command timeout --signal=TERM --kill-after=30s \ "${PROFILE_TIMEOUT_S}s" "${BENCH_CMD[@]}" > "${background_path}/bench_cmd.txt" if [[ "${DRY_RUN}" == "1" ]]; then - log "[DRY] start decode background, wait for Decode batch, start profile, inject 128K prefill" + log "[DRY] start decode background, wait for Decode batch, start profile asynchronously, inject native 128K prefill" start_profile "${case_id}" '["CUDA_PROFILER"]' \ "${NSYS_MIXED_START_STEP}" "${NSYS_MIXED_STEPS}" log "[DRY] wait ${MIXED_PRE_INJECTION_DECODE_STEPS} Decode steps before injection" - run_bench "${injection_id}" "${MIXED_INJECTION_ISL}" \ - "${MIXED_INJECTION_OSL}" 1 1 5202 + run_native_injection "${injection_id}" return 0 fi decode_before="$(server_decode_count)" @@ -595,8 +664,16 @@ run_mixed_profile() { return 1 } before="$(profile_done_count)" + profile_started_before="$(profile_started_count)" start_profile "${case_id}" '["CUDA_PROFILER"]' \ - "${NSYS_MIXED_START_STEP}" "${NSYS_MIXED_STEPS}" + "${NSYS_MIXED_START_STEP}" "${NSYS_MIXED_STEPS}" & + profile_request_pid=$! + wait_for_profile_start "${profile_started_before}" "${profile_request_pid}" || { + kill "${profile_request_pid}" "${background_pid}" 2>/dev/null || true + wait "${profile_request_pid}" "${background_pid}" 2>/dev/null || true + log "ERROR: mixed profile request did not become active" + return 1 + } mixed_baseline_before="$(server_decode_count)" wait_for_decode_steps "${mixed_baseline_before}" \ "${MIXED_PRE_INJECTION_DECODE_STEPS}" "${background_pid}" || { @@ -605,8 +682,8 @@ run_mixed_profile() { log "ERROR: mixed profile did not capture the pre-injection Decode baseline" return 1 } - run_bench "${injection_id}" "${MIXED_INJECTION_ISL}" \ - "${MIXED_INJECTION_OSL}" 1 1 5202 + run_native_injection "${injection_id}" + wait "${profile_request_pid}" wait_for_profile_stop "${before}" wait "${background_pid}" } @@ -668,6 +745,16 @@ cleanup() { fi } +on_exit() { + local exit_code=$? + trap - EXIT + if (( exit_code != 0 )) && [[ -d "${RESULT_DIR}" ]]; then + write_manifest FAILED || true + fi + cleanup + exit "${exit_code}" +} + run_torch_smoke() { PROFILE_MODE="torch" CAPTURE_RANGES=0 @@ -721,7 +808,8 @@ run_all() { write_manifest COMPLETED } -trap cleanup EXIT INT TERM +trap on_exit EXIT +trap 'exit 130' INT TERM case "${ACTION}" in audit)