[BugFix] align mixed Phase 3 capture with prefill injection

README.md

@@ -1,5 +1,9 @@
 # sskj — 多平台大模型推理性能基准测试项目
 
+> **更新（2026-08-02 00:08:00 CST）**
+>
+> 修复 DeepSeek-V4-Pro 双机 Pro6000D SGLang Phase 3 混合 Capture 时序。首轮正式 Run 证明当前 SGLang `/start_profile` 请求会阻塞到捕获结束，旧实现因此在 Profile 完成后才发送 128K Prefill，第二份 Trace 未覆盖注入；随后混合背景触发默认 300 秒 Scheduler watchdog，第三段未执行。新实现异步触发 Profile，先捕获 2 个 Decode Step，再通过原生 `/generate` + `input_ids` 立即注入 128K 请求，避免 benchmark 客户端启动延迟；混合背景缩为单波 32 请求，Profiling 专用 watchdog 提升至 1800 秒，并让异常退出自动将 manifest 标记为 `FAILED`。
+>
 > **更新（2026-08-01 19:34:34 CST）**
 >
 > 完成 DeepSeek-V4-Pro 双机 Pro6000D SGLang Phase 3 正式时间线代码交付。正式 Control 与混合背景统一为 `1K -> 1K, C=32`；所有 `/start_profile` 请求显式记录 `start_step` 和 `num_steps`，Control 在活跃 Decode 后跳过 2 Step，混合场景先保留 2 个纯 Decode Step 再注入 `128K -> 1`，长 Prefill 从首个 Chunk 开始捕获。继续沿用已通过的双节点 PyTorch/Nsight smoke、16 Rank/6 份 Nsight 报告 fail-closed 门禁和单服务三段 Capture Range；正式结果尚未生成，因此不创建 `phase3_exp.html` 或 `phase3_code.html`。

experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/README.md

@@ -31,8 +31,10 @@ staging paths. No `source`, Conda activation, or command on Worker is required.
 1. `decode_control`: `1K -> 1K`, `C=32`; trigger after Decode becomes active,
    skip 2 engine steps, then capture 16 steps.
 2. `mixed_decode_with_128k_prefill`: active `1K -> 1K`, `C=32` decode;
-   capture 2 pure Decode steps before injecting one `128K -> 1` request, then
+   start `/start_profile` asynchronously, capture 2 pure Decode steps, then
-   retain a 32-step range spanning both sides of the injection.
+   inject one native `input_ids` based `128K -> 1` request without benchmark
+   client startup delay. The 32-step range therefore spans both sides of the
+   injection.
 3. `long_prefill`: isolated `128K -> 1`, `C=1`.
 
 The service remains under one Nsight process for all three ranges, so it uses

experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/config.env

@@ -17,6 +17,7 @@ WORKER_STAGE_ROOT="${WORKER_STAGE_ROOT:-/data/hzy/dsv4pro_phase3_worker}"
 NSYS_TRACE="${NSYS_TRACE:-cuda,nvtx,nccl,osrt}"
 NSYS_CUDA_GRAPH_TRACE="${NSYS_CUDA_GRAPH_TRACE:-node}"
 NSYS_EXPORT_TIMEOUT_S="${NSYS_EXPORT_TIMEOUT_S:-300}"
+PROFILE_WATCHDOG_TIMEOUT_S="${PROFILE_WATCHDOG_TIMEOUT_S:-1800}"
 NSYS_CONTROL_STEPS="${NSYS_CONTROL_STEPS:-16}"
 NSYS_MIXED_STEPS="${NSYS_MIXED_STEPS:-32}"
 NSYS_PREFILL_STEPS="${NSYS_PREFILL_STEPS:-16}"
@@ -39,9 +40,10 @@ CONTROL_PROMPTS="${CONTROL_PROMPTS:-32}"
 MIXED_ISL="${MIXED_ISL:-1024}"
 MIXED_OSL="${MIXED_OSL:-1024}"
 MIXED_CONCURRENCY="${MIXED_CONCURRENCY:-32}"
-MIXED_PROMPTS="${MIXED_PROMPTS:-64}"
+MIXED_PROMPTS="${MIXED_PROMPTS:-32}"
 MIXED_INJECTION_ISL="${MIXED_INJECTION_ISL:-131072}"
 MIXED_INJECTION_OSL="${MIXED_INJECTION_OSL:-1}"
+MIXED_INJECTION_TOKEN_ID="${MIXED_INJECTION_TOKEN_ID:-100}"
 
 PREFILL_ISL="${PREFILL_ISL:-131072}"
 PREFILL_OSL="${PREFILL_OSL:-1}"

experiments/pro6000/dsv4pro_pro6000d_2node_sglang_timeline_profiling/run_timeline_profiling.sh

@@ -209,6 +209,7 @@ build_server_command() {
     --mem-fraction-static "${MEM_FRACTION_STATIC}"
     --cuda-graph-max-bs-decode "${CUDA_GRAPH_MAX_BS_DECODE}"
     --max-running-requests "${MAX_RUNNING_REQUESTS}"
+    --watchdog-timeout "${PROFILE_WATCHDOG_TIMEOUT_S}"
   )
   if [[ -n "${SGLANG_EXTRA_ARGS}" ]]; then
     local -a extra_args=()
@@ -364,6 +365,11 @@ profile_done_count() {
     | grep -c 'Profiling done' || true
 }
 
+profile_started_count() {
+  run_on_node "${HEAD_NODE}" "docker logs '${HEAD_CONTAINER}' 2>&1" \
+    | grep -c 'Profiling starts' || true
+}
+
 start_profile() {
   local case_id="$1"
   local activities_json="$2"
@@ -400,6 +406,23 @@ wait_for_profile_stop() {
   return 1
 }
 
+wait_for_profile_start() {
+  local before="$1"
+  local profile_request_pid="$2"
+  local expected=$(( TP_SIZE / NNODES ))
+  local waited=0
+  while (( waited < PROFILE_STOP_WAIT_S )); do
+    if (( $(profile_started_count) >= before + expected )); then
+      return 0
+    fi
+    kill -0 "${profile_request_pid}" 2>/dev/null || return 1
+    sleep 1
+    ((waited+=1))
+  done
+  log "ERROR: profiler did not start within ${PROFILE_STOP_WAIT_S}s"
+  return 1
+}
+
 server_decode_count() {
   run_on_node "${HEAD_NODE}" "docker logs '${HEAD_CONTAINER}' 2>&1" \
     | grep -c 'Decode batch' || true
@@ -487,6 +510,51 @@ run_bench() {
     "${BENCH_CMD[@]}" > "${case_path}/bench.log" 2>&1
 }
 
+prepare_native_injection_payload() {
+  local case_path="$1"
+  local payload_file="${case_path}/request.json"
+  mkdir -p "${case_path}"
+  python3 -c 'import json, sys
+path, length, token_id, output_len = sys.argv[1:]
+payload = {
+    "input_ids": [int(token_id)] * int(length),
+    "sampling_params": {
+        "max_new_tokens": int(output_len),
+        "temperature": 0,
+        "ignore_eos": True,
+    },
+    "stream": False,
+}
+with open(path, "w", encoding="utf-8") as f:
+    json.dump(payload, f, separators=(",", ":"))
+' "${payload_file}" "${MIXED_INJECTION_ISL}" \
+    "${MIXED_INJECTION_TOKEN_ID}" "${MIXED_INJECTION_OSL}"
+}
+
+run_native_injection() {
+  local case_id="$1"
+  local case_path="${CASE_DIR}/${case_id}"
+  local payload_file="${case_path}/request.json"
+  local response_file="${case_path}/response.json"
+  prepare_native_injection_payload "${case_path}"
+  print_command timeout --signal=TERM --kill-after=30s \
+    "${PROFILE_TIMEOUT_S}s" curl --fail --silent --show-error \
+    -X POST "http://${HEAD_IP}:${SGLANG_PORT}/generate" \
+    -H 'Content-Type: application/json' --data-binary "@${payload_file}" \
+    -o "${response_file}" > "${case_path}/curl_cmd.txt"
+  if [[ "${DRY_RUN}" == "1" ]]; then
+    printf '[DRY] '
+    cat "${case_path}/curl_cmd.txt"
+    return 0
+  fi
+  timeout --signal=TERM --kill-after=30s "${PROFILE_TIMEOUT_S}s" \
+    curl --fail --silent --show-error \
+    -X POST "http://${HEAD_IP}:${SGLANG_PORT}/generate" \
+    -H 'Content-Type: application/json' --data-binary "@${payload_file}" \
+    -o "${response_file}"
+  python3 -m json.tool "${response_file}" >/dev/null
+}
+
 run_profiled_case() {
   local case_id="$1"
   local activities_json="$2"
@@ -568,6 +636,8 @@ run_mixed_profile() {
   local before=0
   local decode_before=0
   local mixed_baseline_before=0
+  local profile_started_before=0
+  local profile_request_pid=0
   flush_cache
   mkdir -p "${background_path}"
   prepare_bench_command "${background_path}/bench.jsonl" \
@@ -575,12 +645,11 @@ run_mixed_profile() {
   print_command timeout --signal=TERM --kill-after=30s \
     "${PROFILE_TIMEOUT_S}s" "${BENCH_CMD[@]}" > "${background_path}/bench_cmd.txt"
   if [[ "${DRY_RUN}" == "1" ]]; then
-    log "[DRY] start decode background, wait for Decode batch, start profile, inject 128K prefill"
+    log "[DRY] start decode background, wait for Decode batch, start profile asynchronously, inject native 128K prefill"
     start_profile "${case_id}" '["CUDA_PROFILER"]' \
       "${NSYS_MIXED_START_STEP}" "${NSYS_MIXED_STEPS}"
     log "[DRY] wait ${MIXED_PRE_INJECTION_DECODE_STEPS} Decode steps before injection"
-    run_bench "${injection_id}" "${MIXED_INJECTION_ISL}" \
+    run_native_injection "${injection_id}"
-      "${MIXED_INJECTION_OSL}" 1 1 5202
     return 0
   fi
   decode_before="$(server_decode_count)"
@@ -595,8 +664,16 @@ run_mixed_profile() {
     return 1
   }
   before="$(profile_done_count)"
+  profile_started_before="$(profile_started_count)"
   start_profile "${case_id}" '["CUDA_PROFILER"]' \
-    "${NSYS_MIXED_START_STEP}" "${NSYS_MIXED_STEPS}"
+    "${NSYS_MIXED_START_STEP}" "${NSYS_MIXED_STEPS}" &
+  profile_request_pid=$!
+  wait_for_profile_start "${profile_started_before}" "${profile_request_pid}" || {
+    kill "${profile_request_pid}" "${background_pid}" 2>/dev/null || true
+    wait "${profile_request_pid}" "${background_pid}" 2>/dev/null || true
+    log "ERROR: mixed profile request did not become active"
+    return 1
+  }
   mixed_baseline_before="$(server_decode_count)"
   wait_for_decode_steps "${mixed_baseline_before}" \
     "${MIXED_PRE_INJECTION_DECODE_STEPS}" "${background_pid}" || {
@@ -605,8 +682,8 @@ run_mixed_profile() {
     log "ERROR: mixed profile did not capture the pre-injection Decode baseline"
     return 1
   }
-  run_bench "${injection_id}" "${MIXED_INJECTION_ISL}" \
+  run_native_injection "${injection_id}"
-    "${MIXED_INJECTION_OSL}" 1 1 5202
+  wait "${profile_request_pid}"
   wait_for_profile_stop "${before}"
   wait "${background_pid}"
 }
@@ -668,6 +745,16 @@ cleanup() {
   fi
 }
 
+on_exit() {
+  local exit_code=$?
+  trap - EXIT
+  if (( exit_code != 0 )) && [[ -d "${RESULT_DIR}" ]]; then
+    write_manifest FAILED || true
+  fi
+  cleanup
+  exit "${exit_code}"
+}
+
 run_torch_smoke() {
   PROFILE_MODE="torch"
   CAPTURE_RANGES=0
@@ -721,7 +808,8 @@ run_all() {
   write_manifest COMPLETED
 }
 
-trap cleanup EXIT INT TERM
+trap on_exit EXIT
+trap 'exit 130' INT TERM
 
 case "${ACTION}" in
   audit)