[BugFix] align mixed Phase 3 capture with prefill injection
This commit is contained in:
Zhiyi Hong
parent
4628d49755
commit
bc491eeeed
@ -1,5 +1,9 @@
|
||||
# sskj — 多平台大模型推理性能基准测试项目
|
||||
|
||||
> **更新（2026-08-02 00:08:00 CST）**
|
||||
>
|
||||
> 修复 DeepSeek-V4-Pro 双机 Pro6000D SGLang Phase 3 混合 Capture 时序。首轮正式 Run 证明当前 SGLang `/start_profile` 请求会阻塞到捕获结束，旧实现因此在 Profile 完成后才发送 128K Prefill，第二份 Trace 未覆盖注入；随后混合背景触发默认 300 秒 Scheduler watchdog，第三段未执行。新实现异步触发 Profile，先捕获 2 个 Decode Step，再通过原生 `/generate` + `input_ids` 立即注入 128K 请求，避免 benchmark 客户端启动延迟；混合背景缩为单波 32 请求，Profiling 专用 watchdog 提升至 1800 秒，并让异常退出自动将 manifest 标记为 `FAILED`。
|
||||
>
|
||||
> **更新（2026-08-01 19:34:34 CST）**
|
||||
>
|
||||
> 完成 DeepSeek-V4-Pro 双机 Pro6000D SGLang Phase 3 正式时间线代码交付。正式 Control 与混合背景统一为 `1K -> 1K, C=32`；所有 `/start_profile` 请求显式记录 `start_step` 和 `num_steps`，Control 在活跃 Decode 后跳过 2 Step，混合场景先保留 2 个纯 Decode Step 再注入 `128K -> 1`，长 Prefill 从首个 Chunk 开始捕获。继续沿用已通过的双节点 PyTorch/Nsight smoke、16 Rank/6 份 Nsight 报告 fail-closed 门禁和单服务三段 Capture Range；正式结果尚未生成，因此不创建 `phase3_exp.html` 或 `phase3_code.html`。
|
||||
|
||||
@ -31,8 +31,10 @@ staging paths. No `source`, Conda activation, or command on Worker is required.
|
||||
1. `decode_control`: `1K -> 1K`, `C=32`; trigger after Decode becomes active,
|
||||
skip 2 engine steps, then capture 16 steps.
|
||||
2. `mixed_decode_with_128k_prefill`: active `1K -> 1K`, `C=32` decode;
|
||||
capture 2 pure Decode steps before injecting one `128K -> 1` request, then
|
||||
retain a 32-step range spanning both sides of the injection.
|
||||
start `/start_profile` asynchronously, capture 2 pure Decode steps, then
|
||||
inject one native `input_ids` based `128K -> 1` request without benchmark
|
||||
client startup delay. The 32-step range therefore spans both sides of the
|
||||
injection.
|
||||
3. `long_prefill`: isolated `128K -> 1`, `C=1`.
|
||||
|
||||
The service remains under one Nsight process for all three ranges, so it uses
|
||||
|
||||
@ -17,6 +17,7 @@ WORKER_STAGE_ROOT="${WORKER_STAGE_ROOT:-/data/hzy/dsv4pro_phase3_worker}"
|
||||
NSYS_TRACE="${NSYS_TRACE:-cuda,nvtx,nccl,osrt}"
|
||||
NSYS_CUDA_GRAPH_TRACE="${NSYS_CUDA_GRAPH_TRACE:-node}"
|
||||
NSYS_EXPORT_TIMEOUT_S="${NSYS_EXPORT_TIMEOUT_S:-300}"
|
||||
PROFILE_WATCHDOG_TIMEOUT_S="${PROFILE_WATCHDOG_TIMEOUT_S:-1800}"
|
||||
NSYS_CONTROL_STEPS="${NSYS_CONTROL_STEPS:-16}"
|
||||
NSYS_MIXED_STEPS="${NSYS_MIXED_STEPS:-32}"
|
||||
NSYS_PREFILL_STEPS="${NSYS_PREFILL_STEPS:-16}"
|
||||
@ -39,9 +40,10 @@ CONTROL_PROMPTS="${CONTROL_PROMPTS:-32}"
|
||||
MIXED_ISL="${MIXED_ISL:-1024}"
|
||||
MIXED_OSL="${MIXED_OSL:-1024}"
|
||||
MIXED_CONCURRENCY="${MIXED_CONCURRENCY:-32}"
|
||||
MIXED_PROMPTS="${MIXED_PROMPTS:-64}"
|
||||
MIXED_PROMPTS="${MIXED_PROMPTS:-32}"
|
||||
MIXED_INJECTION_ISL="${MIXED_INJECTION_ISL:-131072}"
|
||||
MIXED_INJECTION_OSL="${MIXED_INJECTION_OSL:-1}"
|
||||
MIXED_INJECTION_TOKEN_ID="${MIXED_INJECTION_TOKEN_ID:-100}"
|
||||
|
||||
PREFILL_ISL="${PREFILL_ISL:-131072}"
|
||||
PREFILL_OSL="${PREFILL_OSL:-1}"
|
||||
|
||||
@ -209,6 +209,7 @@ build_server_command() {
|
||||
--mem-fraction-static "${MEM_FRACTION_STATIC}"
|
||||
--cuda-graph-max-bs-decode "${CUDA_GRAPH_MAX_BS_DECODE}"
|
||||
--max-running-requests "${MAX_RUNNING_REQUESTS}"
|
||||
--watchdog-timeout "${PROFILE_WATCHDOG_TIMEOUT_S}"
|
||||
)
|
||||
if [[ -n "${SGLANG_EXTRA_ARGS}" ]]; then
|
||||
local -a extra_args=()
|
||||
@ -364,6 +365,11 @@ profile_done_count() {
|
||||
| grep -c 'Profiling done' || true
|
||||
}
|
||||
|
||||
profile_started_count() {
|
||||
run_on_node "${HEAD_NODE}" "docker logs '${HEAD_CONTAINER}' 2>&1" \
|
||||
| grep -c 'Profiling starts' || true
|
||||
}
|
||||
|
||||
start_profile() {
|
||||
local case_id="$1"
|
||||
local activities_json="$2"
|
||||
@ -400,6 +406,23 @@ wait_for_profile_stop() {
|
||||
return 1
|
||||
}
|
||||
|
||||
wait_for_profile_start() {
|
||||
local before="$1"
|
||||
local profile_request_pid="$2"
|
||||
local expected=$(( TP_SIZE / NNODES ))
|
||||
local waited=0
|
||||
while (( waited < PROFILE_STOP_WAIT_S )); do
|
||||
if (( $(profile_started_count) >= before + expected )); then
|
||||
return 0
|
||||
fi
|
||||
kill -0 "${profile_request_pid}" 2>/dev/null || return 1
|
||||
sleep 1
|
||||
((waited+=1))
|
||||
done
|
||||
log "ERROR: profiler did not start within ${PROFILE_STOP_WAIT_S}s"
|
||||
return 1
|
||||
}
|
||||
|
||||
server_decode_count() {
|
||||
run_on_node "${HEAD_NODE}" "docker logs '${HEAD_CONTAINER}' 2>&1" \
|
||||
| grep -c 'Decode batch' || true
|
||||
@ -487,6 +510,51 @@ run_bench() {
|
||||
"${BENCH_CMD[@]}" > "${case_path}/bench.log" 2>&1
|
||||
}
|
||||
|
||||
prepare_native_injection_payload() {
|
||||
local case_path="$1"
|
||||
local payload_file="${case_path}/request.json"
|
||||
mkdir -p "${case_path}"
|
||||
python3 -c 'import json, sys
|
||||
path, length, token_id, output_len = sys.argv[1:]
|
||||
payload = {
|
||||
"input_ids": [int(token_id)] * int(length),
|
||||
"sampling_params": {
|
||||
"max_new_tokens": int(output_len),
|
||||
"temperature": 0,
|
||||
"ignore_eos": True,
|
||||
},
|
||||
"stream": False,
|
||||
}
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, separators=(",", ":"))
|
||||
' "${payload_file}" "${MIXED_INJECTION_ISL}" \
|
||||
"${MIXED_INJECTION_TOKEN_ID}" "${MIXED_INJECTION_OSL}"
|
||||
}
|
||||
|
||||
run_native_injection() {
|
||||
local case_id="$1"
|
||||
local case_path="${CASE_DIR}/${case_id}"
|
||||
local payload_file="${case_path}/request.json"
|
||||
local response_file="${case_path}/response.json"
|
||||
prepare_native_injection_payload "${case_path}"
|
||||
print_command timeout --signal=TERM --kill-after=30s \
|
||||
"${PROFILE_TIMEOUT_S}s" curl --fail --silent --show-error \
|
||||
-X POST "http://${HEAD_IP}:${SGLANG_PORT}/generate" \
|
||||
-H 'Content-Type: application/json' --data-binary "@${payload_file}" \
|
||||
-o "${response_file}" > "${case_path}/curl_cmd.txt"
|
||||
if [[ "${DRY_RUN}" == "1" ]]; then
|
||||
printf '[DRY] '
|
||||
cat "${case_path}/curl_cmd.txt"
|
||||
return 0
|
||||
fi
|
||||
timeout --signal=TERM --kill-after=30s "${PROFILE_TIMEOUT_S}s" \
|
||||
curl --fail --silent --show-error \
|
||||
-X POST "http://${HEAD_IP}:${SGLANG_PORT}/generate" \
|
||||
-H 'Content-Type: application/json' --data-binary "@${payload_file}" \
|
||||
-o "${response_file}"
|
||||
python3 -m json.tool "${response_file}" >/dev/null
|
||||
}
|
||||
|
||||
run_profiled_case() {
|
||||
local case_id="$1"
|
||||
local activities_json="$2"
|
||||
@ -568,6 +636,8 @@ run_mixed_profile() {
|
||||
local before=0
|
||||
local decode_before=0
|
||||
local mixed_baseline_before=0
|
||||
local profile_started_before=0
|
||||
local profile_request_pid=0
|
||||
flush_cache
|
||||
mkdir -p "${background_path}"
|
||||
prepare_bench_command "${background_path}/bench.jsonl" \
|
||||
@ -575,12 +645,11 @@ run_mixed_profile() {
|
||||
print_command timeout --signal=TERM --kill-after=30s \
|
||||
"${PROFILE_TIMEOUT_S}s" "${BENCH_CMD[@]}" > "${background_path}/bench_cmd.txt"
|
||||
if [[ "${DRY_RUN}" == "1" ]]; then
|
||||
log "[DRY] start decode background, wait for Decode batch, start profile, inject 128K prefill"
|
||||
log "[DRY] start decode background, wait for Decode batch, start profile asynchronously, inject native 128K prefill"
|
||||
start_profile "${case_id}" '["CUDA_PROFILER"]' \
|
||||
"${NSYS_MIXED_START_STEP}" "${NSYS_MIXED_STEPS}"
|
||||
log "[DRY] wait ${MIXED_PRE_INJECTION_DECODE_STEPS} Decode steps before injection"
|
||||
run_bench "${injection_id}" "${MIXED_INJECTION_ISL}" \
|
||||
"${MIXED_INJECTION_OSL}" 1 1 5202
|
||||
run_native_injection "${injection_id}"
|
||||
return 0
|
||||
fi
|
||||
decode_before="$(server_decode_count)"
|
||||
@ -595,8 +664,16 @@ run_mixed_profile() {
|
||||
return 1
|
||||
}
|
||||
before="$(profile_done_count)"
|
||||
profile_started_before="$(profile_started_count)"
|
||||
start_profile "${case_id}" '["CUDA_PROFILER"]' \
|
||||
"${NSYS_MIXED_START_STEP}" "${NSYS_MIXED_STEPS}"
|
||||
"${NSYS_MIXED_START_STEP}" "${NSYS_MIXED_STEPS}" &
|
||||
profile_request_pid=$!
|
||||
wait_for_profile_start "${profile_started_before}" "${profile_request_pid}" || {
|
||||
kill "${profile_request_pid}" "${background_pid}" 2>/dev/null || true
|
||||
wait "${profile_request_pid}" "${background_pid}" 2>/dev/null || true
|
||||
log "ERROR: mixed profile request did not become active"
|
||||
return 1
|
||||
}
|
||||
mixed_baseline_before="$(server_decode_count)"
|
||||
wait_for_decode_steps "${mixed_baseline_before}" \
|
||||
"${MIXED_PRE_INJECTION_DECODE_STEPS}" "${background_pid}" || {
|
||||
@ -605,8 +682,8 @@ run_mixed_profile() {
|
||||
log "ERROR: mixed profile did not capture the pre-injection Decode baseline"
|
||||
return 1
|
||||
}
|
||||
run_bench "${injection_id}" "${MIXED_INJECTION_ISL}" \
|
||||
"${MIXED_INJECTION_OSL}" 1 1 5202
|
||||
run_native_injection "${injection_id}"
|
||||
wait "${profile_request_pid}"
|
||||
wait_for_profile_stop "${before}"
|
||||
wait "${background_pid}"
|
||||
}
|
||||
@ -668,6 +745,16 @@ cleanup() {
|
||||
fi
|
||||
}
|
||||
|
||||
on_exit() {
|
||||
local exit_code=$?
|
||||
trap - EXIT
|
||||
if (( exit_code != 0 )) && [[ -d "${RESULT_DIR}" ]]; then
|
||||
write_manifest FAILED || true
|
||||
fi
|
||||
cleanup
|
||||
exit "${exit_code}"
|
||||
}
|
||||
|
||||
run_torch_smoke() {
|
||||
PROFILE_MODE="torch"
|
||||
CAPTURE_RANGES=0
|
||||
@ -721,7 +808,8 @@ run_all() {
|
||||
write_manifest COMPLETED
|
||||
}
|
||||
|
||||
trap cleanup EXIT INT TERM
|
||||
trap on_exit EXIT
|
||||
trap 'exit 130' INT TERM
|
||||
|
||||
case "${ACTION}" in
|
||||
audit)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user