# Phase 3: short distributed timelines for the verified TP16 baseline. EXPERIMENT="dsv4pro_pro6000d_2node_sglang_timeline_profiling" # Reuse model, topology, Docker image, serving knobs and benchmark paths from # the completed Phase 1 baseline. This file only defines profiling policy. PHASE1_DIR="${PHASE1_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../dsv4pro_pro6000d_2node_sglang_tp16_quick_map" && pwd)}" PROFILE_PORT="${PROFILE_PORT:-30002}" PROFILE_DIST_INIT_PORT="${PROFILE_DIST_INIT_PORT:-20002}" PROFILE_ROOT_IN_CONTAINER="${PROFILE_ROOT_IN_CONTAINER:-/profiles}" WORKER_STAGE_ROOT="${WORKER_STAGE_ROOT:-/data/hzy/dsv4pro_phase3_worker}" # The primary run keeps CUDA Graph enabled. Layer-wise NVTX with disabled # graphs is deliberately deferred until the production-faithful trace proves # that kernel names are insufficient. NSYS_TRACE="${NSYS_TRACE:-cuda,nvtx,nccl,osrt}" NSYS_CUDA_GRAPH_TRACE="${NSYS_CUDA_GRAPH_TRACE:-node}" NSYS_EXPORT_TIMEOUT_S="${NSYS_EXPORT_TIMEOUT_S:-300}" NSYS_CONTROL_STEPS="${NSYS_CONTROL_STEPS:-16}" NSYS_MIXED_STEPS="${NSYS_MIXED_STEPS:-32}" NSYS_PREFILL_STEPS="${NSYS_PREFILL_STEPS:-16}" NSYS_SMOKE_STEPS="${NSYS_SMOKE_STEPS:-5}" TORCH_SMOKE_STEPS="${TORCH_SMOKE_STEPS:-3}" # Short profiling workloads. They reproduce the Phase 2 scheduler shapes but # avoid its full result matrix and long hardware sampling. CONTROL_ISL="${CONTROL_ISL:-1024}" CONTROL_OSL="${CONTROL_OSL:-256}" CONTROL_CONCURRENCY="${CONTROL_CONCURRENCY:-32}" CONTROL_PROMPTS="${CONTROL_PROMPTS:-32}" MIXED_ISL="${MIXED_ISL:-1024}" MIXED_OSL="${MIXED_OSL:-512}" MIXED_CONCURRENCY="${MIXED_CONCURRENCY:-32}" MIXED_PROMPTS="${MIXED_PROMPTS:-64}" MIXED_INJECTION_ISL="${MIXED_INJECTION_ISL:-131072}" MIXED_INJECTION_OSL="${MIXED_INJECTION_OSL:-1}" PREFILL_ISL="${PREFILL_ISL:-131072}" PREFILL_OSL="${PREFILL_OSL:-1}" PROFILE_TIMEOUT_S="${PROFILE_TIMEOUT_S:-1800}" PROFILE_STOP_WAIT_S="${PROFILE_STOP_WAIT_S:-180}" HEALTH_CHECK_RETRIES="${HEALTH_CHECK_RETRIES:-600}" HEALTH_CHECK_INTERVAL_S="${HEALTH_CHECK_INTERVAL_S:-5}" RESULT_BASE="${RESULT_BASE:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/results}" RUNTIME_BASE="${RUNTIME_BASE:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/runtime}" DRY_RUN="${DRY_RUN:-0}" ALLOW_BUSY_GPU="${ALLOW_BUSY_GPU:-0}"