57 lines
2.6 KiB
Bash
57 lines
2.6 KiB
Bash
# Phase 3: short distributed timelines for the verified TP16 baseline.
|
|
|
|
EXPERIMENT="dsv4pro_pro6000d_2node_sglang_timeline_profiling"
|
|
|
|
# Reuse model, topology, Docker image, serving knobs and benchmark paths from
|
|
# the completed Phase 1 baseline. This file only defines profiling policy.
|
|
PHASE1_DIR="${PHASE1_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/../dsv4pro_pro6000d_2node_sglang_tp16_quick_map" && pwd)}"
|
|
|
|
PROFILE_PORT="${PROFILE_PORT:-30002}"
|
|
PROFILE_DIST_INIT_PORT="${PROFILE_DIST_INIT_PORT:-20002}"
|
|
PROFILE_ROOT_IN_CONTAINER="${PROFILE_ROOT_IN_CONTAINER:-/profiles}"
|
|
WORKER_STAGE_ROOT="${WORKER_STAGE_ROOT:-/data/hzy/dsv4pro_phase3_worker}"
|
|
|
|
# The primary run keeps CUDA Graph enabled. Layer-wise NVTX with disabled
|
|
# graphs is deliberately deferred until the production-faithful trace proves
|
|
# that kernel names are insufficient.
|
|
NSYS_TRACE="${NSYS_TRACE:-cuda,nvtx,nccl,osrt}"
|
|
NSYS_CUDA_GRAPH_TRACE="${NSYS_CUDA_GRAPH_TRACE:-node}"
|
|
NSYS_EXPORT_TIMEOUT_S="${NSYS_EXPORT_TIMEOUT_S:-300}"
|
|
NSYS_CONTROL_STEPS="${NSYS_CONTROL_STEPS:-16}"
|
|
NSYS_MIXED_STEPS="${NSYS_MIXED_STEPS:-32}"
|
|
NSYS_PREFILL_STEPS="${NSYS_PREFILL_STEPS:-16}"
|
|
NSYS_SMOKE_STEPS="${NSYS_SMOKE_STEPS:-5}"
|
|
TORCH_SMOKE_STEPS="${TORCH_SMOKE_STEPS:-3}"
|
|
TORCH_SMOKE_START_STEP="${TORCH_SMOKE_START_STEP:-1}"
|
|
NSYS_SMOKE_START_STEP="${NSYS_SMOKE_START_STEP:-1}"
|
|
NSYS_CONTROL_START_STEP="${NSYS_CONTROL_START_STEP:-2}"
|
|
NSYS_MIXED_START_STEP="${NSYS_MIXED_START_STEP:-0}"
|
|
NSYS_PREFILL_START_STEP="${NSYS_PREFILL_START_STEP:-0}"
|
|
MIXED_PRE_INJECTION_DECODE_STEPS="${MIXED_PRE_INJECTION_DECODE_STEPS:-2}"
|
|
|
|
# Short profiling workloads. They reproduce the Phase 2 scheduler shapes but
|
|
# avoid its full result matrix and long hardware sampling.
|
|
CONTROL_ISL="${CONTROL_ISL:-1024}"
|
|
CONTROL_OSL="${CONTROL_OSL:-1024}"
|
|
CONTROL_CONCURRENCY="${CONTROL_CONCURRENCY:-32}"
|
|
CONTROL_PROMPTS="${CONTROL_PROMPTS:-32}"
|
|
|
|
MIXED_ISL="${MIXED_ISL:-1024}"
|
|
MIXED_OSL="${MIXED_OSL:-1024}"
|
|
MIXED_CONCURRENCY="${MIXED_CONCURRENCY:-32}"
|
|
MIXED_PROMPTS="${MIXED_PROMPTS:-64}"
|
|
MIXED_INJECTION_ISL="${MIXED_INJECTION_ISL:-131072}"
|
|
MIXED_INJECTION_OSL="${MIXED_INJECTION_OSL:-1}"
|
|
|
|
PREFILL_ISL="${PREFILL_ISL:-131072}"
|
|
PREFILL_OSL="${PREFILL_OSL:-1}"
|
|
PROFILE_TIMEOUT_S="${PROFILE_TIMEOUT_S:-1800}"
|
|
PROFILE_STOP_WAIT_S="${PROFILE_STOP_WAIT_S:-180}"
|
|
HEALTH_CHECK_RETRIES="${HEALTH_CHECK_RETRIES:-600}"
|
|
HEALTH_CHECK_INTERVAL_S="${HEALTH_CHECK_INTERVAL_S:-5}"
|
|
|
|
RESULT_BASE="${RESULT_BASE:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/results}"
|
|
RUNTIME_BASE="${RUNTIME_BASE:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/runtime}"
|
|
DRY_RUN="${DRY_RUN:-0}"
|
|
ALLOW_BUSY_GPU="${ALLOW_BUSY_GPU:-0}"
|