diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index 1a4b3bc..5e4ce15 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -163,5 +163,8 @@ export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUE export SEARCH_START_CONCURRENCY=16 export SEARCH_ADDEND=16 +# If the initial concurrency violates the TTFT SLO, search downward. Stop at +# the first acceptable value (16 -> 8; only try 1 when 8 still violates it). +export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1" adaptive_main "$@" diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh index 6e8a180..fd5444e 100755 --- a/scripts/common/adaptive_bench_lib.sh +++ b/scripts/common/adaptive_bench_lib.sh @@ -77,6 +77,19 @@ adaptive_validate_config() { log "ERROR: ENABLE_TTFT_SLO_STOP must be 0 or 1, got ${ENABLE_TTFT_SLO_STOP}" return 1 fi + local previous_backoff="$SEARCH_START_CONCURRENCY" + local backoff + for backoff in ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}; do + if ! [[ "$backoff" =~ ^[0-9]+$ ]] || (( backoff < 1 )); then + log "ERROR: SEARCH_INITIAL_BACKOFF_CONCURRENCIES must contain positive integers, got ${backoff}" + return 1 + fi + if (( backoff >= previous_backoff )); then + log "ERROR: SEARCH_INITIAL_BACKOFF_CONCURRENCIES must be strictly descending below SEARCH_START_CONCURRENCY, got ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES}" + return 1 + fi + previous_backoff="$backoff" + done } adaptive_start_gpu_monitor() { @@ -374,6 +387,7 @@ adaptive_run_shape() { local gain_pct="" local meaningful_gain=1 local point_rc=0 + local initial_backoff_active=0 ADAPTIVE_RESTART_NEEDED=0 ADAPTIVE_FATAL_WORKLOAD=0 @@ -423,6 +437,25 @@ adaptive_run_shape() { "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ "$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + local next_backoff="" + if (( concurrency == SEARCH_START_CONCURRENCY || initial_backoff_active == 1 )); then + local backoff + for backoff in ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}; do + if (( backoff < concurrency )); then + next_backoff="$backoff" + break + fi + done + fi + if [[ -n "$next_backoff" ]]; then + log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms; retrying initial boundary at c=${next_backoff}" + initial_backoff_active=1 + concurrency="$next_backoff" + previous_tps="" + plateau_streak=0 + plateau_start_concurrency="" + continue + fi stop_reason="TTFT_SLO_EXCEEDED" shape_status="TTFT_BOUNDARY" log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape" @@ -436,6 +469,12 @@ adaptive_run_shape() { "$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" log "probe result c=${concurrency} total_tps=${current_tps} gain_pct=${gain_pct:-first} plateau_streak=${plateau_streak}/${PLATEAU_PATIENCE}" + if (( initial_backoff_active == 1 )); then + stop_reason="TTFT_SLO_BACKOFF_FOUND" + shape_status="TTFT_BOUNDARY" + log "initial TTFT backoff found acceptable concurrency c=${concurrency}; stopping shape without probing lower values" + break + fi if (( plateau_streak >= PLATEAU_PATIENCE )); then saturation_concurrency="$plateau_start_concurrency" stop_reason="TPS_PLATEAU" @@ -644,7 +683,7 @@ adaptive_main() { fi log "experiment=${EXPERIMENT_NAME} engine=${ENGINE} run_id=${RUN_ID}" - log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}" + log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} initial_backoff=[${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-none}] min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}" log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}" "$PYTHON" - \ @@ -660,6 +699,7 @@ adaptive_main() { "$SEARCH_START_CONCURRENCY" \ "$SEARCH_MAX_CONCURRENCY" \ "$SEARCH_MULTIPLIER" \ + "${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}" \ "$NUM_PROMPTS_MULTIPLIER" \ "$TPS_MIN_GAIN_PCT" \ "$PLATEAU_PATIENCE" \ @@ -671,8 +711,9 @@ import json, sys (experiment, engine, run_id, model, hardware, matrix, dataset, tokenize_prompt, random_range_ratio, search_start, search_cap, - search_multiplier, prompts_multiplier, min_gain_pct, plateau_patience, - warmup_max_requests, ttft_slo_ms, enable_ttft_slo_stop) = sys.argv[1:19] + search_multiplier, initial_backoff_concurrencies, prompts_multiplier, + min_gain_pct, plateau_patience, warmup_max_requests, ttft_slo_ms, + enable_ttft_slo_stop) = sys.argv[1:20] print(json.dumps({ "experiment": experiment, @@ -688,6 +729,9 @@ print(json.dumps({ "start_concurrency": int(search_start), "max_concurrency": int(search_cap), "multiplier": int(search_multiplier), + "initial_backoff_concurrencies": [ + int(value) for value in initial_backoff_concurrencies.split() + ], "num_prompts_multiplier": int(prompts_multiplier), "min_tps_gain_pct": float(min_gain_pct), "plateau_patience": int(plateau_patience),