qqtang/sskj
1
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update SEARCH_INITIAL_BACKOFF_CONCURRENCIES

Browse Source
This commit is contained in:
SSKJ Dev 2026-07-16 08:52:29 +00:00
parent 58ed6ea633
commit c2a3024a1b
2 changed files with 50 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
View File

@ -163,5 +163,8 @@ export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUE
export SEARCH_START_CONCURRENCY=16
export SEARCH_ADDEND=16
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
adaptive_main "$@"

50
scripts/common/adaptive_bench_lib.sh
View File

@ -77,6 +77,19 @@ adaptive_validate_config() {
log "ERROR: ENABLE_TTFT_SLO_STOP must be 0 or 1, got ${ENABLE_TTFT_SLO_STOP}"
return 1
fi
local previous_backoff="$SEARCH_START_CONCURRENCY"
local backoff
for backoff in ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}; do
if ! [[ "$backoff" =~ ^[0-9]+$ ]] || (( backoff < 1 )); then
log "ERROR: SEARCH_INITIAL_BACKOFF_CONCURRENCIES must contain positive integers, got ${backoff}"
return 1
fi
if (( backoff >= previous_backoff )); then
log "ERROR: SEARCH_INITIAL_BACKOFF_CONCURRENCIES must be strictly descending below SEARCH_START_CONCURRENCY, got ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES}"
return 1
fi
previous_backoff="$backoff"
done
}
adaptive_start_gpu_monitor() {
@ -374,6 +387,7 @@ adaptive_run_shape() {
local gain_pct=""
local meaningful_gain=1
local point_rc=0
local initial_backoff_active=0
ADAPTIVE_RESTART_NEEDED=0
ADAPTIVE_FATAL_WORKLOAD=0
@ -423,6 +437,25 @@ adaptive_run_shape() {
"$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \
"$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \
"$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG"
local next_backoff=""
if (( concurrency == SEARCH_START_CONCURRENCY || initial_backoff_active == 1 )); then
local backoff
for backoff in ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}; do
if (( backoff < concurrency )); then
next_backoff="$backoff"
break
fi
done
fi
if [[ -n "$next_backoff" ]]; then
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms; retrying initial boundary at c=${next_backoff}"
initial_backoff_active=1
concurrency="$next_backoff"
previous_tps=""
plateau_streak=0
plateau_start_concurrency=""
continue
fi
stop_reason="TTFT_SLO_EXCEEDED"
shape_status="TTFT_BOUNDARY"
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape"
@ -436,6 +469,12 @@ adaptive_run_shape() {
"$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG"
log "probe result c=${concurrency} total_tps=${current_tps} gain_pct=${gain_pct:-first} plateau_streak=${plateau_streak}/${PLATEAU_PATIENCE}"
if (( initial_backoff_active == 1 )); then
stop_reason="TTFT_SLO_BACKOFF_FOUND"
shape_status="TTFT_BOUNDARY"
log "initial TTFT backoff found acceptable concurrency c=${concurrency}; stopping shape without probing lower values"
break
fi
if (( plateau_streak >= PLATEAU_PATIENCE )); then
saturation_concurrency="$plateau_start_concurrency"
stop_reason="TPS_PLATEAU"
@ -644,7 +683,7 @@ adaptive_main() {
fi
log "experiment=${EXPERIMENT_NAME} engine=${ENGINE} run_id=${RUN_ID}"
log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}"
log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} initial_backoff=[${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-none}] min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}"
log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}"
"$PYTHON" - \
@ -660,6 +699,7 @@ adaptive_main() {
"$SEARCH_START_CONCURRENCY" \
"$SEARCH_MAX_CONCURRENCY" \
"$SEARCH_MULTIPLIER" \
"${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}" \
"$NUM_PROMPTS_MULTIPLIER" \
"$TPS_MIN_GAIN_PCT" \
"$PLATEAU_PATIENCE" \
@ -671,8 +711,9 @@ import json, sys
(experiment, engine, run_id, model, hardware, matrix, dataset,
tokenize_prompt, random_range_ratio, search_start, search_cap,
search_multiplier, prompts_multiplier, min_gain_pct, plateau_patience,
warmup_max_requests, ttft_slo_ms, enable_ttft_slo_stop) = sys.argv[1:19]
search_multiplier, initial_backoff_concurrencies, prompts_multiplier,
min_gain_pct, plateau_patience, warmup_max_requests, ttft_slo_ms,
enable_ttft_slo_stop) = sys.argv[1:20]
print(json.dumps({
"experiment": experiment,
@ -688,6 +729,9 @@ print(json.dumps({
"start_concurrency": int(search_start),
"max_concurrency": int(search_cap),
"multiplier": int(search_multiplier),
"initial_backoff_concurrencies": [
int(value) for value in initial_backoff_concurrencies.split()
],
"num_prompts_multiplier": int(prompts_multiplier),
"min_tps_gain_pct": float(min_gain_pct),
"plateau_patience": int(plateau_patience),