From dcb8cd21311b53d94f6b1aabd7e31049204e196f Mon Sep 17 00:00:00 2001 From: SSKJ Dev Date: Thu, 16 Jul 2026 05:15:51 +0000 Subject: [PATCH] feat(adaptive): add TTFT SLO early-stop and fix relative paths for h20 experiments - Add TTFT_SLO_MS and ENABLE_TTFT_SLO_STOP config to adaptive search - Stop searching current shape when TTFT P95 exceeds SLO (default 3000ms) - Fix relative paths in h20 scripts: ../../scripts/common -> ../../../scripts/common - Update compare.py default TTFT limit to 4000ms --- .../adaptive_config.env | 6 ++++ .../run_adaptive_concurrency.sh | 6 ++-- .../run_adaptive_concurrency_add16.sh | 6 ++-- .../dsv4_h20_sglang_tp_dp_matrix/run_bench.sh | 6 ++-- .../adaptive_config.env | 6 ++++ .../h20/dsv4_h20_vllm_tp_dp_matrix/compare.py | 4 +-- .../run_adaptive_concurrency.sh | 6 ++-- .../run_adaptive_concurrency_add16.sh | 6 ++-- .../dsv4_h20_vllm_tp_dp_matrix/run_bench.sh | 6 ++-- scripts/common/adaptive_bench_lib.sh | 31 ++++++++++++++++++- 10 files changed, 62 insertions(+), 21 deletions(-) diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env index 1eda1f0..e35c41f 100644 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env @@ -15,6 +15,12 @@ NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" +# TTFT SLO early-stop settings. +# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL) +# shape and move on to the next scenario. +TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" +ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" + # Keep the same random workload semantics as the fixed matrix baseline. # DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times # NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh index 0979a46..7972a9e 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh @@ -6,15 +6,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" # shellcheck source=/dev/null source "${SCRIPT_DIR}/adaptive_config.env" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" ENGINE="sglang" ENGINE_PORT="$SGLANG_PORT" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh index b9b28f3..2f95353 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -6,15 +6,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" # shellcheck source=/dev/null source "${SCRIPT_DIR}/adaptive_config.env" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" ENGINE="sglang" ENGINE_PORT="$SGLANG_PORT" diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh index d27b870..c2d3a37 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh @@ -6,9 +6,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" @@ -522,7 +522,7 @@ run_parallel_config() { # Parse results. log "parsing ${config_label} results" - "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend sglang \ + "$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend sglang \ >> "${phase_log_dir}/parse.log" 2>&1 || { log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" } diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env index 1eda1f0..e35c41f 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env @@ -15,6 +15,12 @@ NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" +# TTFT SLO early-stop settings. +# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL) +# shape and move on to the next scenario. +TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" +ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" + # Keep the same random workload semantics as the fixed matrix baseline. # DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times # NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/compare.py b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/compare.py index 002fa0c..15f277c 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/compare.py +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/compare.py @@ -18,7 +18,7 @@ def load_result(result_root: Path) -> dict: def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, - ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_limit_ms: float = 4000.0, tpot_limit_ms: float = 50.0) -> str: ttft_ok = ttft_p95_ms < ttft_limit_ms tpot_ok = tpot_mean_ms < tpot_limit_ms if ttft_ok and tpot_ok: @@ -42,7 +42,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--run-root", type=Path, required=True) parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) - parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--ttft-limit", type=float, default=4000.0) parser.add_argument("--tpot-limit", type=float, default=50.0) args = parser.parse_args() diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh index ca23f6a..f76a93d 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -6,15 +6,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" # shellcheck source=/dev/null source "${SCRIPT_DIR}/adaptive_config.env" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" ENGINE="vllm" ENGINE_PORT="$VLLM_PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index 482886d..1a4b3bc 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -6,15 +6,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" # shellcheck source=/dev/null source "${SCRIPT_DIR}/adaptive_config.env" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" ENGINE="vllm" ENGINE_PORT="$VLLM_PORT" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh index 7bb82c8..1db82ea 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh @@ -6,9 +6,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/lib.sh" +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" # shellcheck source=/dev/null -source "${SCRIPT_DIR}/../../scripts/common/platform.sh" +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" # shellcheck source=/dev/null source "${SCRIPT_DIR}/config.env" @@ -512,7 +512,7 @@ run_parallel_config() { # Parse results. log "parsing ${config_label} results" - "$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend vllm \ + "$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend vllm \ >> "${phase_log_dir}/parse.log" 2>&1 || { log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" } diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh index 35dc5d2..fcd06f4 100755 --- a/scripts/common/adaptive_bench_lib.sh +++ b/scripts/common/adaptive_bench_lib.sh @@ -69,6 +69,14 @@ adaptive_validate_config() { log "ERROR: RANDOM_RANGE_RATIO must be between 0.0 and 1.0" return 1 fi + if ! [[ "${TTFT_SLO_MS:-4000}" =~ ^[0-9]+$ ]] || (( TTFT_SLO_MS < 1 )); then + log "ERROR: TTFT_SLO_MS must be a positive integer, got ${TTFT_SLO_MS}" + return 1 + fi + if [[ "${ENABLE_TTFT_SLO_STOP:-1}" != "0" && "${ENABLE_TTFT_SLO_STOP:-1}" != "1" ]]; then + log "ERROR: ENABLE_TTFT_SLO_STOP must be 0 or 1, got ${ENABLE_TTFT_SLO_STOP}" + return 1 + fi } adaptive_start_gpu_monitor() { @@ -408,6 +416,23 @@ adaptive_run_shape() { fi max_successful_concurrency="$concurrency" last_total_tps="$current_tps" + + # TTFT SLO check: stop searching if TTFT P95 exceeds the SLO threshold + local ttft_p95_ms="" + if [[ "${ENABLE_TTFT_SLO_STOP:-1}" == "1" ]]; then + ttft_p95_ms="$(jq -r '.ttft_p95_ms' "$POINT_METRICS_FILE")" + if awk -v ttft="$ttft_p95_ms" -v slo="${TTFT_SLO_MS:-4000}" 'BEGIN { exit !(ttft > slo) }'; then + adaptive_append_point_from_metrics \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ + "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ + "$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + stop_reason="TTFT_SLO_EXCEEDED" + shape_status="TTFT_BOUNDARY" + log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape" + break + fi + fi + adaptive_append_point_from_metrics \ "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ @@ -625,6 +650,8 @@ adaptive_main() { --argjson min_gain_pct "$TPS_MIN_GAIN_PCT" \ --argjson plateau_patience "$PLATEAU_PATIENCE" \ --argjson warmup_max_requests "$BENCH_WARMUP_MAX_REQUESTS" \ + --argjson ttft_slo_ms "${TTFT_SLO_MS:-4000}" \ + --argjson enable_ttft_slo_stop "${ENABLE_TTFT_SLO_STOP:-1}" \ '{ experiment: $experiment, engine: $engine, @@ -642,7 +669,9 @@ adaptive_main() { num_prompts_multiplier: $prompts_multiplier, min_tps_gain_pct: $min_gain_pct, plateau_patience: $plateau_patience, - warmup_max_requests: $warmup_max_requests + warmup_max_requests: $warmup_max_requests, + ttft_slo_ms: $ttft_slo_ms, + enable_ttft_slo_stop: $enable_ttft_slo_stop } }' > "${ADAPTIVE_RUN_ROOT}/run_manifest.json"