qqtang/sskj
1
2
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(adaptive): add TTFT SLO early-stop and fix relative paths for h20 experiments

Browse Source 
- Add TTFT_SLO_MS and ENABLE_TTFT_SLO_STOP config to adaptive search
- Stop searching current shape when TTFT P95 exceeds SLO (default 3000ms)
- Fix relative paths in h20 scripts: ../../scripts/common -> ../../../scripts/common
- Update compare.py default TTFT limit to 4000ms
This commit is contained in:
SSKJ Dev 2026-07-16 05:15:51 +00:00
parent a4e38b9e33
commit dcb8cd2131
10 changed files with 62 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env
View File

@ -15,6 +15,12 @@ NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
# TTFT SLO early-stop settings.
# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL)
# shape and move on to the next scenario.
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
# Keep the same random workload semantics as the fixed matrix baseline. # Keep the same random workload semantics as the fixed matrix baseline.
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times # DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to # NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to

6
experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
View File

@ -6,15 +6,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh" source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env" source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/adaptive_config.env" source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
ENGINE="sglang" ENGINE="sglang"
ENGINE_PORT="$SGLANG_PORT" ENGINE_PORT="$SGLANG_PORT"

6
experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
View File

@ -6,15 +6,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh" source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env" source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/adaptive_config.env" source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
ENGINE="sglang" ENGINE="sglang"
ENGINE_PORT="$SGLANG_PORT" ENGINE_PORT="$SGLANG_PORT"

6
experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh
View File

@ -6,9 +6,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh" source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env" source "${SCRIPT_DIR}/config.env"
@ -522,7 +522,7 @@ run_parallel_config() {
# Parse results. # Parse results.
log "parsing ${config_label} results" log "parsing ${config_label} results"
"$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend sglang \ "$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend sglang \
>> "${phase_log_dir}/parse.log" 2>&1 || { >> "${phase_log_dir}/parse.log" 2>&1 || {
log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log"
} }

6
experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env
View File

@ -15,6 +15,12 @@ NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
# TTFT SLO early-stop settings.
# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL)
# shape and move on to the next scenario.
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
# Keep the same random workload semantics as the fixed matrix baseline. # Keep the same random workload semantics as the fixed matrix baseline.
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times # DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to # NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to

4
experiments/h20/dsv4_h20_vllm_tp_dp_matrix/compare.py
View File

@ -18,7 +18,7 @@ def load_result(result_root: Path) -> dict:
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, def slo_status(ttft_p95_ms: float, tpot_mean_ms: float,
ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: ttft_limit_ms: float = 4000.0, tpot_limit_ms: float = 50.0) -> str:
ttft_ok = ttft_p95_ms < ttft_limit_ms ttft_ok = ttft_p95_ms < ttft_limit_ms
tpot_ok = tpot_mean_ms < tpot_limit_ms tpot_ok = tpot_mean_ms < tpot_limit_ms
if ttft_ok and tpot_ok: if ttft_ok and tpot_ok:
@ -42,7 +42,7 @@ def main():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument("--run-root", type=Path, required=True) parser.add_argument("--run-root", type=Path, required=True)
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
parser.add_argument("--ttft-limit", type=float, default=3000.0) parser.add_argument("--ttft-limit", type=float, default=4000.0)
parser.add_argument("--tpot-limit", type=float, default=50.0) parser.add_argument("--tpot-limit", type=float, default=50.0)
args = parser.parse_args() args = parser.parse_args()

6
experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh
View File

@ -6,15 +6,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh" source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env" source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/adaptive_config.env" source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
ENGINE="vllm" ENGINE="vllm"
ENGINE_PORT="$VLLM_PORT" ENGINE_PORT="$VLLM_PORT"

6
experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
View File

@ -6,15 +6,15 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh" source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env" source "${SCRIPT_DIR}/config.env"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/adaptive_config.env" source "${SCRIPT_DIR}/adaptive_config.env"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/adaptive_bench_lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
ENGINE="vllm" ENGINE="vllm"
ENGINE_PORT="$VLLM_PORT" ENGINE_PORT="$VLLM_PORT"

6
experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh
View File

@ -6,9 +6,9 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/lib.sh" source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/../../scripts/common/platform.sh" source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
# shellcheck source=/dev/null # shellcheck source=/dev/null
source "${SCRIPT_DIR}/config.env" source "${SCRIPT_DIR}/config.env"
@ -512,7 +512,7 @@ run_parallel_config() {
# Parse results. # Parse results.
log "parsing ${config_label} results" log "parsing ${config_label} results"
"$PYTHON" "${SCRIPT_DIR}/../../scripts/common/parse_backend.py" "$result_root" --backend vllm \ "$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend vllm \
>> "${phase_log_dir}/parse.log" 2>&1 || { >> "${phase_log_dir}/parse.log" 2>&1 || {
log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log"
} }

31
scripts/common/adaptive_bench_lib.sh
View File

@ -69,6 +69,14 @@ adaptive_validate_config() {
log "ERROR: RANDOM_RANGE_RATIO must be between 0.0 and 1.0" log "ERROR: RANDOM_RANGE_RATIO must be between 0.0 and 1.0"
return 1 return 1
fi fi
if ! [[ "${TTFT_SLO_MS:-4000}" =~ ^[0-9]+$ ]] || (( TTFT_SLO_MS < 1 )); then
log "ERROR: TTFT_SLO_MS must be a positive integer, got ${TTFT_SLO_MS}"
return 1
fi
if [[ "${ENABLE_TTFT_SLO_STOP:-1}" != "0" && "${ENABLE_TTFT_SLO_STOP:-1}" != "1" ]]; then
log "ERROR: ENABLE_TTFT_SLO_STOP must be 0 or 1, got ${ENABLE_TTFT_SLO_STOP}"
return 1
fi
} }
adaptive_start_gpu_monitor() { adaptive_start_gpu_monitor() {
@ -408,6 +416,23 @@ adaptive_run_shape() {
fi fi
max_successful_concurrency="$concurrency" max_successful_concurrency="$concurrency"
last_total_tps="$current_tps" last_total_tps="$current_tps"
# TTFT SLO check: stop searching if TTFT P95 exceeds the SLO threshold
local ttft_p95_ms=""
if [[ "${ENABLE_TTFT_SLO_STOP:-1}" == "1" ]]; then
ttft_p95_ms="$(jq -r '.ttft_p95_ms' "$POINT_METRICS_FILE")"
if awk -v ttft="$ttft_p95_ms" -v slo="${TTFT_SLO_MS:-4000}" 'BEGIN { exit !(ttft > slo) }'; then
adaptive_append_point_from_metrics \
"$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \
"$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \
"$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG"
stop_reason="TTFT_SLO_EXCEEDED"
shape_status="TTFT_BOUNDARY"
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape"
break
fi
fi
adaptive_append_point_from_metrics \ adaptive_append_point_from_metrics \
"$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \
"$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \
@ -625,6 +650,8 @@ adaptive_main() {
--argjson min_gain_pct "$TPS_MIN_GAIN_PCT" \ --argjson min_gain_pct "$TPS_MIN_GAIN_PCT" \
--argjson plateau_patience "$PLATEAU_PATIENCE" \ --argjson plateau_patience "$PLATEAU_PATIENCE" \
--argjson warmup_max_requests "$BENCH_WARMUP_MAX_REQUESTS" \ --argjson warmup_max_requests "$BENCH_WARMUP_MAX_REQUESTS" \
--argjson ttft_slo_ms "${TTFT_SLO_MS:-4000}" \
--argjson enable_ttft_slo_stop "${ENABLE_TTFT_SLO_STOP:-1}" \
'{ '{
experiment: $experiment, experiment: $experiment,
engine: $engine, engine: $engine,
@ -642,7 +669,9 @@ adaptive_main() {
num_prompts_multiplier: $prompts_multiplier, num_prompts_multiplier: $prompts_multiplier,
min_tps_gain_pct: $min_gain_pct, min_tps_gain_pct: $min_gain_pct,
plateau_patience: $plateau_patience, plateau_patience: $plateau_patience,
warmup_max_requests: $warmup_max_requests warmup_max_requests: $warmup_max_requests,
ttft_slo_ms: $ttft_slo_ms,
enable_ttft_slo_stop: $enable_ttft_slo_stop
} }
}' > "${ADAPTIVE_RUN_ROOT}/run_manifest.json" }' > "${ADAPTIVE_RUN_ROOT}/run_manifest.json"