diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/adaptive_config.env b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/adaptive_config.env index 3b05c7c..ce122ad 100644 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/adaptive_config.env +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/adaptive_config.env @@ -19,8 +19,10 @@ TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" # TTFT SLO early-stop settings. -# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL) +# When ttft_p95_ms exceeds the SLO, stop searching the current (ISL, OSL) # shape and move on to the next scenario. +# TTFT_SLO_MS is the flat fallback; if config.env defines get_ttft_slo_ms(), +# the per-ISL tiered SLO returned by that function takes precedence. TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" diff --git a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env index e2ab628..fb65300 100644 --- a/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env +++ b/experiments/910c/glm52_910c_vllm_tp_dp_matrix/config.env @@ -117,3 +117,22 @@ GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" DRY_RUN="${DRY_RUN:-0}" GRID_LIMIT="${GRID_LIMIT:-0}" + +# --------------------------------------------------------------------------- +# Tiered TTFT SLO by input sequence length. +# GLM-5.2 (w4a8c8) has simpler architecture than DSv4-Pro (no MLA/DeepSeek MoE), +# so its SLO thresholds are set slightly lower (~70-80% of DSv4-Pro values). +# This function is called by adaptive_bench_lib.sh during the concurrency +# sweep: when ttft_p95_ms exceeds the returned SLO, the current ISL/OSL shape +# search stops. +# --------------------------------------------------------------------------- +get_ttft_slo_ms() { + local isl="$1" + if (( isl <= 2048 )); then echo 5000 + elif (( isl <= 8192 )); then echo 8000 + elif (( isl <= 32768 )); then echo 12000 + elif (( isl <= 131072 )); then echo 20000 + else echo 30000 + fi +} +TTFT_SLO_TIERS_DESC="≤2048:5000ms, ≤8192:8000ms, ≤32768:12000ms, ≤131072:20000ms, >131072:30000ms" diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh index 10c4e84..a357757 100755 --- a/scripts/common/adaptive_bench_lib.sh +++ b/scripts/common/adaptive_bench_lib.sh @@ -96,6 +96,19 @@ adaptive_validate_config() { done } +# --------------------------------------------------------------------------- +# Per-ISL TTFT SLO lookup (tiered SLO support). +# Experiments can override this by defining get_ttft_slo_ms() in their +# config.env BEFORE sourcing this library. The default implementation +# returns the flat TTFT_SLO_MS value, preserving backward compatibility. +# --------------------------------------------------------------------------- +if ! declare -f get_ttft_slo_ms >/dev/null 2>&1; then + get_ttft_slo_ms() { + local isl="$1" + echo "${TTFT_SLO_MS:-4000}" + } +fi + adaptive_start_gpu_monitor() { local csv_path="$1" mkdir -p "$(dirname "$csv_path")" @@ -439,7 +452,9 @@ adaptive_run_shape() { local ttft_p95_ms="" if [[ "${ENABLE_TTFT_SLO_STOP:-1}" == "1" ]]; then ttft_p95_ms="$("$PYTHON" -c 'import json, sys; print(json.load(open(sys.argv[1]))["ttft_p95_ms"])' "$POINT_METRICS_FILE")" - if awk -v ttft="$ttft_p95_ms" -v slo="${TTFT_SLO_MS:-4000}" 'BEGIN { exit !(ttft > slo) }'; then + local slo_ms + slo_ms=$(get_ttft_slo_ms "$isl") + if awk -v ttft="$ttft_p95_ms" -v slo="$slo_ms" 'BEGIN { exit !(ttft > slo) }'; then adaptive_append_point_from_metrics \ "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ @@ -455,7 +470,7 @@ adaptive_run_shape() { done fi if [[ -n "$next_backoff" ]]; then - log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms; retrying initial boundary at c=${next_backoff}" + log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${slo_ms}ms (isl=${isl}); retrying initial boundary at c=${next_backoff}" initial_backoff_active=1 initial_backoff_reason="TTFT" concurrency="$next_backoff" @@ -476,7 +491,7 @@ adaptive_run_shape() { fi stop_reason="TTFT_SLO_EXCEEDED" shape_status="TTFT_BOUNDARY" - log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape" + log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${slo_ms}ms (isl=${isl}), stopping shape" break fi fi @@ -826,6 +841,7 @@ adaptive_main() { "${TTFT_SLO_MS:-4000}" \ "${ENABLE_TTFT_SLO_STOP:-1}" \ "${TTFT_GROUP_SKIP_MS:-0}" \ + "${TTFT_SLO_TIERS_DESC:-}" \ > "${ADAPTIVE_RUN_ROOT}/run_manifest.json" <<'PY' import json, sys @@ -833,7 +849,8 @@ import json, sys tokenize_prompt, random_range_ratio, search_start, search_cap, search_multiplier, initial_backoff_concurrencies, prompts_multiplier, min_gain_pct, plateau_patience, warmup_max_requests, ttft_slo_ms, - enable_ttft_slo_stop, ttft_group_skip_ms) = sys.argv[1:21] + enable_ttft_slo_stop, ttft_group_skip_ms, + ttft_slo_tiers_desc) = sys.argv[1:22] print(json.dumps({ "experiment": experiment, @@ -857,6 +874,7 @@ print(json.dumps({ "plateau_patience": int(plateau_patience), "warmup_max_requests": int(warmup_max_requests), "ttft_slo_ms": float(ttft_slo_ms), + "ttft_slo_tiers_desc": ttft_slo_tiers_desc, "enable_ttft_slo_stop": int(enable_ttft_slo_stop), "ttft_group_skip_ms": float(ttft_group_skip_ms), },