feat(adaptive): support tiered per-ISL TTFT SLO via get_ttft_slo_ms()
- adaptive_bench_lib.sh: add default get_ttft_slo_ms() fallback (flat TTFT_SLO_MS), use it instead of hardcoded TTFT_SLO_MS in SLO comparison and logs, add ttft_slo_tiers_desc to run_manifest.json - glm52_910c config.env: define tiered SLO for GLM-5.2: ≤2048:5000ms, ≤8192:8000ms, ≤32768:12000ms, ≤131072:20000ms, >131072:30000ms (~70-80% of DSv4-Pro values, since GLM-5.2 has simpler architecture) - glm52_910c adaptive_config.env: update TTFT_SLO_MS comment noting tiered override Backward compatible: experiments without get_ttft_slo_ms() keep flat 4000ms behavior.
This commit is contained in:
shishi
parent
ad4fd2b878
commit
e885fd0dc2
@ -19,8 +19,10 @@ TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
|
||||
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
|
||||
|
||||
# TTFT SLO early-stop settings.
|
||||
# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL)
|
||||
# When ttft_p95_ms exceeds the SLO, stop searching the current (ISL, OSL)
|
||||
# shape and move on to the next scenario.
|
||||
# TTFT_SLO_MS is the flat fallback; if config.env defines get_ttft_slo_ms(),
|
||||
# the per-ISL tiered SLO returned by that function takes precedence.
|
||||
TTFT_SLO_MS="${TTFT_SLO_MS:-4000}"
|
||||
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
|
||||
|
||||
|
||||
@ -117,3 +117,22 @@ GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||
|
||||
DRY_RUN="${DRY_RUN:-0}"
|
||||
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tiered TTFT SLO by input sequence length.
|
||||
# GLM-5.2 (w4a8c8) has simpler architecture than DSv4-Pro (no MLA/DeepSeek MoE),
|
||||
# so its SLO thresholds are set slightly lower (~70-80% of DSv4-Pro values).
|
||||
# This function is called by adaptive_bench_lib.sh during the concurrency
|
||||
# sweep: when ttft_p95_ms exceeds the returned SLO, the current ISL/OSL shape
|
||||
# search stops.
|
||||
# ---------------------------------------------------------------------------
|
||||
get_ttft_slo_ms() {
|
||||
local isl="$1"
|
||||
if (( isl <= 2048 )); then echo 5000
|
||||
elif (( isl <= 8192 )); then echo 8000
|
||||
elif (( isl <= 32768 )); then echo 12000
|
||||
elif (( isl <= 131072 )); then echo 20000
|
||||
else echo 30000
|
||||
fi
|
||||
}
|
||||
TTFT_SLO_TIERS_DESC="≤2048:5000ms, ≤8192:8000ms, ≤32768:12000ms, ≤131072:20000ms, >131072:30000ms"
|
||||
|
||||
@ -96,6 +96,19 @@ adaptive_validate_config() {
|
||||
done
|
||||
}
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Per-ISL TTFT SLO lookup (tiered SLO support).
|
||||
# Experiments can override this by defining get_ttft_slo_ms() in their
|
||||
# config.env BEFORE sourcing this library. The default implementation
|
||||
# returns the flat TTFT_SLO_MS value, preserving backward compatibility.
|
||||
# ---------------------------------------------------------------------------
|
||||
if ! declare -f get_ttft_slo_ms >/dev/null 2>&1; then
|
||||
get_ttft_slo_ms() {
|
||||
local isl="$1"
|
||||
echo "${TTFT_SLO_MS:-4000}"
|
||||
}
|
||||
fi
|
||||
|
||||
adaptive_start_gpu_monitor() {
|
||||
local csv_path="$1"
|
||||
mkdir -p "$(dirname "$csv_path")"
|
||||
@ -439,7 +452,9 @@ adaptive_run_shape() {
|
||||
local ttft_p95_ms=""
|
||||
if [[ "${ENABLE_TTFT_SLO_STOP:-1}" == "1" ]]; then
|
||||
ttft_p95_ms="$("$PYTHON" -c 'import json, sys; print(json.load(open(sys.argv[1]))["ttft_p95_ms"])' "$POINT_METRICS_FILE")"
|
||||
if awk -v ttft="$ttft_p95_ms" -v slo="${TTFT_SLO_MS:-4000}" 'BEGIN { exit !(ttft > slo) }'; then
|
||||
local slo_ms
|
||||
slo_ms=$(get_ttft_slo_ms "$isl")
|
||||
if awk -v ttft="$ttft_p95_ms" -v slo="$slo_ms" 'BEGIN { exit !(ttft > slo) }'; then
|
||||
adaptive_append_point_from_metrics \
|
||||
"$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \
|
||||
"$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \
|
||||
@ -455,7 +470,7 @@ adaptive_run_shape() {
|
||||
done
|
||||
fi
|
||||
if [[ -n "$next_backoff" ]]; then
|
||||
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms; retrying initial boundary at c=${next_backoff}"
|
||||
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${slo_ms}ms (isl=${isl}); retrying initial boundary at c=${next_backoff}"
|
||||
initial_backoff_active=1
|
||||
initial_backoff_reason="TTFT"
|
||||
concurrency="$next_backoff"
|
||||
@ -476,7 +491,7 @@ adaptive_run_shape() {
|
||||
fi
|
||||
stop_reason="TTFT_SLO_EXCEEDED"
|
||||
shape_status="TTFT_BOUNDARY"
|
||||
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape"
|
||||
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${slo_ms}ms (isl=${isl}), stopping shape"
|
||||
break
|
||||
fi
|
||||
fi
|
||||
@ -826,6 +841,7 @@ adaptive_main() {
|
||||
"${TTFT_SLO_MS:-4000}" \
|
||||
"${ENABLE_TTFT_SLO_STOP:-1}" \
|
||||
"${TTFT_GROUP_SKIP_MS:-0}" \
|
||||
"${TTFT_SLO_TIERS_DESC:-}" \
|
||||
> "${ADAPTIVE_RUN_ROOT}/run_manifest.json" <<'PY'
|
||||
import json, sys
|
||||
|
||||
@ -833,7 +849,8 @@ import json, sys
|
||||
tokenize_prompt, random_range_ratio, search_start, search_cap,
|
||||
search_multiplier, initial_backoff_concurrencies, prompts_multiplier,
|
||||
min_gain_pct, plateau_patience, warmup_max_requests, ttft_slo_ms,
|
||||
enable_ttft_slo_stop, ttft_group_skip_ms) = sys.argv[1:21]
|
||||
enable_ttft_slo_stop, ttft_group_skip_ms,
|
||||
ttft_slo_tiers_desc) = sys.argv[1:22]
|
||||
|
||||
print(json.dumps({
|
||||
"experiment": experiment,
|
||||
@ -857,6 +874,7 @@ print(json.dumps({
|
||||
"plateau_patience": int(plateau_patience),
|
||||
"warmup_max_requests": int(warmup_max_requests),
|
||||
"ttft_slo_ms": float(ttft_slo_ms),
|
||||
"ttft_slo_tiers_desc": ttft_slo_tiers_desc,
|
||||
"enable_ttft_slo_stop": int(enable_ttft_slo_stop),
|
||||
"ttft_group_skip_ms": float(ttft_group_skip_ms),
|
||||
},
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user