fix(bench): use default limits and back off after OOM

2026-07-20 11:37:18 +08:00 · 2026-07-20 11:37:18 +08:00 · 5ffbd0a21a
commit 5ffbd0a21a
parent ceb1170969
30 changed files with 60 additions and 51 deletions
--- a/README.md
+++ b/README.md
@ -1,5 +1,11 @@
 # sskj — 多平台大模型推理性能基准测试项目

+> **更新（2026-07-20，`903084c`）**
+> - H20 / RTX 6000D 的 TPxDP matrix baseline 不再显式限制模型上下文或服务内并发：vLLM 使用默认 `max-model-len` / `max-num-seqs`，SGLang 使用默认 `context-length` / `max-running-requests`。
+> - SGLang 固定使用 `--moe-runner-backend marlin`；vLLM 保持框架默认 MoE backend。
+> - `run_adaptive_concurrency_add16.sh` 在首点 OOM 时会重启服务并按 `C=16 -> 8 -> 1` 回退；仅 H20 / RTX 6000D matrix 默认开启，H200 / P800 保持原行为。
+> - RTX 6000D 旧 SGLang 结果使用 262K context cap，超过该范围或 C>32 的点不可与新 baseline 混用；请使用新的 `RUN_ID` 重跑受影响场景。
+
 > **项目目的**：当新显卡（GPU/NPU）到货时，用最短时间完成大模型在该平台上的推理性能评估与部署配置选型。
 > 当前模型：DeepSeek-V4-Flash（FP8 / INT8）；后续接入 GLM5.2，**完全复用**本项目的实验与报告流程。
 > 新平台接入 SOP：[`docs/NEW_PLATFORM_GUIDE.md`](docs/NEW_PLATFORM_GUIDE.md)。
--- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env
+++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/adaptive_config.env
@ -7,6 +7,9 @@

 SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
 SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+
+# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
+ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
 SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
 NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"

--- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env
+++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/config.env
@ -33,8 +33,7 @@ declare -a PARALLEL_CONFIGS=(
 )

 # SGLang server settings
-CONTEXT_LENGTH="${CONTEXT_LENGTH:-1048576}"
-MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-128}"
+# Context length and scheduler capacity use SGLang's model/default settings.
 MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.88}"
 MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}"

--- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
+++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
@ -72,8 +72,6 @@ engine_build_server_args() {
    --trust-remote-code
    --tp-size "$tp"
    --moe-runner-backend "$MOE_RUNNER_BACKEND"
-    --context-length "$CONTEXT_LENGTH"
-    --max-running-requests "$MAX_RUNNING_REQUESTS"
    --mem-fraction-static "$MEM_FRACTION_STATIC"
    --host 0.0.0.0
    --port "$ENGINE_PORT"
--- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
+++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
@ -72,8 +72,6 @@ engine_build_server_args() {
    --trust-remote-code
    --tp-size "$tp"
    --moe-runner-backend "$MOE_RUNNER_BACKEND"
-    --context-length "$CONTEXT_LENGTH"
-    --max-running-requests "$MAX_RUNNING_REQUESTS"
    --mem-fraction-static "$MEM_FRACTION_STATIC"
    --host 0.0.0.0
    --port "$ENGINE_PORT"
--- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh
+++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh
@ -94,8 +94,6 @@ build_server_args() {
    --trust-remote-code
    --tp-size "$tp"
    --moe-runner-backend "$MOE_RUNNER_BACKEND"
-    --context-length "$CONTEXT_LENGTH"
-    --max-running-requests "$MAX_RUNNING_REQUESTS"
    --mem-fraction-static "$MEM_FRACTION_STATIC"
    --host 0.0.0.0
    --port "$SGLANG_PORT"
--- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh
+++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_sglang_in_container.sh
@ -27,8 +27,6 @@ SERVER_ARGS=(
  --trust-remote-code
  --tp-size "$TP"
  --moe-runner-backend "$MOE_RUNNER_BACKEND"
-  --context-length "$CONTEXT_LENGTH"
-  --max-running-requests "$MAX_RUNNING_REQUESTS"
  --mem-fraction-static "$MEM_FRACTION_STATIC"
  --host 0.0.0.0
  --port "$PORT"
--- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh
+++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_docker.sh
@ -33,8 +33,6 @@ SERVER_ARGS=(
  --trust-remote-code
  --tp-size "$TP"
  --moe-runner-backend "$MOE_RUNNER_BACKEND"
-  --context-length "$CONTEXT_LENGTH"
-  --max-running-requests "$MAX_RUNNING_REQUESTS"
  --mem-fraction-static "$MEM_FRACTION_STATIC"
  --host 0.0.0.0
  --port "$PORT"
--- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh
+++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/start_sglang_dp.sh
@ -38,8 +38,6 @@ SERVER_ARGS=(
  --trust-remote-code
  --tp-size "$TP"
  --moe-runner-backend "$MOE_RUNNER_BACKEND"
-  --context-length "$CONTEXT_LENGTH"
-  --max-running-requests "$MAX_RUNNING_REQUESTS"
  --mem-fraction-static "$MEM_FRACTION_STATIC"
  --host 0.0.0.0
  --port "$SGLANG_PORT"
--- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env
+++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_config.env
@ -7,6 +7,9 @@

 SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
 SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+
+# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
+ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
 SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
 NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"

--- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env
+++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/config.env
@ -33,8 +33,7 @@ declare -a PARALLEL_CONFIGS=(
 )

 # vLLM server settings
-MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
-MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
+# Context length and scheduler capacity use vLLM's model/default settings.
 GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
 KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
 BLOCK_SIZE="${BLOCK_SIZE:-256}"
--- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh
+++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh
@ -66,8 +66,6 @@ engine_build_server_args() {
    --block-size "$BLOCK_SIZE"
    --tensor-parallel-size "$tp"
    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
-    --max-model-len "$MAX_MODEL_LEN"
-    --max-num-seqs "$MAX_NUM_SEQS"
    --no-enable-flashinfer-autotune
    --host 0.0.0.0
    --port "$ENGINE_PORT"
--- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
+++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
@ -66,8 +66,6 @@ engine_build_server_args() {
    --block-size "$BLOCK_SIZE"
    --tensor-parallel-size "$tp"
    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
-    --max-model-len "$MAX_MODEL_LEN"
-    --max-num-seqs "$MAX_NUM_SEQS"
    --no-enable-flashinfer-autotune
    --host 0.0.0.0
    --port "$ENGINE_PORT"
--- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh
+++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_bench.sh
@ -86,8 +86,6 @@ build_server_args() {
    --block-size "$BLOCK_SIZE"
    --tensor-parallel-size "$tp"
    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
-    --max-model-len "$MAX_MODEL_LEN"
-    --max-num-seqs "$MAX_NUM_SEQS"
    --no-enable-flashinfer-autotune
    --host 0.0.0.0
    --port "$VLLM_PORT"
--- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh
+++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_docker.sh
@ -34,8 +34,6 @@ SERVER_ARGS=(
  --block-size "$BLOCK_SIZE"
  --tensor-parallel-size "$TP"
  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
-  --max-model-len "$MAX_MODEL_LEN"
-  --max-num-seqs "$MAX_NUM_SEQS"
  --no-enable-flashinfer-autotune
  --host 0.0.0.0
  --port "$PORT"
--- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh
+++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/start_vllm_dp.sh
@ -39,8 +39,6 @@ SERVER_ARGS=(
  --block-size "$BLOCK_SIZE"
  --tensor-parallel-size "$TP"
  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
-  --max-model-len "$MAX_MODEL_LEN"
-  --max-num-seqs "$MAX_NUM_SEQS"
  --no-enable-flashinfer-autotune
  --host 0.0.0.0
  --port "$VLLM_PORT"
--- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/adaptive_config.env
@ -7,6 +7,9 @@

 SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
 SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+
+# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
+ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
 SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
 NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"

--- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/config.env
@ -34,8 +34,7 @@ declare -a PARALLEL_CONFIGS=(
 )

 # SGLang server settings (tuned for RTX 6000D 83GB)
-CONTEXT_LENGTH="${CONTEXT_LENGTH:-262144}"
-MAX_RUNNING_REQUESTS="${MAX_RUNNING_REQUESTS:-32}"
+# Context length and scheduler capacity use SGLang's model/default settings.
 MEM_FRACTION_STATIC="${MEM_FRACTION_STATIC:-0.80}"
 MOE_RUNNER_BACKEND="${MOE_RUNNER_BACKEND:-marlin}"

--- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency.sh
@ -72,8 +72,6 @@ engine_build_server_args() {
    --trust-remote-code
    --tp-size "$tp"
    --moe-runner-backend "$MOE_RUNNER_BACKEND"
-    --context-length "$CONTEXT_LENGTH"
-    --max-running-requests "$MAX_RUNNING_REQUESTS"
    --mem-fraction-static "$MEM_FRACTION_STATIC"
    --host 0.0.0.0
    --port "$ENGINE_PORT"
--- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_adaptive_concurrency_add16.sh
@ -72,8 +72,6 @@ engine_build_server_args() {
    --trust-remote-code
    --tp-size "$tp"
    --moe-runner-backend "$MOE_RUNNER_BACKEND"
-    --context-length "$CONTEXT_LENGTH"
-    --max-running-requests "$MAX_RUNNING_REQUESTS"
    --mem-fraction-static "$MEM_FRACTION_STATIC"
    --host 0.0.0.0
    --port "$ENGINE_PORT"
--- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_bench.sh
@ -94,8 +94,6 @@ build_server_args() {
    --trust-remote-code
    --tp-size "$tp"
    --moe-runner-backend "$MOE_RUNNER_BACKEND"
-    --context-length "$CONTEXT_LENGTH"
-    --max-running-requests "$MAX_RUNNING_REQUESTS"
    --mem-fraction-static "$MEM_FRACTION_STATIC"
    --host 0.0.0.0
    --port "$SGLANG_PORT"
--- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/run_sglang_in_container.sh
@ -27,8 +27,6 @@ SERVER_ARGS=(
  --trust-remote-code
  --tp-size "$TP"
  --moe-runner-backend "$MOE_RUNNER_BACKEND"
-  --context-length "$CONTEXT_LENGTH"
-  --max-running-requests "$MAX_RUNNING_REQUESTS"
  --mem-fraction-static "$MEM_FRACTION_STATIC"
  --host 0.0.0.0
  --port "$PORT"
--- a/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh
+++ b/experiments/pro6000/dsv4_pro6000_sglang_tp_dp_matrix/start_sglang_docker.sh
@ -33,8 +33,6 @@ SERVER_ARGS=(
  --trust-remote-code
  --tp-size "$TP"
  --moe-runner-backend "$MOE_RUNNER_BACKEND"
-  --context-length "$CONTEXT_LENGTH"
-  --max-running-requests "$MAX_RUNNING_REQUESTS"
  --mem-fraction-static "$MEM_FRACTION_STATIC"
  --host 0.0.0.0
  --port "$PORT"
--- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/adaptive_config.env
@ -7,6 +7,9 @@

 SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
 SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
+
+# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
+ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
 SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
 NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"

--- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/config.env
@ -33,8 +33,7 @@ declare -a PARALLEL_CONFIGS=(
 )

 # vLLM server settings
-MAX_MODEL_LEN="${MAX_MODEL_LEN:-1048576}"
-MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
+# Context length and scheduler capacity use vLLM's model/default settings.
 GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
 KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}"
 BLOCK_SIZE="${BLOCK_SIZE:-256}"
--- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency.sh
@ -66,8 +66,6 @@ engine_build_server_args() {
    --block-size "$BLOCK_SIZE"
    --tensor-parallel-size "$tp"
    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
-    --max-model-len "$MAX_MODEL_LEN"
-    --max-num-seqs "$MAX_NUM_SEQS"
    --host 0.0.0.0
    --port "$ENGINE_PORT"
  )
--- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
@ -66,8 +66,6 @@ engine_build_server_args() {
    --block-size "$BLOCK_SIZE"
    --tensor-parallel-size "$tp"
    --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
-    --max-model-len "$MAX_MODEL_LEN"
-    --max-num-seqs "$MAX_NUM_SEQS"
    --host 0.0.0.0
    --port "$ENGINE_PORT"
  )
--- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_docker.sh
@ -34,8 +34,6 @@ SERVER_ARGS=(
  --block-size "$BLOCK_SIZE"
  --tensor-parallel-size "$TP"
  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
-  --max-model-len "$MAX_MODEL_LEN"
-  --max-num-seqs "$MAX_NUM_SEQS"
  --host 0.0.0.0
  --port "$PORT"
 )
--- a/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh
+++ b/experiments/pro6000/dsv4_pro6000_vllm_tp_dp_matrix/start_vllm_dp.sh
@ -39,8 +39,6 @@ SERVER_ARGS=(
  --block-size "$BLOCK_SIZE"
  --tensor-parallel-size "$TP"
  --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
-  --max-model-len "$MAX_MODEL_LEN"
-  --max-num-seqs "$MAX_NUM_SEQS"
  --host 0.0.0.0
  --port "$VLLM_PORT"
 )
--- a/scripts/common/adaptive_bench_lib.sh
+++ b/scripts/common/adaptive_bench_lib.sh
@ -392,6 +392,7 @@ adaptive_run_shape() {
  local meaningful_gain=1
  local point_rc=0
  local initial_backoff_active=0
+  local initial_backoff_reason=""

  ADAPTIVE_RESTART_NEEDED=0
  ADAPTIVE_FATAL_WORKLOAD=0
@ -455,6 +456,7 @@ adaptive_run_shape() {
          if [[ -n "$next_backoff" ]]; then
            log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms; retrying initial boundary at c=${next_backoff}"
            initial_backoff_active=1
+            initial_backoff_reason="TTFT"
            concurrency="$next_backoff"
            previous_tps=""
            plateau_streak=0
@ -484,9 +486,15 @@ adaptive_run_shape() {

      log "probe result c=${concurrency} total_tps=${current_tps} gain_pct=${gain_pct:-first} plateau_streak=${plateau_streak}/${PLATEAU_PATIENCE}"
      if (( initial_backoff_active == 1 )); then
+        if [[ "$initial_backoff_reason" == "OOM" ]]; then
+          stop_reason="OOM_BACKOFF_FOUND"
+          shape_status="OOM_BOUNDARY"
+          log "initial OOM backoff found acceptable concurrency c=${concurrency}; stopping shape without probing higher values"
+        else
          stop_reason="TTFT_SLO_BACKOFF_FOUND"
          shape_status="TTFT_BOUNDARY"
          log "initial TTFT backoff found acceptable concurrency c=${concurrency}; stopping shape without probing lower values"
+        fi
        break
      fi
      if (( plateau_streak >= PLATEAU_PATIENCE )); then
@ -499,6 +507,33 @@ adaptive_run_shape() {
      adaptive_append_failed_point \
        "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \
        "$POINT_ATTEMPT" "OOM" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG"
+      local next_backoff=""
+      if [[ "${ENABLE_INITIAL_OOM_BACKOFF:-0}" == "1" ]] && \
+        (( concurrency == SEARCH_START_CONCURRENCY || initial_backoff_active == 1 )); then
+        local backoff
+        for backoff in ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}; do
+          if (( backoff < concurrency )); then
+            next_backoff="$backoff"
+            break
+          fi
+        done
+      fi
+      if [[ -n "$next_backoff" ]]; then
+        log "probe result c=${concurrency} OOM; restarting server and retrying initial boundary at c=${next_backoff}"
+        if ! adaptive_restart_server "$tp" "$dp"; then
+          shape_status="FAILED"
+          stop_reason="OOM_RESTART_FAILED"
+          ADAPTIVE_RESTART_NEEDED=1
+          break
+        fi
+        initial_backoff_active=1
+        initial_backoff_reason="OOM"
+        concurrency="$next_backoff"
+        previous_tps=""
+        plateau_streak=0
+        plateau_start_concurrency=""
+        continue
+      fi
      shape_status="OOM_BOUNDARY"
      stop_reason="OOM"
      ADAPTIVE_RESTART_NEEDED=1