90fc378d2b
- Add dsv4_h20_vllm_tp_dp_matrix experiment (vLLM backend) - Add dsv4_h20_sglang_tp_dp_matrix experiment (SGLang backend) - Add nvidia_h20 platform config with H20-specific paths - Fix platform.sh auto-detection to distinguish H20 from H200 - Fix vLLM Docker startup: remove --entrypoint override for CDI compat - Fix vLLM 0.25.1 CLI args: remove redundant 'serve' from SERVER_ARGS - Download ShareGPT dataset to local datasets dir - Rename DSL to OSL across both experiments
50 lines
2.3 KiB
Bash
50 lines
2.3 KiB
Bash
# Adaptive concurrency search settings.
|
|
#
|
|
# For each fixed (TP, DP, ISL, OSL), probe:
|
|
# C = start, start * multiplier, ... up to max
|
|
# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
|
|
# PLATEAU_PATIENCE consecutive points.
|
|
|
|
SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
|
|
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
|
|
SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
|
|
NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
|
|
|
|
# A gain below 2% is treated as throughput saturation. Two consecutive
|
|
# low-gain points prevent one noisy measurement from stopping the search.
|
|
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
|
|
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
|
|
|
|
# Keep the same random workload semantics as the fixed matrix baseline.
|
|
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
|
|
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
|
|
# random-ids to use generated token IDs without a ShareGPT seed dataset.
|
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
|
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
|
|
# ISL/OSL points; lower values intentionally benchmark a length distribution.
|
|
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
|
|
# Before each measured point, warm up with the same concurrency so lazy kernel
|
|
# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
|
|
# cap; set a positive cap only when very high-concurrency warmup is impractical.
|
|
BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
|
|
|
|
# Reject a point if the completed request count or actual token lengths do not
|
|
# match the requested workload.
|
|
INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
|
|
OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
|
|
|
|
MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
|
|
SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
|
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
|
|
|
# Optional space-separated filters, useful for smoke tests:
|
|
# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
|
|
TP_LIST="${TP_LIST:-}"
|
|
ISL_LIST="${ISL_LIST:-}"
|
|
OSL_LIST="${OSL_LIST:-}"
|
|
|
|
DRY_RUN="${DRY_RUN:-0}"
|
|
# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
|
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|