!9 update VLLM + GLM + H20
Merge pull request !9 from iiGray/auto/main/16070382/c54513b9-1
This commit is contained in:
Quantong Qiu
Gitee
commit
eaf005845a
6
.gitignore
vendored
6
.gitignore
vendored
@ -46,3 +46,9 @@ experiments/*/runtime/
|
|||||||
|
|
||||||
# 无关项目
|
# 无关项目
|
||||||
loomeval_yy/
|
loomeval_yy/
|
||||||
|
|
||||||
|
.gitignore
|
||||||
|
|
||||||
|
dsv4_dspark_h20_sglang_tp_dp_matrix
|
||||||
|
dsv4_h20_sglang_tp_dp_matrix
|
||||||
|
glm_old_h20_vllm_tp_dp_matrix
|
||||||
@ -0,0 +1,58 @@
|
|||||||
|
# Adaptive concurrency search settings.
|
||||||
|
#
|
||||||
|
# For each fixed (TP, DP, ISL, OSL), probe:
|
||||||
|
# C = start, start * multiplier, ... up to max
|
||||||
|
# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for
|
||||||
|
# PLATEAU_PATIENCE consecutive points.
|
||||||
|
|
||||||
|
SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}"
|
||||||
|
SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-512}"
|
||||||
|
|
||||||
|
# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM.
|
||||||
|
ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}"
|
||||||
|
SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}"
|
||||||
|
NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}"
|
||||||
|
|
||||||
|
# A gain below 2% is treated as throughput saturation. Two consecutive
|
||||||
|
# low-gain points prevent one noisy measurement from stopping the search.
|
||||||
|
TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}"
|
||||||
|
PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}"
|
||||||
|
|
||||||
|
# TTFT SLO early-stop settings.
|
||||||
|
# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL)
|
||||||
|
# shape and move on to the next scenario.
|
||||||
|
TTFT_SLO_MS="${TTFT_SLO_MS:-6000}"
|
||||||
|
ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}"
|
||||||
|
|
||||||
|
# Keep the same random workload semantics as the fixed matrix baseline.
|
||||||
|
# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times
|
||||||
|
# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to
|
||||||
|
# random-ids to use generated token IDs without a ShareGPT seed dataset.
|
||||||
|
BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}"
|
||||||
|
# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed
|
||||||
|
# ISL/OSL points; lower values intentionally benchmark a length distribution.
|
||||||
|
RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}"
|
||||||
|
# Before each measured point, warm up with the same concurrency so lazy kernel
|
||||||
|
# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no
|
||||||
|
# cap; set a positive cap only when very high-concurrency warmup is impractical.
|
||||||
|
BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}"
|
||||||
|
|
||||||
|
# Reject a point if the completed request count or actual token lengths do not
|
||||||
|
# match the requested workload.
|
||||||
|
INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}"
|
||||||
|
OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}"
|
||||||
|
|
||||||
|
MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}"
|
||||||
|
SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}"
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
|
||||||
|
# Optional space-separated filters, useful for smoke tests:
|
||||||
|
# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128"
|
||||||
|
TP_LIST="${TP_LIST:-}"
|
||||||
|
ISL_LIST="${ISL_LIST:-}"
|
||||||
|
OSL_LIST="${OSL_LIST:-}"
|
||||||
|
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes.
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
@ -0,0 +1,12 @@
|
|||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":66.07890335196862,"request_tps":1.2106738450830639,"input_tps":1239.7300173650574,"output_tps":154.96625217063217,"total_tps":1394.6962695356895,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":2246.366287989076,"ttft_p95_ms":26584.918520136853,"ttft_p99_ms":27947.81640406112,"tpot_p50_ms":32.686295791246785,"tpot_p95_ms":284.34243235967625,"tpot_p99_ms":284.3454002905967,"e2e_p50_ms":5641.515022027306,"e2e_p95_ms":43821.97715669463,"e2e_p99_ms":43826.05055155698,"itl_p50_ms":24.928085505962372,"itl_p95_ms":25.751776251127012,"itl_p99_ms":29.517283142195083,"validation_errors":[],"timestamp":"2026-07-20T13:13:15+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c16_i1024_o128_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":40,"failed":0,"duration_s":17.45319774496602,"request_tps":2.2918436257067616,"input_tps":2346.847872723724,"output_tps":293.3559840904655,"total_tps":2640.2038568141897,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":150.9479565138463,"ttft_p95_ms":3209.506361820968,"ttft_p99_ms":3211.302565172664,"tpot_p50_ms":21.474110523805276,"tpot_p95_ms":22.064654289847457,"tpot_p99_ms":36.181361379750854,"e2e_p50_ms":2888.627921027364,"e2e_p95_ms":5882.144264594535,"e2e_p99_ms":5883.902202068712,"itl_p50_ms":21.082962019136176,"itl_p95_ms":21.524865497485735,"itl_p99_ms":22.77547527395655,"validation_errors":[],"timestamp":"2026-07-20T13:14:15+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c8_i1024_o128_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c8_i1024_o128_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":73.13139667996438,"request_tps":1.0939214021864483,"input_tps":1120.175515838923,"output_tps":280.04387895973076,"total_tps":1400.2193947986536,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":3393.642414477654,"ttft_p95_ms":18858.08570390509,"ttft_p99_ms":18859.413588086027,"tpot_p50_ms":25.162762243131763,"tpot_p95_ms":98.46200510415512,"tpot_p99_ms":98.62357294589381,"e2e_p50_ms":9816.588030516868,"e2e_p95_ms":25310.153337288648,"e2e_p99_ms":25311.50291909813,"itl_p50_ms":24.936357513070107,"itl_p95_ms":25.62914474401623,"itl_p99_ms":26.600702406722117,"validation_errors":[],"timestamp":"2026-07-20T13:16:07+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c16_i1024_o256_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c16_i1024_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":40,"failed":0,"duration_s":28.223284891981166,"request_tps":1.4172694692730416,"input_tps":1451.2839365355946,"output_tps":362.82098413389866,"total_tps":1814.1049206694931,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":154.1392895160243,"ttft_p95_ms":379.09841130604036,"ttft_p99_ms":381.0434712585993,"tpot_p50_ms":21.32736587264648,"tpot_p95_ms":21.66258262403026,"tpot_p99_ms":21.946868581752128,"e2e_p50_ms":5611.429856478935,"e2e_p95_ms":5777.431349424296,"e2e_p99_ms":5779.790019096108,"itl_p50_ms":21.225339034572244,"itl_p95_ms":21.48797080735676,"itl_p99_ms":22.732056326931342,"validation_errors":[],"timestamp":"2026-07-20T13:17:11+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c8_i1024_o256_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c8_i1024_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":82.09354979795171,"request_tps":0.9744979988914556,"input_tps":997.8859508648505,"output_tps":498.94297543242527,"total_tps":1496.8289262972758,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":351.80018850951456,"ttft_p95_ms":16082.066819473404,"ttft_p99_ms":16083.18490182457,"tpot_p50_ms":25.351316181016966,"tpot_p95_ms":45.228610313900695,"tpot_p99_ms":56.46314928515786,"e2e_p50_ms":13332.221442979062,"e2e_p95_ms":29036.639165173983,"e2e_p99_ms":29038.089263379225,"itl_p50_ms":25.177069997880608,"itl_p95_ms":26.03796001058072,"itl_p99_ms":26.54033866361715,"validation_errors":[],"timestamp":"2026-07-20T13:19:11+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c16_i1024_o512_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c16_i1024_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":40,"failed":0,"duration_s":55.474242116033565,"request_tps":0.7210553668553664,"input_tps":738.3606956598952,"output_tps":369.1803478299476,"total_tps":1107.5410434898429,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":150.102293031523,"ttft_p95_ms":379.60697482922114,"ttft_p99_ms":381.0291459149448,"tpot_p50_ms":21.299922477486753,"tpot_p95_ms":21.498661930547524,"tpot_p99_ms":21.6191849751485,"e2e_p50_ms":11071.723695495166,"e2e_p95_ms":11224.84234466392,"e2e_p99_ms":11225.585542463232,"itl_p50_ms":21.26789599424228,"itl_p95_ms":21.488468878669664,"itl_p99_ms":21.97634468670003,"validation_errors":[],"timestamp":"2026-07-20T13:20:40+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c8_i1024_o512_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c8_i1024_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":133.75688559300033,"request_tps":0.5981000502914409,"input_tps":612.4544514984354,"output_tps":612.4544514984354,"total_tps":1224.908902996871,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":302.7113805001136,"ttft_p95_ms":400.9890645131236,"ttft_p99_ms":404.13553857652,"tpot_p50_ms":25.827961465794928,"tpot_p95_ms":26.28664292794489,"tpot_p99_ms":26.335648955562892,"e2e_p50_ms":26626.903105468955,"e2e_p95_ms":27081.094395113178,"e2e_p99_ms":27082.44388371997,"itl_p50_ms":25.617743958719075,"itl_p95_ms":27.75817652582191,"itl_p99_ms":28.319485834799703,"validation_errors":[],"timestamp":"2026-07-20T13:23:27+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c16_i1024_o1024_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c16_i1024_o1024_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":160,"failed":0,"duration_s":213.9958370079985,"request_tps":0.7476780961585702,"input_tps":765.6223704663759,"output_tps":765.6223704663759,"total_tps":1531.2447409327517,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":3110.6292720069177,"ttft_p95_ms":8488.881127754576,"ttft_p99_ms":8975.266096224077,"tpot_p50_ms":38.09060934700713,"tpot_p95_ms":44.891730105997254,"tpot_p99_ms":45.06499475674779,"e2e_p50_ms":42191.93685048958,"e2e_p95_ms":50177.619773335755,"e2e_p99_ms":50950.24837674457,"itl_p50_ms":36.72923950944096,"itl_p95_ms":38.72285781544633,"itl_p99_ms":39.56491003627888,"validation_errors":[],"timestamp":"2026-07-20T13:28:00+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":1024,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":25.008867,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c32_i1024_o1024_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c32_i1024_o1024_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":269.6854031760013,"request_tps":0.2966419355955674,"input_tps":303.761342049861,"output_tps":607.522684099722,"total_tps":911.284026149583,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":364.3683975096792,"ttft_p95_ms":2007.0290236442813,"ttft_p99_ms":2013.6113321600715,"tpot_p50_ms":26.085042208103296,"tpot_p95_ms":26.900448583844003,"tpot_p99_ms":27.08882838621471,"e2e_p50_ms":53703.43658546335,"e2e_p95_ms":55784.151440593996,"e2e_p99_ms":55822.559498712064,"itl_p50_ms":25.97841649549082,"itl_p95_ms":27.365541239851154,"itl_p99_ms":28.078064389992505,"validation_errors":[],"timestamp":"2026-07-20T13:33:06+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c16_i1024_o2048_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c16_i1024_o2048_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":160,"failed":0,"duration_s":400.4523162249825,"request_tps":0.39954819467221825,"input_tps":409.1373513443515,"output_tps":818.274702688703,"total_tps":1227.4120540330543,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":1562.171744502848,"ttft_p95_ms":3573.763989112922,"ttft_p99_ms":3709.1987722582408,"tpot_p50_ms":38.22010309941808,"tpot_p95_ms":39.27668409369036,"tpot_p99_ms":39.69951655217825,"e2e_p50_ms":80368.12274745898,"e2e_p95_ms":81558.74147544673,"e2e_p99_ms":82177.91087441437,"itl_p50_ms":38.162919983733445,"itl_p95_ms":38.95309220533818,"itl_p99_ms":39.51284954557195,"validation_errors":[],"timestamp":"2026-07-20T13:40:25+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":2048,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":34.690395,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c32_i1024_o2048_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c32_i1024_o2048_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":240,"failed":0,"duration_s":512.3003129389836,"request_tps":0.4684752164666054,"input_tps":479.7186216618039,"output_tps":959.4372433236078,"total_tps":1439.1558649854117,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":2483.1260589999147,"ttft_p95_ms":6005.64803337038,"ttft_p99_ms":7046.867935974151,"tpot_p50_ms":48.54345994357542,"tpot_p95_ms":49.545350488314874,"tpot_p99_ms":49.915874086901454,"e2e_p50_ms":102551.14213997149,"e2e_p95_ms":104216.0961176909,"e2e_p99_ms":105386.35410067334,"itl_p50_ms":47.52948897657916,"itl_p95_ms":48.73407759005204,"itl_p99_ms":49.2540165700484,"validation_errors":[],"timestamp":"2026-07-20T13:49:39+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":2048,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":17.251241,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c48_i1024_o2048_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c48_i1024_o2048_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":546.3728505309555,"request_tps":0.1464201596442016,"input_tps":149.93424347566244,"output_tps":599.7369739026498,"total_tps":749.6712173783122,"mean_input_tokens":1024.0,"mean_output_tokens":4096.0,"ttft_p50_ms":1404.011996492045,"ttft_p95_ms":2345.6240130530205,"ttft_p99_ms":2348.1909073353745,"tpot_p50_ms":26.30868239560101,"tpot_p95_ms":26.55169738825895,"tpot_p99_ms":26.783168154521316,"e2e_p50_ms":109413.76537151518,"e2e_p95_ms":110134.41487025993,"e2e_p99_ms":110135.74619777035,"itl_p50_ms":26.173043996095657,"itl_p95_ms":27.01872760662809,"itl_p99_ms":28.021232602186487,"validation_errors":[],"timestamp":"2026-07-20T13:59:21+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/raw_outputs/vllm_adaptive_c16_i1024_o4096_a1.jsonl","detail_log":"/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260720-130042/tp8_dp1/logs/vllm_c16_i1024_o4096_a1.log"}
|
||||||
@ -0,0 +1,5 @@
|
|||||||
|
{"timestamp":"2026-07-20T13:14:15+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":512,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":8,"best_total_tps":2640.2038568141897,"last_total_tps":2640.2038568141897}
|
||||||
|
{"timestamp":"2026-07-20T13:17:11+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":512,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":8,"best_total_tps":1814.1049206694931,"last_total_tps":1814.1049206694931}
|
||||||
|
{"timestamp":"2026-07-20T13:20:40+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":512,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":16,"best_total_tps":1496.8289262972758,"last_total_tps":1107.5410434898429}
|
||||||
|
{"timestamp":"2026-07-20T13:28:00+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":512,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":32,"best_total_tps":1531.2447409327517,"last_total_tps":1531.2447409327517}
|
||||||
|
{"timestamp":"2026-07-20T13:49:39+00:00","engine":"vllm","tp":8,"dp":1,"mark":"Y","isl":1024,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":1439.1558649854117,"last_total_tps":1439.1558649854117}
|
||||||
@ -0,0 +1,27 @@
|
|||||||
|
{
|
||||||
|
"experiment": "glm_h20_vllm_tp_dp_matrix",
|
||||||
|
"engine": "vllm",
|
||||||
|
"run_id": "adaptive_20260720-130042",
|
||||||
|
"model": "/data1/hf_models/GLM-5.2-FP8",
|
||||||
|
"hardware": "8x NVIDIA H20 96GB",
|
||||||
|
"matrix": "/data1/bbj/sskj/experiments/h20/glm_h20_vllm_tp_dp_matrix/matrix.json",
|
||||||
|
"dataset": "random",
|
||||||
|
"tokenize_prompt": false,
|
||||||
|
"random_range_ratio": 1.0,
|
||||||
|
"search": {
|
||||||
|
"start_concurrency": 16,
|
||||||
|
"max_concurrency": 512,
|
||||||
|
"multiplier": 2,
|
||||||
|
"initial_backoff_concurrencies": [
|
||||||
|
8,
|
||||||
|
1
|
||||||
|
],
|
||||||
|
"num_prompts_multiplier": 5,
|
||||||
|
"min_tps_gain_pct": 2.0,
|
||||||
|
"plateau_patience": 2,
|
||||||
|
"warmup_max_requests": 0,
|
||||||
|
"ttft_slo_ms": 4000.0,
|
||||||
|
"enable_ttft_slo_stop": 1,
|
||||||
|
"ttft_group_skip_ms": 12000.0
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -0,0 +1,40 @@
|
|||||||
|
mark input_len output_len
|
||||||
|
Y 1024 128
|
||||||
|
Y 1024 256
|
||||||
|
Y 1024 512
|
||||||
|
Y 1024 1024
|
||||||
|
Y 1024 2048
|
||||||
|
Y 1024 4096
|
||||||
|
Y 4096 128
|
||||||
|
Y 4096 256
|
||||||
|
Y 4096 512
|
||||||
|
Y 4096 1024
|
||||||
|
Y 4096 2048
|
||||||
|
Y 4096 4096
|
||||||
|
Y 8192 128
|
||||||
|
Y 8192 256
|
||||||
|
Y 8192 512
|
||||||
|
Y 8192 1024
|
||||||
|
Y 8192 2048
|
||||||
|
Y 8192 4096
|
||||||
|
Y 16384 128
|
||||||
|
Y 16384 256
|
||||||
|
Y 16384 512
|
||||||
|
Y 16384 1024
|
||||||
|
Y 16384 2048
|
||||||
|
Y 32768 128
|
||||||
|
Y 32768 256
|
||||||
|
Y 32768 512
|
||||||
|
Y 32768 1024
|
||||||
|
Y 32768 2048
|
||||||
|
Y 65536 128
|
||||||
|
Y 65536 256
|
||||||
|
Y 65536 512
|
||||||
|
Y 65536 1024
|
||||||
|
Y 131072 128
|
||||||
|
Y 131072 256
|
||||||
|
Y 131072 512
|
||||||
|
Y 262144 128
|
||||||
|
Y 262144 256
|
||||||
|
Y 524288 128
|
||||||
|
Y 1048576 128
|
||||||
|
163
experiments/h20/glm_h20_vllm_tp_dp_matrix/compare.py
Normal file
163
experiments/h20/glm_h20_vllm_tp_dp_matrix/compare.py
Normal file
@ -0,0 +1,163 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Cross TP×DP configuration comparison for dsv4_h20_vllm_tp_dp_matrix.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python3 compare.py --run-root results/<run_id> [--output comparison.md]
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from collections import defaultdict
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def load_result(result_root: Path) -> dict:
|
||||||
|
path = result_root / "results.json"
|
||||||
|
with open(path, "r", encoding="utf-8") as f:
|
||||||
|
return json.load(f)
|
||||||
|
|
||||||
|
|
||||||
|
def slo_status(ttft_p95_ms: float, tpot_mean_ms: float,
|
||||||
|
ttft_limit_ms: float = 5000.0, tpot_limit_ms: float = 50.0) -> str:
|
||||||
|
ttft_ok = ttft_p95_ms < ttft_limit_ms
|
||||||
|
tpot_ok = tpot_mean_ms < tpot_limit_ms
|
||||||
|
if ttft_ok and tpot_ok:
|
||||||
|
return "PASS"
|
||||||
|
if ttft_ok or tpot_ok:
|
||||||
|
return "PARTIAL"
|
||||||
|
return "FAIL"
|
||||||
|
|
||||||
|
|
||||||
|
def gpu_memory_str(gpu: dict | None) -> str:
|
||||||
|
if not gpu:
|
||||||
|
return "-"
|
||||||
|
peak = gpu.get("peak_used_mb", 0)
|
||||||
|
total = gpu.get("memory_total_mb", 0)
|
||||||
|
if total:
|
||||||
|
return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)"
|
||||||
|
return f"{peak:.0f}"
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--run-root", type=Path, required=True)
|
||||||
|
parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md"))
|
||||||
|
parser.add_argument("--ttft-limit", type=float, default=3000.0)
|
||||||
|
parser.add_argument("--tpot-limit", type=float, default=50.0)
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
# Discover configurations: tp*_dp* directories.
|
||||||
|
configs = []
|
||||||
|
for subdir in sorted(args.run_root.iterdir()):
|
||||||
|
if not subdir.is_dir():
|
||||||
|
continue
|
||||||
|
name = subdir.name
|
||||||
|
if not (name.startswith("tp") and "_dp" in name):
|
||||||
|
continue
|
||||||
|
results_json = subdir / "results.json"
|
||||||
|
if not results_json.exists():
|
||||||
|
continue
|
||||||
|
configs.append((name, load_result(subdir)))
|
||||||
|
|
||||||
|
if not configs:
|
||||||
|
print(f"No tp*_dp* results found under {args.run_root}")
|
||||||
|
return
|
||||||
|
|
||||||
|
model = configs[0][1].get("metadata", {}).get("model", "unknown")
|
||||||
|
hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown")
|
||||||
|
|
||||||
|
# Group by scenario name.
|
||||||
|
by_scenario: dict[str, dict[str, dict]] = defaultdict(dict)
|
||||||
|
skipped: dict[str, dict[str, str]] = defaultdict(dict)
|
||||||
|
for label, data in configs:
|
||||||
|
for s in data.get("scenarios", []):
|
||||||
|
key = s["name"]
|
||||||
|
if s.get("status") == "skipped_oom":
|
||||||
|
skipped[key][label] = s.get("note", "skipped")
|
||||||
|
else:
|
||||||
|
by_scenario[key][label] = s
|
||||||
|
|
||||||
|
with open(args.output, "w", encoding="utf-8") as f:
|
||||||
|
f.write(f"# vLLM TP×DP matrix comparison ({hardware})\n\n")
|
||||||
|
f.write("## Summary\n\n")
|
||||||
|
f.write(f"- Model: `{model}`\n")
|
||||||
|
f.write(f"- Hardware: {hardware}\n")
|
||||||
|
f.write("- Backend: vLLM (Docker)\n")
|
||||||
|
f.write("- Benchmark client: `sglang.bench_serving`\n")
|
||||||
|
f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n")
|
||||||
|
|
||||||
|
# Configuration overview.
|
||||||
|
f.write("### Configurations\n\n")
|
||||||
|
f.write("| Config | TP | DP | GPUs/replica | Notes |\n")
|
||||||
|
f.write("|---|---:|---:|---:|---|\n")
|
||||||
|
for label, data in configs:
|
||||||
|
cfg = data.get("config", {})
|
||||||
|
tp = cfg.get("tp", "?")
|
||||||
|
dp = cfg.get("dp", "?")
|
||||||
|
f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n")
|
||||||
|
f.write("\n")
|
||||||
|
|
||||||
|
# Side-by-side table.
|
||||||
|
f.write("## Side-by-side results\n\n")
|
||||||
|
headers = [
|
||||||
|
"Scenario", "ISL", "OSL", "Config", "Conc", "Req/s", "OutTok/s",
|
||||||
|
"TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)",
|
||||||
|
"TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO"
|
||||||
|
]
|
||||||
|
f.write("| " + " | ".join(headers) + " |\n")
|
||||||
|
f.write("|" + "|".join(["---"] * len(headers)) + "|\n")
|
||||||
|
|
||||||
|
for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))):
|
||||||
|
_, isl, osl = re.findall(r"\d+", scenario_name)
|
||||||
|
# cfg_part not used; just for readability.
|
||||||
|
for label, data in configs:
|
||||||
|
s = by_scenario[scenario_name].get(label)
|
||||||
|
if s is None:
|
||||||
|
if scenario_name in skipped and label in skipped[scenario_name]:
|
||||||
|
note = skipped[scenario_name][label]
|
||||||
|
f.write(f"| {scenario_name} | {isl} | {osl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n")
|
||||||
|
continue
|
||||||
|
cfg = s["config"]
|
||||||
|
m = s["metrics"]
|
||||||
|
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||||||
|
gpu = m.get("gpu_memory")
|
||||||
|
f.write(
|
||||||
|
f"| {scenario_name} | {isl} | {osl} | {label} | {cfg['concurrency']} | "
|
||||||
|
f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | "
|
||||||
|
f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | "
|
||||||
|
f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | "
|
||||||
|
f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
# Best throughput per ISL/OSL.
|
||||||
|
f.write("\n## Best throughput per (ISL, OSL)\n\n")
|
||||||
|
f.write("| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n")
|
||||||
|
f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n")
|
||||||
|
best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {}
|
||||||
|
for scenario_name, backends in by_scenario.items():
|
||||||
|
_, isl, osl = re.findall(r"\d+", scenario_name)
|
||||||
|
isl_i, osl_i = int(isl), int(osl)
|
||||||
|
for label, s in backends.items():
|
||||||
|
m = s["metrics"]
|
||||||
|
out_tok = m["output_token_throughput"]
|
||||||
|
if (isl_i, osl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, osl_i)][0]:
|
||||||
|
best_by_shape[(isl_i, osl_i)] = (out_tok, label, s)
|
||||||
|
for (isl_i, osl_i), (out_tok, label, s) in sorted(best_by_shape.items()):
|
||||||
|
m = s["metrics"]
|
||||||
|
status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit)
|
||||||
|
f.write(
|
||||||
|
f"| {isl_i} | {osl_i} | {label} | {s['config']['concurrency']} | "
|
||||||
|
f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
f.write("\n## Notes\n\n")
|
||||||
|
f.write("- SLO check uses TTFT P95 and TPOT mean.\n")
|
||||||
|
f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n")
|
||||||
|
f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n")
|
||||||
|
f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n")
|
||||||
|
|
||||||
|
print(f"Wrote comparison to {args.output}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
75
experiments/h20/glm_h20_vllm_tp_dp_matrix/config.env
Normal file
75
experiments/h20/glm_h20_vllm_tp_dp_matrix/config.env
Normal file
@ -0,0 +1,75 @@
|
|||||||
|
# TP×DP matrix experiment for DeepSeek-V4-Flash on H20 (8 GPUs) using vLLM.
|
||||||
|
# Tests vLLM with three parallel configurations:
|
||||||
|
# TP=2, DP=4 -> 2 GPUs per replica, 4 replicas
|
||||||
|
# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas
|
||||||
|
# TP=8, DP=1 -> 8 GPUs, no data parallelism
|
||||||
|
|
||||||
|
EXPERIMENT="glm_h20_vllm_tp_dp_matrix"
|
||||||
|
MODEL_NAME="GLM-5.2-FP8"
|
||||||
|
MODEL_PATH="/data1/hf_models/GLM-5.2-FP8"
|
||||||
|
SERVED_MODEL_NAME="GLM-5.2-FP8"
|
||||||
|
|
||||||
|
VLLM_PORT="${VLLM_PORT:-30030}"
|
||||||
|
|
||||||
|
# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.)
|
||||||
|
# and the benchmark client. Defaults to the system python3 if the sglang venv
|
||||||
|
# does not exist on the host.
|
||||||
|
VENV_CLIENT="${VENV_CLIENT:-./envs/sglang}"
|
||||||
|
|
||||||
|
# Run the benchmark client natively (0) or inside Docker (1).
|
||||||
|
USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}"
|
||||||
|
|
||||||
|
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}"
|
||||||
|
|
||||||
|
# Runtime working directory for logs, pid files, and tmp. Defaults to a local
|
||||||
|
# directory under this experiment so the benchmark is self-contained.
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
|
||||||
|
# Parallel configurations to test. Format: "TP DP"
|
||||||
|
declare -a PARALLEL_CONFIGS=(
|
||||||
|
"8 1"
|
||||||
|
)
|
||||||
|
|
||||||
|
# vLLM server settings
|
||||||
|
# Context length and scheduler capacity use vLLM's model/default settings.
|
||||||
|
MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}"
|
||||||
|
MAX_NUM_SEQS="${MAX_NUM_SEQS:-128}"
|
||||||
|
GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}"
|
||||||
|
KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-auto}"
|
||||||
|
BLOCK_SIZE="${BLOCK_SIZE:-64}"
|
||||||
|
|
||||||
|
# Deployment switch. 0 = native vllm venv, 1 = Docker.
|
||||||
|
USE_DOCKER="${USE_DOCKER:-1}"
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
|
||||||
|
|
||||||
|
# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving,
|
||||||
|
# so the client runs inside the SGLang image and targets the vLLM backend via the
|
||||||
|
# OpenAI-compatible API.
|
||||||
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||||||
|
|
||||||
|
# Dataset used by sglang.bench_serving --dataset-name random.
|
||||||
|
# The random sampler needs a ShareGPT-style JSON file locally; it falls back to
|
||||||
|
# downloading from HuggingFace, which usually fails on offline H20 nodes.
|
||||||
|
DATASET_PATH="${DATASET_PATH:-/data1/bbj/sskj/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}"
|
||||||
|
|
||||||
|
# Matrix and concurrency rules are defined in matrix.json by default.
|
||||||
|
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}"
|
||||||
|
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||||
|
|
||||||
|
# Sampling density for concurrency.
|
||||||
|
# 0 = use the default heuristic in generate_scenarios.py (6-8 points).
|
||||||
|
# 2 = only test the low and high endpoints.
|
||||||
|
export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}"
|
||||||
|
|
||||||
|
# Per-scenario timeout to avoid hangs (seconds).
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
|
||||||
|
# GPU memory sampling interval (seconds).
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
|
||||||
|
# Dry-run mode: if 1, only log the server args and scenario plan without starting
|
||||||
|
# any server or sending requests.
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
|
||||||
|
# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios.
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
@ -0,0 +1,98 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Generate the scenario list for the TP×DP matrix experiment.
|
||||||
|
|
||||||
|
Reads matrix.json and prints TSV lines:
|
||||||
|
|
||||||
|
mark input_len output_len concurrency num_prompts
|
||||||
|
|
||||||
|
mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import math
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
def sample_concurrency(low: int, high: int, target: int) -> list[int]:
|
||||||
|
"""Return only the low and high concurrency values in [low, high].
|
||||||
|
|
||||||
|
For the TP×DP matrix we only need the two endpoints of the concurrency
|
||||||
|
range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for
|
||||||
|
API compatibility but is ignored.
|
||||||
|
"""
|
||||||
|
assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}"
|
||||||
|
if low == high:
|
||||||
|
return [low]
|
||||||
|
return [low, high]
|
||||||
|
|
||||||
|
|
||||||
|
def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]:
|
||||||
|
with open(matrix_path, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
matrix = data["matrix"]
|
||||||
|
concurrency_cfg = data["concurrency"]
|
||||||
|
|
||||||
|
scenarios = []
|
||||||
|
for isl_str in sorted(matrix.keys(), key=int):
|
||||||
|
osl_map = matrix[isl_str]
|
||||||
|
low = concurrency_cfg[isl_str]["low"]
|
||||||
|
high = concurrency_cfg[isl_str]["high"]
|
||||||
|
concurrencies = sample_concurrency(low, high, target_samples)
|
||||||
|
|
||||||
|
for osl_str in sorted(osl_map.keys(), key=int):
|
||||||
|
mark = osl_map[osl_str]
|
||||||
|
if mode == "Y" and mark != "Y":
|
||||||
|
continue
|
||||||
|
if mode == "Y+P" and mark not in ("Y", "P"):
|
||||||
|
continue
|
||||||
|
# mode == "all" keeps everything, including N.
|
||||||
|
|
||||||
|
for conc in concurrencies:
|
||||||
|
scenarios.append(
|
||||||
|
{
|
||||||
|
"mark": mark,
|
||||||
|
"input_len": int(isl_str),
|
||||||
|
"output_len": int(osl_str),
|
||||||
|
"concurrency": conc,
|
||||||
|
"num_prompts": conc * 5,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
return scenarios
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--matrix", type=Path, default=Path("matrix.json"))
|
||||||
|
parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None,
|
||||||
|
help="Scenario selection mode. Defaults to matrix.mode.")
|
||||||
|
parser.add_argument("--target-samples", type=int, default=0,
|
||||||
|
help="Target number of concurrency samples. 0 = heuristic (6-8).")
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
with open(args.matrix, "r", encoding="utf-8") as f:
|
||||||
|
data = json.load(f)
|
||||||
|
|
||||||
|
mode = args.mode if args.mode else data.get("mode", "Y+P")
|
||||||
|
|
||||||
|
target_samples = args.target_samples
|
||||||
|
if target_samples <= 0:
|
||||||
|
env_samples = os.getenv("CONCURRENCY_SAMPLES", "0")
|
||||||
|
try:
|
||||||
|
target_samples = int(env_samples)
|
||||||
|
except ValueError:
|
||||||
|
target_samples = 0
|
||||||
|
if target_samples <= 0:
|
||||||
|
target_samples = 7
|
||||||
|
|
||||||
|
scenarios = generate_scenarios(args.matrix, mode, target_samples)
|
||||||
|
|
||||||
|
print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts")
|
||||||
|
for s in scenarios:
|
||||||
|
print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
128
experiments/h20/glm_h20_vllm_tp_dp_matrix/matrix.json
Normal file
128
experiments/h20/glm_h20_vllm_tp_dp_matrix/matrix.json
Normal file
@ -0,0 +1,128 @@
|
|||||||
|
{
|
||||||
|
"comment": "Only mandatory (Y) combinations are tested; 1M ISL is excluded per user request.",
|
||||||
|
"mode": "Y",
|
||||||
|
"matrix": {
|
||||||
|
"1024": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "Y"
|
||||||
|
},
|
||||||
|
"4096": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "Y"
|
||||||
|
},
|
||||||
|
"8192": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "Y"
|
||||||
|
},
|
||||||
|
"16384": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "P"
|
||||||
|
},
|
||||||
|
"32768": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "Y",
|
||||||
|
"4096": "P"
|
||||||
|
},
|
||||||
|
"65536": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "Y",
|
||||||
|
"2048": "P",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"131072": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "Y",
|
||||||
|
"1024": "P",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"262144": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "Y",
|
||||||
|
"512": "P",
|
||||||
|
"1024": "N",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"524288": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "P",
|
||||||
|
"512": "N",
|
||||||
|
"1024": "N",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
},
|
||||||
|
"1048576": {
|
||||||
|
"128": "Y",
|
||||||
|
"256": "P",
|
||||||
|
"512": "N",
|
||||||
|
"1024": "N",
|
||||||
|
"2048": "N",
|
||||||
|
"4096": "N"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"concurrency": {
|
||||||
|
"1024": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 128
|
||||||
|
},
|
||||||
|
"4096": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 64
|
||||||
|
},
|
||||||
|
"8192": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 64
|
||||||
|
},
|
||||||
|
"16384": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 32
|
||||||
|
},
|
||||||
|
"32768": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 16
|
||||||
|
},
|
||||||
|
"65536": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 8
|
||||||
|
},
|
||||||
|
"131072": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 4
|
||||||
|
},
|
||||||
|
"262144": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 2
|
||||||
|
},
|
||||||
|
"524288": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 2
|
||||||
|
},
|
||||||
|
"1048576": {
|
||||||
|
"low": 1,
|
||||||
|
"high": 2
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
162
experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh
Executable file
162
experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh
Executable file
@ -0,0 +1,162 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
|
||||||
|
ENGINE="vllm"
|
||||||
|
ENGINE_PORT="$VLLM_PORT"
|
||||||
|
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
|
||||||
|
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||||
|
PYTHON="${VENV_CLIENT}/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="$(command -v python3)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
|
||||||
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||||||
|
|
||||||
|
engine_is_healthy() {
|
||||||
|
curl --fail --silent --show-error --max-time 5 \
|
||||||
|
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_stop_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
||||||
|
|
||||||
|
if [[ -f "$pid_file" ]]; then
|
||||||
|
local pid
|
||||||
|
pid="$(cat "$pid_file")"
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 5
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -f "$pid_file"
|
||||||
|
fi
|
||||||
|
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
sleep 2
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_build_server_args() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local -a args=(
|
||||||
|
vllm serve "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||||
|
--block-size "$BLOCK_SIZE"
|
||||||
|
--tensor-parallel-size "$tp"
|
||||||
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
|
--no-enable-flashinfer-autotune
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
)
|
||||||
|
if (( dp > 1 )); then
|
||||||
|
args+=(--data-parallel-size "$dp")
|
||||||
|
fi
|
||||||
|
printf '%q ' "${args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_start_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
log "starting vllm server tp=${tp} dp=${dp}"
|
||||||
|
bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
if ! engine_is_healthy; then
|
||||||
|
log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||||
|
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||||
|
-name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
|
||||||
|
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||||
|
)"
|
||||||
|
log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_detect_oom() {
|
||||||
|
local detail_log="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
local -a logs=("$detail_log" "$outer_log")
|
||||||
|
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||||
|
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||||
|
fi
|
||||||
|
grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_run_bench() {
|
||||||
|
local isl="$1"
|
||||||
|
local osl="$2"
|
||||||
|
local concurrency="$3"
|
||||||
|
local num_prompts="$4"
|
||||||
|
local output_file="$5"
|
||||||
|
local warmup_requests
|
||||||
|
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||||
|
local -a bench_args=(
|
||||||
|
--backend vllm
|
||||||
|
--host 127.0.0.1
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
--dataset-name "$BENCH_DATASET_NAME"
|
||||||
|
--random-input-len "$isl"
|
||||||
|
--random-output-len "$osl"
|
||||||
|
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||||
|
--num-prompts "$num_prompts"
|
||||||
|
--max-concurrency "$concurrency"
|
||||||
|
--request-rate 10000
|
||||||
|
--warmup-requests "$warmup_requests"
|
||||||
|
--output-file "$output_file"
|
||||||
|
--output-details
|
||||||
|
--disable-tqdm
|
||||||
|
)
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||||
|
else
|
||||||
|
bench_args+=(--tokenize-prompt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||||
|
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||||
|
fi
|
||||||
|
docker run --rm \
|
||||||
|
--network host \
|
||||||
|
"${volume_args[@]}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
-e HF_HUB_OFFLINE=1 \
|
||||||
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
|
"$DOCKER_CLIENT_IMAGE" \
|
||||||
|
python -m sglang.bench_serving "${bench_args[@]}"
|
||||||
|
else
|
||||||
|
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
export -f engine_run_bench
|
||||||
|
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
|
||||||
|
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
|
||||||
|
|
||||||
|
adaptive_main "$@"
|
||||||
171
experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
171
experiments/h20/glm_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh
Executable file
@ -0,0 +1,171 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
|
||||||
|
ENGINE="vllm"
|
||||||
|
ENGINE_PORT="$VLLM_PORT"
|
||||||
|
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
|
||||||
|
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||||
|
PYTHON="${VENV_CLIENT}/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="$(command -v python3)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:glm52}"
|
||||||
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||||||
|
engine_is_healthy() {
|
||||||
|
curl --fail --silent --show-error --max-time 5 \
|
||||||
|
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_stop_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
||||||
|
|
||||||
|
if [[ -f "$pid_file" ]]; then
|
||||||
|
local pid
|
||||||
|
pid="$(cat "$pid_file")"
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 5
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -f "$pid_file"
|
||||||
|
fi
|
||||||
|
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
sleep 2
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_build_server_args() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local -a args=(
|
||||||
|
vllm serve "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||||
|
--block-size "$BLOCK_SIZE"
|
||||||
|
--tensor-parallel-size "$tp"
|
||||||
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
|
--max-model-len "$MAX_MODEL_LEN"
|
||||||
|
--no-enable-flashinfer-autotune
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
)
|
||||||
|
if (( dp > 1 )); then
|
||||||
|
args+=(--data-parallel-size "$dp")
|
||||||
|
fi
|
||||||
|
printf '%q ' "${args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_start_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
log "starting vllm server tp=${tp} dp=${dp}"
|
||||||
|
bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
if ! engine_is_healthy; then
|
||||||
|
log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||||
|
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||||
|
-name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
|
||||||
|
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||||
|
)"
|
||||||
|
log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_detect_oom() {
|
||||||
|
local detail_log="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
local -a logs=("$detail_log" "$outer_log")
|
||||||
|
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||||
|
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||||
|
fi
|
||||||
|
grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_run_bench() {
|
||||||
|
local isl="$1"
|
||||||
|
local osl="$2"
|
||||||
|
local concurrency="$3"
|
||||||
|
local num_prompts="$4"
|
||||||
|
local output_file="$5"
|
||||||
|
local warmup_requests
|
||||||
|
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||||
|
local -a bench_args=(
|
||||||
|
--backend vllm
|
||||||
|
--host 127.0.0.1
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
--dataset-name "$BENCH_DATASET_NAME"
|
||||||
|
--random-input-len "$isl"
|
||||||
|
--random-output-len "$osl"
|
||||||
|
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||||
|
--num-prompts "$num_prompts"
|
||||||
|
--max-concurrency "$concurrency"
|
||||||
|
--request-rate 10000
|
||||||
|
--warmup-requests "$warmup_requests"
|
||||||
|
--output-file "$output_file"
|
||||||
|
--output-details
|
||||||
|
--disable-tqdm
|
||||||
|
)
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||||
|
else
|
||||||
|
bench_args+=(--tokenize-prompt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||||
|
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||||
|
fi
|
||||||
|
docker run --rm \
|
||||||
|
--network host \
|
||||||
|
"${volume_args[@]}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
-e HF_HUB_OFFLINE=1 \
|
||||||
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
|
"$DOCKER_CLIENT_IMAGE" \
|
||||||
|
python -m sglang.bench_serving "${bench_args[@]}"
|
||||||
|
else
|
||||||
|
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
export -f engine_run_bench
|
||||||
|
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
|
||||||
|
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
|
||||||
|
|
||||||
|
export SEARCH_START_CONCURRENCY=16
|
||||||
|
export SEARCH_ADDEND=16
|
||||||
|
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
||||||
|
# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
|
||||||
|
export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
|
||||||
|
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
|
||||||
|
# shapes in this TP/DP group. Zero disables this rule.
|
||||||
|
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-12000}"
|
||||||
|
|
||||||
|
adaptive_main "$@"
|
||||||
546
experiments/h20/glm_h20_vllm_tp_dp_matrix/run_bench.sh
Executable file
546
experiments/h20/glm_h20_vllm_tp_dp_matrix/run_bench.sh
Executable file
@ -0,0 +1,546 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# TP×DP matrix benchmark for DeepSeek-V4-Flash on vLLM (Docker).
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
# Export variables used inside functions that are called via bash -c subshells.
|
||||||
|
export DATASET_PATH MODEL_PATH RESULT_BASE \
|
||||||
|
DOCKER_IMAGE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
|
||||||
|
|
||||||
|
RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}"
|
||||||
|
RESULT_BASE="${SCRIPT_DIR}/results"
|
||||||
|
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}"
|
||||||
|
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||||
|
SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}"
|
||||||
|
GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}"
|
||||||
|
DRY_RUN="${DRY_RUN:-0}"
|
||||||
|
GRID_LIMIT="${GRID_LIMIT:-0}"
|
||||||
|
|
||||||
|
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||||
|
PYTHON="${VENV_CLIENT}/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="$(command -v python3)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
|
||||||
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||||||
|
|
||||||
|
log_dir_global="${RESULT_BASE}/${RUN_ID}/logs"
|
||||||
|
mkdir -p "$log_dir_global"
|
||||||
|
log_init "${log_dir_global}/orchestrator.log"
|
||||||
|
|
||||||
|
log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}"
|
||||||
|
log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}"
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
is_server_healthy() {
|
||||||
|
curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
|
||||||
|
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
||||||
|
if [[ -f "$pid_file" ]]; then
|
||||||
|
local pid
|
||||||
|
pid="$(cat "$pid_file")"
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
log "stopping vllm server pid=${pid} (tp=${tp}, dp=${dp})"
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 5
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -f "$pid_file"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Fallback: remove any Docker container started by this experiment.
|
||||||
|
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
# Fallback: kill any vllm serve processes for this model.
|
||||||
|
pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true
|
||||||
|
pkill -9 -f "vllm serve.*${MODEL_PATH}" 2>/dev/null || true
|
||||||
|
sleep 2
|
||||||
|
}
|
||||||
|
|
||||||
|
build_server_args() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
|
||||||
|
local args=(
|
||||||
|
"vllm serve" "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||||
|
--block-size "$BLOCK_SIZE"
|
||||||
|
--tensor-parallel-size "$tp"
|
||||||
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
|
--no-enable-flashinfer-autotune
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$VLLM_PORT"
|
||||||
|
)
|
||||||
|
if [[ "$dp" -gt 1 ]]; then
|
||||||
|
args+=(
|
||||||
|
--data-parallel-size "$dp"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
printf '%s ' "${args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
start_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
|
||||||
|
log "starting vllm server tp=${tp} dp=${dp}"
|
||||||
|
bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" \
|
||||||
|
>> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1
|
||||||
|
|
||||||
|
if ! is_server_healthy; then
|
||||||
|
log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${VLLM_PORT}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
log "vllm server tp=${tp} dp=${dp} is healthy on port ${VLLM_PORT}"
|
||||||
|
}
|
||||||
|
|
||||||
|
restart_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
log "restarting vllm server tp=${tp} dp=${dp} after non-OOM failure"
|
||||||
|
stop_server "$tp" "$dp"
|
||||||
|
sleep 10
|
||||||
|
start_server "$tp" "$dp"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_bench_serving() {
|
||||||
|
# Run sglang.bench_serving either natively or inside the SGLang Docker image.
|
||||||
|
if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then
|
||||||
|
local vol_args=()
|
||||||
|
vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro")
|
||||||
|
if [[ -n "${DATASET_PATH:-}" && -f "${DATASET_PATH}" ]]; then
|
||||||
|
vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||||
|
fi
|
||||||
|
vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}")
|
||||||
|
docker run --rm \
|
||||||
|
--network host \
|
||||||
|
"${vol_args[@]}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
-e HF_HUB_OFFLINE=1 \
|
||||||
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
|
-e HF_DATASETS_OFFLINE=1 \
|
||||||
|
"${DOCKER_CLIENT_IMAGE}" \
|
||||||
|
python -m sglang.bench_serving "$@"
|
||||||
|
else
|
||||||
|
"$PYTHON" -m sglang.bench_serving "$@"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
export -f run_bench_serving
|
||||||
|
|
||||||
|
run_warmup() {
|
||||||
|
local input_len="$1"
|
||||||
|
local output_len="$2"
|
||||||
|
|
||||||
|
log "warming up (input=${input_len}, output=${output_len}, num=1)"
|
||||||
|
bash -c '
|
||||||
|
run_bench_serving \
|
||||||
|
--backend vllm \
|
||||||
|
--host 127.0.0.1 \
|
||||||
|
--port "'"$VLLM_PORT"'" \
|
||||||
|
--dataset-name random \
|
||||||
|
--dataset-path "'"$DATASET_PATH"'" \
|
||||||
|
--random-input-len "'"$input_len"'" \
|
||||||
|
--random-output-len "'"$output_len"'" \
|
||||||
|
--num-prompts 1 \
|
||||||
|
--max-concurrency 1 \
|
||||||
|
--request-rate 10000 \
|
||||||
|
--output-file /dev/null \
|
||||||
|
--output-details \
|
||||||
|
>> "'"${log_dir_global}/warmup.log"'" 2>&1
|
||||||
|
'
|
||||||
|
log "warmup completed"
|
||||||
|
}
|
||||||
|
|
||||||
|
scenario_already_completed() {
|
||||||
|
local output_file="$1"
|
||||||
|
local expected="$2"
|
||||||
|
[[ -s "$output_file" ]] || return 1
|
||||||
|
local completed
|
||||||
|
completed="$("$PYTHON" -c "
|
||||||
|
import json, sys
|
||||||
|
path = sys.argv[1]
|
||||||
|
try:
|
||||||
|
with open(path, 'r', encoding='utf-8') as f:
|
||||||
|
for line in f:
|
||||||
|
line = line.strip()
|
||||||
|
if line:
|
||||||
|
data = json.loads(line)
|
||||||
|
print(data.get('completed', 0))
|
||||||
|
break
|
||||||
|
except Exception:
|
||||||
|
print(0)
|
||||||
|
" "$output_file")"
|
||||||
|
[[ "${completed:-0}" -ge "$expected" ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
scenario_already_processed() {
|
||||||
|
local result_root="$1"
|
||||||
|
local scenario_name="$2"
|
||||||
|
local json_path="${result_root}/results.json"
|
||||||
|
[[ -f "$json_path" ]] || return 1
|
||||||
|
"$PYTHON" -c "
|
||||||
|
import json, sys
|
||||||
|
path, name = sys.argv[1], sys.argv[2]
|
||||||
|
try:
|
||||||
|
with open(path, 'r', encoding='utf-8') as f:
|
||||||
|
data = json.load(f)
|
||||||
|
for s in data.get('scenarios', []):
|
||||||
|
if s.get('name') == name:
|
||||||
|
if s.get('status') or s.get('metrics', {}).get('success', 0) > 0:
|
||||||
|
sys.exit(0)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
sys.exit(1)
|
||||||
|
" "$json_path" "$scenario_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
detect_oom() {
|
||||||
|
local detail_log="$1"
|
||||||
|
local server_outer_log="$2"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||||
|
if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
start_gpu_monitor() {
|
||||||
|
local csv_path="$1"
|
||||||
|
mkdir -p "$(dirname "$csv_path")"
|
||||||
|
nvidia-smi \
|
||||||
|
--query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \
|
||||||
|
--format=csv \
|
||||||
|
-l "$GPU_MEM_SAMPLE_INTERVAL_S" \
|
||||||
|
> "$csv_path" 2>/dev/null &
|
||||||
|
echo $!
|
||||||
|
}
|
||||||
|
|
||||||
|
stop_gpu_monitor() {
|
||||||
|
local pid="$1"
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 1
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
append_scenario_record() {
|
||||||
|
local result_root="$1"
|
||||||
|
local json_path="$result_root/results.json"
|
||||||
|
shift
|
||||||
|
local scenario_json
|
||||||
|
scenario_json="$("$PYTHON" -c "
|
||||||
|
import json, sys
|
||||||
|
pairs = [a.split('=', 1) for a in sys.argv[1:]]
|
||||||
|
d = {}
|
||||||
|
for k, v in pairs:
|
||||||
|
try:
|
||||||
|
d[k] = json.loads(v)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
d[k] = v
|
||||||
|
print(json.dumps(d, ensure_ascii=False))
|
||||||
|
" "$@")"
|
||||||
|
PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json"
|
||||||
|
}
|
||||||
|
|
||||||
|
record_skipped_csv() {
|
||||||
|
local csv_path="$1"
|
||||||
|
shift
|
||||||
|
# Args: key=value
|
||||||
|
local row
|
||||||
|
row="$("$PYTHON" -c "
|
||||||
|
import csv, json, sys, io
|
||||||
|
pairs = [a.split('=', 1) for a in sys.argv[1:]]
|
||||||
|
d = {}
|
||||||
|
for k, v in pairs:
|
||||||
|
try:
|
||||||
|
d[k] = json.loads(v)
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
d[k] = v
|
||||||
|
buf = io.StringIO()
|
||||||
|
writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','osl','concurrency','status','reason','detail_log'], extrasaction='ignore')
|
||||||
|
writer.writerow(d)
|
||||||
|
print(buf.getvalue().strip())
|
||||||
|
" "$@")"
|
||||||
|
echo "$row" >> "$csv_path"
|
||||||
|
}
|
||||||
|
|
||||||
|
skip_remaining_scenarios() {
|
||||||
|
local result_root="$1"
|
||||||
|
local scenario_tsv="$2"
|
||||||
|
local start_index="$3"
|
||||||
|
local status="$4"
|
||||||
|
local reason="$5"
|
||||||
|
local tp="$6"
|
||||||
|
local dp="$7"
|
||||||
|
local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
|
||||||
|
|
||||||
|
local i=0
|
||||||
|
tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do
|
||||||
|
if (( i < start_index )); then
|
||||||
|
i=$((i + 1))
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
i=$((i + 1))
|
||||||
|
local sname="c${conc}_i${isl}_o${osl}"
|
||||||
|
if scenario_already_processed "$result_root" "$sname"; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"${status}\"" \
|
||||||
|
"note=\"${reason}\""
|
||||||
|
record_skipped_csv "$skipped_csv" \
|
||||||
|
"engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=${status}" "reason=${reason}"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Per-configuration runner
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
run_parallel_config() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local config_label="tp${tp}_dp${dp}"
|
||||||
|
local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}"
|
||||||
|
local raw_dir="${result_root}/raw_outputs"
|
||||||
|
local gpu_log_dir="${result_root}/gpu_logs"
|
||||||
|
local phase_log_dir="${result_root}/logs"
|
||||||
|
mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir"
|
||||||
|
|
||||||
|
log "===== ${config_label} START ====="
|
||||||
|
|
||||||
|
# Generate scenario list for this config.
|
||||||
|
local scenario_tsv="${result_root}/scenarios.tsv"
|
||||||
|
"$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \
|
||||||
|
--matrix "$MATRIX_FILE" \
|
||||||
|
--mode "$MATRIX_MODE" \
|
||||||
|
> "$scenario_tsv"
|
||||||
|
local total_scenarios
|
||||||
|
total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)"
|
||||||
|
log "generated ${total_scenarios} scenarios for ${config_label}"
|
||||||
|
|
||||||
|
# Write metadata.
|
||||||
|
ensure_result_root "$result_root"
|
||||||
|
write_metadata_json \
|
||||||
|
"${result_root}/results.json" \
|
||||||
|
"${EXPERIMENT_NAME}_${config_label}" \
|
||||||
|
"$RUN_ID" \
|
||||||
|
"$MODEL_PATH" \
|
||||||
|
"vllm" \
|
||||||
|
"vllm" \
|
||||||
|
"$HARDWARE" \
|
||||||
|
"$ACCELERATOR" \
|
||||||
|
"$CHIP" \
|
||||||
|
"experiments/${EXPERIMENT_NAME}/run_bench.sh" \
|
||||||
|
"$DOCKER_IMAGE" \
|
||||||
|
"H20 vLLM TP×DP matrix for DeepSeek-V4-Flash"
|
||||||
|
|
||||||
|
local server_args_str
|
||||||
|
server_args_str="$(build_server_args "$tp" "$dp")"
|
||||||
|
jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \
|
||||||
|
'.config = {
|
||||||
|
"tp": ($tp | tonumber),
|
||||||
|
"dp": ($dp | tonumber),
|
||||||
|
"cuda_visible_devices": $cuda,
|
||||||
|
"backend": "vllm",
|
||||||
|
"server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_vllm_dp.sh",
|
||||||
|
"server_args": $args
|
||||||
|
}' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \
|
||||||
|
mv "${result_root}/results.json.tmp" "${result_root}/results.json"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
log "DRY_RUN: would start server with args: ${server_args_str}"
|
||||||
|
local line
|
||||||
|
tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do
|
||||||
|
log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}"
|
||||||
|
done
|
||||||
|
log "===== ${config_label} DONE (dry run) ====="
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Initialize skipped_after_oom.csv for this run.
|
||||||
|
local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv"
|
||||||
|
if [[ ! -f "$skipped_csv" ]]; then
|
||||||
|
echo "engine,tp,dp,mark,isl,osl,concurrency,status,reason,detail_log" > "$skipped_csv"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Start server once for this TP×DP config.
|
||||||
|
if ! start_server "$tp" "$dp"; then
|
||||||
|
log "ERROR: ${config_label} failed to start; skipping all scenarios"
|
||||||
|
skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp"
|
||||||
|
log "===== ${config_label} DONE ====="
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Warmup with a small prompt before the first scenario.
|
||||||
|
run_warmup 1024 128 || true
|
||||||
|
|
||||||
|
# Read scenarios into an array so we can skip remaining entries on failure.
|
||||||
|
local -a scenarios=()
|
||||||
|
while IFS= read -r line; do
|
||||||
|
scenarios+=("$line")
|
||||||
|
done < <(tail -n +2 "$scenario_tsv")
|
||||||
|
|
||||||
|
local i mark isl osl conc num
|
||||||
|
local output_file detail_log gpu_csv sname bench_rc
|
||||||
|
for (( i = 0; i < ${#scenarios[@]}; i++ )); do
|
||||||
|
IFS=$'\t' read -r mark isl osl conc num <<< "${scenarios[$i]}"
|
||||||
|
|
||||||
|
if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then
|
||||||
|
log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios"
|
||||||
|
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
sname="c${conc}_i${isl}_o${osl}"
|
||||||
|
output_file="${raw_dir}/vllm_main_${conc}_${isl}_${osl}.jsonl"
|
||||||
|
detail_log="${phase_log_dir}/vllm_${config_label}_${sname}.log"
|
||||||
|
gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${osl}.csv"
|
||||||
|
|
||||||
|
if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then
|
||||||
|
log "skipping already-processed ${config_label} scenario: ${sname}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}"
|
||||||
|
|
||||||
|
local gpu_pid
|
||||||
|
gpu_pid="$(start_gpu_monitor "$gpu_csv")"
|
||||||
|
|
||||||
|
bench_rc=0
|
||||||
|
timeout "$SCENARIO_TIMEOUT_S" bash -c '
|
||||||
|
run_bench_serving \
|
||||||
|
--backend vllm \
|
||||||
|
--host 127.0.0.1 \
|
||||||
|
--port "'"$VLLM_PORT"'" \
|
||||||
|
--dataset-name random \
|
||||||
|
--dataset-path "'"$DATASET_PATH"'" \
|
||||||
|
--random-input-len "'"$isl"'" \
|
||||||
|
--random-output-len "'"$osl"'" \
|
||||||
|
--num-prompts "'"$num"'" \
|
||||||
|
--max-concurrency "'"$conc"'" \
|
||||||
|
--request-rate 10000 \
|
||||||
|
--output-file "'"$output_file"'" \
|
||||||
|
--output-details \
|
||||||
|
> "'"$detail_log"'" 2>&1
|
||||||
|
' || bench_rc=$?
|
||||||
|
|
||||||
|
stop_gpu_monitor "$gpu_pid"
|
||||||
|
|
||||||
|
if [[ "$bench_rc" -eq 0 ]]; then
|
||||||
|
log "finished ${config_label} scenario: output=${output_file}"
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"completed\"" \
|
||||||
|
"note=\"benchmark finished successfully\""
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Failure handling.
|
||||||
|
if detect_oom "$detail_log" "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log"; then
|
||||||
|
log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config"
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"OOM\"" \
|
||||||
|
"note=\"detected CUDA out-of-memory\""
|
||||||
|
record_skipped_csv "$skipped_csv" \
|
||||||
|
"engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}"
|
||||||
|
stop_server "$tp" "$dp"
|
||||||
|
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}"
|
||||||
|
if [[ "$mark" == "P" ]]; then
|
||||||
|
log "optional (P) scenario failed; recording as skipped and continuing"
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"skipped_optional\"" \
|
||||||
|
"note=\"optional scenario failed (rc=${bench_rc})\""
|
||||||
|
record_skipped_csv "$skipped_csv" \
|
||||||
|
"engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Mandatory scenario failed but not OOM: try to restart the server.
|
||||||
|
if restart_server "$tp" "$dp"; then
|
||||||
|
run_warmup 1024 128 || true
|
||||||
|
log "resuming ${config_label} after server restart"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "ERROR: ${config_label} server restart failed; skipping remaining scenarios"
|
||||||
|
append_scenario_record "$result_root" \
|
||||||
|
"name=${sname}" \
|
||||||
|
"config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \
|
||||||
|
"status=\"FAILED\"" \
|
||||||
|
"note=\"scenario failed and server restart failed (rc=${bench_rc})\""
|
||||||
|
record_skipped_csv "$skipped_csv" \
|
||||||
|
"engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}"
|
||||||
|
skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp"
|
||||||
|
break
|
||||||
|
done
|
||||||
|
|
||||||
|
stop_server "$tp" "$dp"
|
||||||
|
|
||||||
|
# Parse results.
|
||||||
|
log "parsing ${config_label} results"
|
||||||
|
"$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend vllm \
|
||||||
|
>> "${phase_log_dir}/parse.log" 2>&1 || {
|
||||||
|
log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
log "===== ${config_label} DONE ====="
|
||||||
|
}
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Main
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
# Cleanup any leftovers.
|
||||||
|
for cfg in "${PARALLEL_CONFIGS[@]}"; do
|
||||||
|
read -r tp dp <<< "$cfg"
|
||||||
|
stop_server "$tp" "$dp"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Run each parallel configuration.
|
||||||
|
for cfg in "${PARALLEL_CONFIGS[@]}"; do
|
||||||
|
read -r tp dp <<< "$cfg"
|
||||||
|
run_parallel_config "$tp" "$dp"
|
||||||
|
done
|
||||||
|
|
||||||
|
# Generate cross-configuration comparison.
|
||||||
|
log "generating comparison report"
|
||||||
|
"$PYTHON" "${SCRIPT_DIR}/compare.py" \
|
||||||
|
--run-root "${RESULT_BASE}/${RUN_ID}" \
|
||||||
|
--output "${RESULT_BASE}/${RUN_ID}/comparison.md" \
|
||||||
|
>> "${log_dir_global}/compare.log" 2>&1 || {
|
||||||
|
log "WARNING: comparison script failed; see ${log_dir_global}/compare.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
log "all results saved to ${RESULT_BASE}/${RUN_ID}"
|
||||||
102
experiments/h20/glm_h20_vllm_tp_dp_matrix/start_vllm_docker.sh
Executable file
102
experiments/h20/glm_h20_vllm_tp_dp_matrix/start_vllm_docker.sh
Executable file
@ -0,0 +1,102 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Start vLLM server in Docker for a given TPxDP configuration.
|
||||||
|
# Usage: start_vllm_docker.sh <TP> <DP>
|
||||||
|
#
|
||||||
|
# Uses the vllm/vllm-openai image and the same argument set as the
|
||||||
|
# bare-metal start script. The container is removed automatically on stop.
|
||||||
|
set -e
|
||||||
|
|
||||||
|
TP="${1}"
|
||||||
|
DP="${2}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||||
|
|
||||||
|
IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:glm52}"
|
||||||
|
PORT="${VLLM_PORT:-30030}"
|
||||||
|
NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}"
|
||||||
|
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
|
||||||
|
|
||||||
|
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
||||||
|
rm -f "$PID_FILE"
|
||||||
|
|
||||||
|
# Clean up any stale container with the same name.
|
||||||
|
docker rm -f "$NAME" >/dev/null 2>&1 || true
|
||||||
|
|
||||||
|
SERVER_ARGS=(
|
||||||
|
"$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||||
|
--block-size "$BLOCK_SIZE"
|
||||||
|
--max-model-len "$MAX_MODEL_LEN"
|
||||||
|
--max-num-seqs "$MAX_NUM_SEQS"
|
||||||
|
--tensor-parallel-size "$TP"
|
||||||
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
|
--no-enable-flashinfer-autotune
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$PORT"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$DP" -gt 1 ]]; then
|
||||||
|
SERVER_ARGS+=(
|
||||||
|
--data-parallel-size "$DP"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
|
||||||
|
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
||||||
|
|
||||||
|
echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ==="
|
||||||
|
echo "Image: $IMAGE"
|
||||||
|
echo "Model: $MODEL_PATH"
|
||||||
|
echo "Container name: $NAME"
|
||||||
|
echo "Host port: $PORT"
|
||||||
|
echo "Command: vllm ${SERVER_ARGS_STR}"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
|
||||||
|
# Run docker in the foreground so that killing the host process stops the
|
||||||
|
# container (the --rm flag ensures cleanup). nohup lets us background it and
|
||||||
|
# capture the host PID in the same way as the bare-metal start script.
|
||||||
|
nohup docker run --rm \
|
||||||
|
--name "$NAME" \
|
||||||
|
--gpus all \
|
||||||
|
--ipc host \
|
||||||
|
--shm-size 16g \
|
||||||
|
--ulimit memlock=-1 \
|
||||||
|
-p "${PORT}:${PORT}" \
|
||||||
|
-v "${MODEL_PATH}:${MODEL_PATH}:ro" \
|
||||||
|
-v "${RUNTIME_BASE}/tmp:/tmp" \
|
||||||
|
-e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
-e HF_HUB_OFFLINE=1 \
|
||||||
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
|
"$IMAGE" \
|
||||||
|
"${SERVER_ARGS[@]}" \
|
||||||
|
> "$LOG" 2>&1 &
|
||||||
|
|
||||||
|
PID=$!
|
||||||
|
echo $PID > "$PID_FILE"
|
||||||
|
echo "PID: $PID"
|
||||||
|
echo "Waiting for health on port ${PORT}..."
|
||||||
|
|
||||||
|
for i in $(seq 1 240); do
|
||||||
|
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then
|
||||||
|
echo "vLLM server is ready at http://127.0.0.1:${PORT}"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if ! kill -0 $PID 2>/dev/null; then
|
||||||
|
echo "ERROR: Docker vLLM server exited early"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Waiting... ($i/240)"
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "ERROR: Docker vLLM server not healthy after 240 retries"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
85
experiments/h20/glm_h20_vllm_tp_dp_matrix/start_vllm_dp.sh
Executable file
85
experiments/h20/glm_h20_vllm_tp_dp_matrix/start_vllm_dp.sh
Executable file
@ -0,0 +1,85 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Start vLLM server for a given TP×DP configuration.
|
||||||
|
# Usage: start_vllm_dp.sh <TP> <DP>
|
||||||
|
#
|
||||||
|
# By default this delegates to the Docker start script because the experiment
|
||||||
|
# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the
|
||||||
|
# local VENV_VLLM environment instead.
|
||||||
|
set -e
|
||||||
|
|
||||||
|
TP="${1}"
|
||||||
|
DP="${2}"
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
|
||||||
|
if [[ "${USE_DOCKER:-1}" == "1" ]]; then
|
||||||
|
exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@"
|
||||||
|
fi
|
||||||
|
|
||||||
|
RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}"
|
||||||
|
mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp"
|
||||||
|
|
||||||
|
VENV="${VENV_VLLM}"
|
||||||
|
export PATH="$VENV/bin:$PATH"
|
||||||
|
export PYTHONUNBUFFERED=1
|
||||||
|
export TMPDIR="${RUNTIME_BASE}/tmp"
|
||||||
|
export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}"
|
||||||
|
|
||||||
|
LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log"
|
||||||
|
PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid"
|
||||||
|
|
||||||
|
rm -f "$PID_FILE"
|
||||||
|
|
||||||
|
SERVER_ARGS=(
|
||||||
|
vllm serve "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||||
|
--block-size "$BLOCK_SIZE"
|
||||||
|
--tensor-parallel-size "$TP"
|
||||||
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
|
--no-enable-flashinfer-autotune
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$VLLM_PORT"
|
||||||
|
)
|
||||||
|
|
||||||
|
if [[ "$DP" -gt 1 ]]; then
|
||||||
|
SERVER_ARGS+=(
|
||||||
|
--data-parallel-size "$DP"
|
||||||
|
)
|
||||||
|
fi
|
||||||
|
|
||||||
|
SERVER_ARGS_STR="${SERVER_ARGS[*]}"
|
||||||
|
|
||||||
|
echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ==="
|
||||||
|
echo "Model: $MODEL_PATH"
|
||||||
|
echo "Port: $VLLM_PORT"
|
||||||
|
echo "Command: $SERVER_ARGS_STR"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
|
||||||
|
nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 &
|
||||||
|
|
||||||
|
PID=$!
|
||||||
|
echo $PID > "$PID_FILE"
|
||||||
|
echo "PID: $PID"
|
||||||
|
echo "Waiting for health on port ${VLLM_PORT}..."
|
||||||
|
|
||||||
|
for i in $(seq 1 240); do
|
||||||
|
if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then
|
||||||
|
echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}"
|
||||||
|
echo "Log: $LOG"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
if ! kill -0 $PID 2>/dev/null; then
|
||||||
|
echo "ERROR: vLLM server exited early"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "Waiting... ($i/240)"
|
||||||
|
sleep 5
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "ERROR: vLLM server not healthy after 240 retries"
|
||||||
|
tail -200 "$LOG"
|
||||||
|
exit 1
|
||||||
Loading…
x
Reference in New Issue
Block a user