From d13f61f7b8fbeb988be76bdb3cac694106f5e50b Mon Sep 17 00:00:00 2001 From: qqtang Date: Thu, 23 Jul 2026 02:06:47 +0000 Subject: [PATCH] =?UTF-8?q?Add=20vLLM=20TP=C3=97DP=20matrix=20benchmarking?= =?UTF-8?q?=20scripts=20and=20Docker=20support?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Introduced `run_adaptive_concurrency_add16.sh` for adaptive concurrency testing with vLLM. - Added `run_bench.sh` to execute TP×DP matrix benchmarks for Hy3-FP8 on vLLM. - Created `start_vllm_docker.sh` to launch the vLLM server in a Docker container for specified TP and DP configurations. - Implemented `start_vllm_dp.sh` to manage starting the vLLM server, defaulting to Docker unless specified otherwise. - Enhanced logging and health check mechanisms for server readiness. - Included GPU monitoring and OOM detection in benchmarking scripts. --- .../adaptive_config.env | 58 ++ .../adaptive_points.jsonl | 123 ++++ .../adaptive_shapes.jsonl | 31 + .../run_manifest.json | 27 + .../adaptive_20260722-092437/shapes.tsv | 36 ++ .../h20/hy3_h20_vllm_tp_dp_matrix/compare.py | 163 ++++++ .../h20/hy3_h20_vllm_tp_dp_matrix/config.env | 76 +++ .../generate_scenarios.py | 98 ++++ .../h20/hy3_h20_vllm_tp_dp_matrix/matrix.json | 92 +++ .../results/20260722-084347/comparison.md | 34 ++ .../20260722-084347/tp2_dp4/results.json | 27 + .../20260722-084347/tp2_dp4/scenarios.tsv | 79 +++ .../20260722-084347/tp4_dp2/results.json | 27 + .../20260722-084347/tp4_dp2/scenarios.tsv | 79 +++ .../20260722-084347/tp8_dp1/results.json | 27 + .../20260722-084347/tp8_dp1/scenarios.tsv | 79 +++ .../results/20260722-085021/comparison.md | 33 ++ .../20260722-085021/tp4_dp2/results.json | 27 + .../20260722-085021/tp4_dp2/scenarios.tsv | 71 +++ .../20260722-085021/tp8_dp1/results.json | 27 + .../20260722-085021/tp8_dp1/scenarios.tsv | 71 +++ .../results/20260722-085141/comparison.md | 33 ++ .../20260722-085141/tp4_dp2/results.json | 27 + .../20260722-085141/tp4_dp2/scenarios.tsv | 71 +++ .../20260722-085141/tp8_dp1/results.json | 27 + .../20260722-085141/tp8_dp1/scenarios.tsv | 71 +++ .../run_adaptive_concurrency.sh | 164 ++++++ .../run_adaptive_concurrency_add16.sh | 173 ++++++ .../hy3_h20_vllm_tp_dp_matrix/run_bench.sh | 548 ++++++++++++++++++ .../start_vllm_docker.sh | 102 ++++ .../start_vllm_dp.sh | 87 +++ 31 files changed, 2588 insertions(+) create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_config.env create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/adaptive_points.jsonl create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/adaptive_shapes.jsonl create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/run_manifest.json create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/shapes.tsv create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/compare.py create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/config.env create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/generate_scenarios.py create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/matrix.json create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/comparison.md create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp2_dp4/results.json create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp2_dp4/scenarios.tsv create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp4_dp2/results.json create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp4_dp2/scenarios.tsv create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp8_dp1/results.json create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp8_dp1/scenarios.tsv create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/comparison.md create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp4_dp2/results.json create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp4_dp2/scenarios.tsv create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp8_dp1/results.json create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp8_dp1/scenarios.tsv create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/comparison.md create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp4_dp2/results.json create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp4_dp2/scenarios.tsv create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp8_dp1/results.json create mode 100644 experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp8_dp1/scenarios.tsv create mode 100755 experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh create mode 100755 experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh create mode 100755 experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_bench.sh create mode 100755 experiments/h20/hy3_h20_vllm_tp_dp_matrix/start_vllm_docker.sh create mode 100755 experiments/h20/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_config.env b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_config.env new file mode 100644 index 0000000..3b05c7c --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_config.env @@ -0,0 +1,58 @@ +# Adaptive concurrency search settings. +# +# For each fixed (TP, DP, ISL, OSL), probe: +# C = start, start * multiplier, ... up to max +# and stop after Total TPS has less than TPS_MIN_GAIN_PCT meaningful growth for +# PLATEAU_PATIENCE consecutive points. + +SEARCH_START_CONCURRENCY="${SEARCH_START_CONCURRENCY:-1}" +SEARCH_MAX_CONCURRENCY="${SEARCH_MAX_CONCURRENCY:-256}" + +# At the add16 initial probe, restart and retry C=8 then C=1 after an OOM. +ENABLE_INITIAL_OOM_BACKOFF="${ENABLE_INITIAL_OOM_BACKOFF:-1}" +SEARCH_MULTIPLIER="${SEARCH_MULTIPLIER:-2}" +NUM_PROMPTS_MULTIPLIER="${NUM_PROMPTS_MULTIPLIER:-5}" + +# A gain below 2% is treated as throughput saturation. Two consecutive +# low-gain points prevent one noisy measurement from stopping the search. +TPS_MIN_GAIN_PCT="${TPS_MIN_GAIN_PCT:-2.0}" +PLATEAU_PATIENCE="${PLATEAU_PATIENCE:-2}" + +# TTFT SLO early-stop settings. +# When ttft_p95_ms exceeds TTFT_SLO_MS, stop searching the current (ISL, OSL) +# shape and move on to the next scenario. +TTFT_SLO_MS="${TTFT_SLO_MS:-4000}" +ENABLE_TTFT_SLO_STOP="${ENABLE_TTFT_SLO_STOP:-1}" + +# Keep the same random workload semantics as the fixed matrix baseline. +# DATASET_PATH must contain at least SEARCH_MAX_CONCURRENCY times +# NUM_PROMPTS_MULTIPLIER valid two-turn conversations. Set this explicitly to +# random-ids to use generated token IDs without a ShareGPT seed dataset. +BENCH_DATASET_NAME="${BENCH_DATASET_NAME:-random}" +# SGLang interprets 0.0 as Uniform[1, requested_len]. Use 1.0 for fixed +# ISL/OSL points; lower values intentionally benchmark a length distribution. +RANDOM_RANGE_RATIO="${RANDOM_RANGE_RATIO:-1.0}" +# Before each measured point, warm up with the same concurrency so lazy kernel +# compilation and CUDA graph capture are excluded from TTFT/TPS. 0 means no +# cap; set a positive cap only when very high-concurrency warmup is impractical. +BENCH_WARMUP_MAX_REQUESTS="${BENCH_WARMUP_MAX_REQUESTS:-0}" + +# Reject a point if the completed request count or actual token lengths do not +# match the requested workload. +INPUT_LENGTH_TOLERANCE_PCT="${INPUT_LENGTH_TOLERANCE_PCT:-5.0}" +OUTPUT_LENGTH_TOLERANCE_PCT="${OUTPUT_LENGTH_TOLERANCE_PCT:-10.0}" + +MAX_POINT_RETRIES="${MAX_POINT_RETRIES:-1}" +SERVER_RESTART_COOLDOWN_S="${SERVER_RESTART_COOLDOWN_S:-10}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Optional space-separated filters, useful for smoke tests: +# TP_LIST="8" ISL_LIST="1024" OSL_LIST="128" +TP_LIST="${TP_LIST:-}" +ISL_LIST="${ISL_LIST:-}" +OSL_LIST="${OSL_LIST:-}" + +DRY_RUN="${DRY_RUN:-0}" +# Counts ISL/OSL shapes per TP/DP config, not individual concurrency probes. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/adaptive_points.jsonl b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/adaptive_points.jsonl new file mode 100644 index 0000000..9f6c9bb --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/adaptive_points.jsonl @@ -0,0 +1,123 @@ +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":55.658951326971874,"request_tps":1.4373249601853826,"input_tps":1471.8207592298318,"output_tps":183.97759490372897,"total_tps":1655.798354133561,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":998.023018008098,"ttft_p95_ms":1525.8213487919418,"ttft_p99_ms":1620.738287908025,"tpot_p50_ms":79.17198305170412,"tpot_p95_ms":82.81305485027366,"tpot_p99_ms":85.49516478243582,"e2e_p50_ms":11058.11054550577,"e2e_p95_ms":11551.822056132369,"e2e_p99_ms":11606.824755155249,"itl_p50_ms":76.19416801026091,"itl_p95_ms":79.62533283280209,"itl_p99_ms":80.91577647137456,"validation_errors":[],"timestamp":"2026-07-22T09:33:30+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":68.47190907795448,"request_tps":2.3367246824948587,"input_tps":2392.8060748747353,"output_tps":299.1007593593419,"total_tps":2691.906834234077,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1497.1878519863822,"ttft_p95_ms":2237.6658564666263,"ttft_p99_ms":2513.0041722976603,"tpot_p50_ms":95.29230881099201,"tpot_p95_ms":102.31239204734973,"tpot_p99_ms":104.82103100200513,"e2e_p50_ms":13684.24536200473,"e2e_p95_ms":14067.33377816854,"e2e_p99_ms":14172.20837556175,"itl_p50_ms":90.84106399677694,"itl_p95_ms":93.14442635513842,"itl_p99_ms":100.91778859030462,"validation_errors":[],"timestamp":"2026-07-22T09:35:23+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":128,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":62.574557,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":72.95438530400861,"request_tps":3.289726847809007,"input_tps":3368.680292156423,"output_tps":421.0850365195529,"total_tps":3789.765328675976,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1825.7132159778848,"ttft_p95_ms":2657.4594282952603,"ttft_p99_ms":2670.6660944398027,"tpot_p50_ms":100.8457303034347,"tpot_p95_ms":108.2334446668988,"tpot_p99_ms":112.06474051562384,"e2e_p50_ms":14621.375515009277,"e2e_p95_ms":14941.062037023949,"e2e_p99_ms":14959.779972407268,"itl_p50_ms":94.7329520713538,"itl_p95_ms":98.66021520574576,"itl_p99_ms":186.59311065799554,"validation_errors":[],"timestamp":"2026-07-22T09:37:16+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":128,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":40.783674,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c48_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c48_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":80.81526879698504,"request_tps":3.959647783933847,"input_tps":4054.6793307482594,"output_tps":506.8349163435324,"total_tps":4561.514247091792,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":2070.25937049184,"ttft_p95_ms":3243.854808190372,"ttft_p99_ms":3343.9458154083695,"tpot_p50_ms":109.67821817740607,"tpot_p95_ms":125.018606107044,"tpot_p99_ms":130.03312890856034,"e2e_p50_ms":16275.687272485811,"e2e_p95_ms":17288.60468767234,"e2e_p99_ms":17295.486344917445,"itl_p50_ms":106.18017299566418,"itl_p95_ms":109.09209708916023,"itl_p99_ms":197.55648438120272,"validation_errors":[],"timestamp":"2026-07-22T09:39:19+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":128,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":20.364029,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c64_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c64_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":85.67964237404522,"request_tps":4.6685535667124975,"input_tps":4780.5988523135975,"output_tps":597.5748565391997,"total_tps":5378.173708852797,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":2569.543256482575,"ttft_p95_ms":3587.0666204369627,"ttft_p99_ms":4497.193116999697,"tpot_p50_ms":116.55307746094498,"tpot_p95_ms":132.84249960057122,"tpot_p99_ms":137.3420862551764,"e2e_p50_ms":16547.02737199841,"e2e_p95_ms":18478.82525275345,"e2e_p99_ms":18487.921478349017,"itl_p50_ms":108.79829805344343,"itl_p95_ms":111.8351846234873,"itl_p99_ms":206.21089754160465,"validation_errors":[],"timestamp":"2026-07-22T09:41:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":128,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":17.903254,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c80_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":88.84744871407747,"request_tps":5.402518664826289,"input_tps":5532.17911278212,"output_tps":691.522389097765,"total_tps":6223.701501879885,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":2774.8617280158214,"ttft_p95_ms":4068.1753366719936,"ttft_p99_ms":4864.421937608859,"tpot_p50_ms":118.0297743463082,"tpot_p95_ms":135.6934698358156,"tpot_p99_ms":143.44382293156613,"e2e_p50_ms":17567.455728480127,"e2e_p95_ms":19313.52732470841,"e2e_p99_ms":19320.5547705316,"itl_p50_ms":110.15663808211684,"itl_p95_ms":113.24595627374946,"itl_p99_ms":615.6030731741395,"validation_errors":[],"timestamp":"2026-07-22T09:43:41+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":128,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":15.721467,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c96_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":97.44120113493409,"request_tps":0.8210079418994234,"input_tps":840.7121325050095,"output_tps":210.1780331262524,"total_tps":1050.8901656312619,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":540.9864374669269,"ttft_p95_ms":875.4830864956602,"ttft_p99_ms":882.0958340133075,"tpot_p50_ms":76.66140260978365,"tpot_p95_ms":79.60002403646963,"tpot_p99_ms":79.82127059782015,"e2e_p50_ms":20119.740098074544,"e2e_p95_ms":20908.32041829708,"e2e_p99_ms":20916.718422693666,"itl_p50_ms":76.24241709709167,"itl_p95_ms":79.29171475116163,"itl_p99_ms":80.23532548453659,"validation_errors":[],"timestamp":"2026-07-22T09:45:57+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":120.44145261694212,"request_tps":1.3284462826006576,"input_tps":1360.3289933830733,"output_tps":340.08224834576833,"total_tps":1700.4112417288418,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":803.5511740017682,"ttft_p95_ms":1177.9417676036246,"ttft_p99_ms":1183.5946559393778,"tpot_p50_ms":91.33895425281177,"tpot_p95_ms":93.43013852010664,"tpot_p99_ms":93.51173143914225,"e2e_p50_ms":24023.008311982267,"e2e_p95_ms":24536.617009580368,"e2e_p99_ms":24541.204523463966,"itl_p50_ms":90.25161346653476,"itl_p95_ms":92.79853900079615,"itl_p99_ms":94.47223590104844,"validation_errors":[],"timestamp":"2026-07-22T09:48:35+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":61.806752,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":128.96827153104823,"request_tps":1.8609228235040867,"input_tps":1905.5849712681847,"output_tps":476.3962428170462,"total_tps":2381.981214085231,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1099.0791025105864,"ttft_p95_ms":1750.188358669402,"ttft_p99_ms":1761.037156197708,"tpot_p50_ms":96.69847268812065,"tpot_p95_ms":99.72455385921742,"tpot_p99_ms":99.7429436136804,"e2e_p50_ms":25794.568624987733,"e2e_p95_ms":26083.441730274353,"e2e_p99_ms":26089.25869425875,"itl_p50_ms":95.2005804865621,"itl_p95_ms":97.748573042918,"itl_p99_ms":99.82526833657174,"validation_errors":[],"timestamp":"2026-07-22T09:51:24+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":40.082655,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c48_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c48_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":144.76527097797953,"request_tps":2.2104749145855274,"input_tps":2263.52631253558,"output_tps":565.881578133895,"total_tps":2829.407890669475,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1352.596461540088,"ttft_p95_ms":2718.4078295133077,"ttft_p99_ms":2856.5018487500492,"tpot_p50_ms":107.26805311951823,"tpot_p95_ms":111.6913128800818,"tpot_p99_ms":115.78271392668032,"e2e_p50_ms":28657.141319010407,"e2e_p95_ms":30146.80186479236,"e2e_p99_ms":30213.001879800577,"itl_p50_ms":106.0353799839504,"itl_p95_ms":108.62895884783939,"itl_p99_ms":110.72380699333736,"validation_errors":[],"timestamp":"2026-07-22T09:54:32+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":18.783804,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c64_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c64_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":149.1890631710412,"request_tps":2.681161685031904,"input_tps":2745.5095654726697,"output_tps":686.3773913681674,"total_tps":3431.8869568408372,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1694.5647604879923,"ttft_p95_ms":2911.3155239028856,"ttft_p99_ms":3225.802575610578,"tpot_p50_ms":110.32132656272391,"tpot_p95_ms":117.60770156641728,"tpot_p99_ms":118.94406361973334,"e2e_p50_ms":29541.509745526128,"e2e_p95_ms":31168.568641890306,"e2e_p99_ms":31179.1814301291,"itl_p50_ms":108.67713746847585,"itl_p95_ms":111.47720502340235,"itl_p99_ms":113.85996548226103,"validation_errors":[],"timestamp":"2026-07-22T09:57:44+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":21.293468,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c80_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":152.16150209505577,"request_tps":3.1545429914338157,"input_tps":3230.2520232282272,"output_tps":807.5630058070568,"total_tps":4037.8150290352837,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1748.0963669950143,"ttft_p95_ms":2952.049240260385,"ttft_p99_ms":3736.494510208722,"tpot_p50_ms":111.70047747254299,"tpot_p95_ms":119.54554093383051,"tpot_p99_ms":121.22193075639798,"e2e_p50_ms":30059.81994397007,"e2e_p95_ms":31470.661072456278,"e2e_p99_ms":31480.93801497249,"itl_p50_ms":109.49944658204913,"itl_p95_ms":111.9222282723058,"itl_p99_ms":115.6350452802144,"validation_errors":[],"timestamp":"2026-07-22T10:00:55+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":17.655828,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c96_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":560,"failed":0,"duration_s":157.33171246910933,"request_tps":3.5593587027786975,"input_tps":3644.783311645386,"output_tps":911.1958279113466,"total_tps":4555.979139556733,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":2301.001067040488,"ttft_p95_ms":4415.9571976808365,"ttft_p99_ms":5087.647556982702,"tpot_p50_ms":114.59260311950108,"tpot_p95_ms":122.99181459247882,"tpot_p99_ms":129.24640555873367,"e2e_p50_ms":31532.437274989206,"e2e_p95_ms":33747.342583478894,"e2e_p99_ms":33783.66813468048,"itl_p50_ms":110.81579409074038,"itl_p95_ms":114.1926016076468,"itl_p99_ms":118.3506605355069,"validation_errors":[],"timestamp":"2026-07-22T10:04:20+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":12.832785,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c112_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c112_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":198.5715282439487,"request_tps":0.40287749561819636,"input_tps":412.54655551303307,"output_tps":206.27327775651653,"total_tps":618.8198332695496,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":477.3354135104455,"ttft_p95_ms":672.3686925950459,"ttft_p99_ms":689.9595684988888,"tpot_p50_ms":76.43424688645688,"tpot_p95_ms":78.44968007595202,"tpot_p99_ms":78.55757054432665,"e2e_p50_ms":39511.08387147542,"e2e_p95_ms":40585.86324537173,"e2e_p99_ms":40638.91178958351,"itl_p50_ms":76.41267991857603,"itl_p95_ms":78.94930833135732,"itl_p99_ms":79.79717813432217,"validation_errors":[],"timestamp":"2026-07-22T10:08:16+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":237.011167305056,"request_tps":0.6750736761448237,"input_tps":691.2754443722995,"output_tps":345.63772218614974,"total_tps":1036.9131665584491,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":697.4429484689608,"ttft_p95_ms":946.5491847891825,"ttft_p99_ms":979.824201061856,"tpot_p50_ms":91.42735195307446,"tpot_p95_ms":92.55752247917,"tpot_p99_ms":92.6773990701689,"e2e_p50_ms":47363.53933351347,"e2e_p95_ms":47932.62797475327,"e2e_p99_ms":47937.403975153575,"itl_p50_ms":91.04861645027995,"itl_p95_ms":93.13828542362899,"itl_p99_ms":94.28897259989753,"validation_errors":[],"timestamp":"2026-07-22T10:12:58+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":67.563014,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":249.4713263490703,"request_tps":0.9620344089732475,"input_tps":985.1232347886055,"output_tps":492.56161739430274,"total_tps":1477.6848521829083,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":880.3551605087705,"ttft_p95_ms":1216.2732776720077,"ttft_p99_ms":1287.569403712405,"tpot_p50_ms":95.6341791663359,"tpot_p95_ms":97.39841856101143,"tpot_p99_ms":97.4127374586117,"e2e_p50_ms":49747.12255253689,"e2e_p95_ms":50490.25874370709,"e2e_p99_ms":50512.514556034,"itl_p50_ms":95.39004298858345,"itl_p95_ms":97.66021484974772,"itl_p99_ms":99.15664625354111,"validation_errors":[],"timestamp":"2026-07-22T10:17:52+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":42.508061,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c48_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c48_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":277.13105527102016,"request_tps":1.154688346591313,"input_tps":1182.4008669095044,"output_tps":591.2004334547522,"total_tps":1773.6013003642565,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":1045.1401854516007,"ttft_p95_ms":1373.2652821927331,"ttft_p99_ms":1380.6284993689042,"tpot_p50_ms":106.70872350688568,"tpot_p95_ms":108.55739284666718,"tpot_p99_ms":108.58494479266368,"e2e_p50_ms":55387.255592504516,"e2e_p95_ms":56143.28455313225,"e2e_p99_ms":56162.48124839272,"itl_p50_ms":105.89231451740488,"itl_p95_ms":108.46393702668138,"itl_p99_ms":109.76766810636035,"validation_errors":[],"timestamp":"2026-07-22T10:23:15+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":20.025681,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c64_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c64_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":286.47854159399867,"request_tps":1.3962651365591128,"input_tps":1429.7754998365315,"output_tps":714.8877499182657,"total_tps":2144.663249754797,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":1345.2447130111977,"ttft_p95_ms":2099.1558350040577,"ttft_p99_ms":2107.1739520551637,"tpot_p50_ms":109.43118745411898,"tpot_p95_ms":112.62472195365356,"tpot_p99_ms":113.47417497177473,"e2e_p50_ms":57085.345196013805,"e2e_p95_ms":58682.12491324521,"e2e_p99_ms":58690.845183586935,"itl_p50_ms":108.7271414580755,"itl_p95_ms":111.7575934913475,"itl_p99_ms":113.15335653489456,"validation_errors":[],"timestamp":"2026-07-22T10:28:48+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":20.921385,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c80_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":290.9049088830361,"request_tps":1.6500237202700254,"input_tps":1689.624289556506,"output_tps":844.812144778253,"total_tps":2534.436434334759,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":1556.998702522833,"ttft_p95_ms":2408.2528069033287,"ttft_p99_ms":2794.552087507909,"tpot_p50_ms":110.76965863206343,"tpot_p95_ms":113.5476250002692,"tpot_p99_ms":114.07733408343265,"e2e_p50_ms":58294.65898300987,"e2e_p95_ms":58887.260264821816,"e2e_p99_ms":58896.98926171288,"itl_p50_ms":110.07151403464377,"itl_p95_ms":112.56422526203096,"itl_p99_ms":115.06696843309328,"validation_errors":[],"timestamp":"2026-07-22T10:34:27+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":18.174097,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c96_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":560,"failed":0,"duration_s":296.28418022196274,"request_tps":1.89007728856962,"input_tps":1935.4391434952909,"output_tps":967.7195717476454,"total_tps":2903.1587152429365,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":1960.9920189832337,"ttft_p95_ms":3681.2162980262656,"ttft_p99_ms":4033.88597704703,"tpot_p50_ms":112.25082001369994,"tpot_p95_ms":115.15699230649813,"tpot_p99_ms":117.72208107635151,"e2e_p50_ms":58808.65262955194,"e2e_p95_ms":60801.88988603186,"e2e_p99_ms":60811.82590764831,"itl_p50_ms":110.86707899812609,"itl_p95_ms":113.64379501901567,"itl_p99_ms":116.62393808364868,"validation_errors":[],"timestamp":"2026-07-22T10:40:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":14.548492,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c112_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c112_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":640,"failed":0,"duration_s":303.0636334869778,"request_tps":2.111767725597139,"input_tps":2162.4501510114706,"output_tps":1081.2250755057353,"total_tps":3243.6752265172054,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":2030.6891299551353,"ttft_p95_ms":5390.535102447029,"ttft_p99_ms":5408.510366003029,"tpot_p50_ms":113.9165531241776,"tpot_p95_ms":117.93363074093844,"tpot_p99_ms":121.9429818276428,"e2e_p50_ms":60122.86441703327,"e2e_p95_ms":63174.861903337296,"e2e_p99_ms":63199.78691241937,"itl_p50_ms":111.9914895389229,"itl_p95_ms":115.50216606119646,"itl_p99_ms":119.55715498421337,"validation_errors":[],"timestamp":"2026-07-22T10:45:57+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"concurrency":128,"num_prompts":640,"warmup_requests":128,"attempt":1,"gain_pct":11.729173,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c128_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c128_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":393.34612702101003,"request_tps":0.20338321519999844,"input_tps":208.2644123647984,"output_tps":208.2644123647984,"total_tps":416.5288247295968,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":423.64722199272364,"ttft_p95_ms":697.9574323806446,"ttft_p99_ms":716.8945163022722,"tpot_p50_ms":75.83411786268726,"tpot_p95_ms":78.24663562111931,"tpot_p99_ms":78.27533739331845,"e2e_p50_ms":78001.40782305971,"e2e_p95_ms":80541.5996691212,"e2e_p99_ms":80547.54214013694,"itl_p50_ms":76.16359897656366,"itl_p95_ms":78.84043626254424,"itl_p99_ms":79.82159599196166,"validation_errors":[],"timestamp":"2026-07-22T10:53:15+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":466.0983221239876,"request_tps":0.3432752112706343,"input_tps":351.51381634112954,"output_tps":351.51381634112954,"total_tps":703.0276326822591,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":657.0419929921627,"ttft_p95_ms":939.1321116301692,"ttft_p99_ms":1068.0073573172558,"tpot_p50_ms":90.0607265322401,"tpot_p95_ms":92.15446616982028,"tpot_p99_ms":92.45358161842869,"e2e_p50_ms":92929.94681448909,"e2e_p95_ms":95142.54621355794,"e2e_p99_ms":95152.15506395092,"itl_p50_ms":90.1291009504348,"itl_p95_ms":92.99481101334095,"itl_p99_ms":93.87714767362921,"validation_errors":[],"timestamp":"2026-07-22T11:01:45+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":68.782469,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":489.033361623995,"request_tps":0.4907640640364527,"input_tps":502.54240157332754,"output_tps":502.54240157332754,"total_tps":1005.0848031466551,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":779.1502340114675,"ttft_p95_ms":1306.8270372925326,"ttft_p99_ms":1321.3517123262864,"tpot_p50_ms":94.31840192233207,"tpot_p95_ms":97.54460981638145,"tpot_p99_ms":97.54586658175495,"e2e_p50_ms":97250.092606002,"e2e_p95_ms":100574.5975138736,"e2e_p99_ms":100592.73460182361,"itl_p50_ms":94.26938847173005,"itl_p95_ms":97.80129413702525,"itl_p99_ms":99.07463278970681,"validation_errors":[],"timestamp":"2026-07-22T11:10:32+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":42.965192,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c48_i1024_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c48_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":549.5010828940431,"request_tps":0.5823464410928261,"input_tps":596.3227556790539,"output_tps":596.3227556790539,"total_tps":1192.6455113581078,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":1172.1398730296642,"ttft_p95_ms":1869.9064777581953,"ttft_p99_ms":1883.5479418607429,"tpot_p50_ms":105.84371568173542,"tpot_p95_ms":109.7309860802221,"tpot_p99_ms":110.36839662221936,"e2e_p50_ms":109758.45538749127,"e2e_p95_ms":113488.53046541335,"e2e_p99_ms":113498.93703315989,"itl_p50_ms":105.7323370478116,"itl_p95_ms":109.72609696327709,"itl_p99_ms":110.85189913748764,"validation_errors":[],"timestamp":"2026-07-22T11:20:23+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":18.661182,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c64_i1024_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c64_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":566.0479497109773,"request_tps":0.7066539154575846,"input_tps":723.6136094285666,"output_tps":723.6136094285666,"total_tps":1447.2272188571333,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":1397.739619424101,"ttft_p95_ms":2431.6686553589534,"ttft_p99_ms":2443.1632214901038,"tpot_p50_ms":109.51810902442496,"tpot_p95_ms":111.40187210688445,"tpot_p99_ms":112.03331967997151,"e2e_p50_ms":113330.38794650929,"e2e_p95_ms":115353.13531512511,"e2e_p99_ms":115360.51145990612,"itl_p50_ms":109.07510691322386,"itl_p95_ms":111.33035912644118,"itl_p99_ms":112.63196206651628,"validation_errors":[],"timestamp":"2026-07-22T11:30:30+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":21.345966,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c80_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":576.15679446992,"request_tps":0.8331065512151308,"input_tps":853.1011084442939,"output_tps":853.1011084442939,"total_tps":1706.2022168885878,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":1880.2635120227933,"ttft_p95_ms":2609.9704524327535,"ttft_p99_ms":2625.516084743431,"tpot_p50_ms":110.8733203953641,"tpot_p95_ms":113.41721869777876,"tpot_p99_ms":113.96313429875768,"e2e_p50_ms":114933.01162001444,"e2e_p95_ms":117314.95794957737,"e2e_p99_ms":117333.24453663663,"itl_p50_ms":110.24937499314547,"itl_p95_ms":112.93484661728144,"itl_p99_ms":114.88543374929577,"validation_errors":[],"timestamp":"2026-07-22T11:40:49+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":17.894564,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c96_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":560,"failed":0,"duration_s":588.3658873480745,"request_tps":0.9517886948274189,"input_tps":974.631623503277,"output_tps":974.631623503277,"total_tps":1949.263247006554,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":2436.142202524934,"ttft_p95_ms":3749.6355051873247,"ttft_p99_ms":4167.46369121247,"tpot_p50_ms":112.87590066418926,"tpot_p95_ms":115.52360198736025,"tpot_p99_ms":116.45022596083291,"e2e_p50_ms":118061.27154297428,"e2e_p95_ms":119799.01035545627,"e2e_p99_ms":119866.76122553181,"itl_p50_ms":111.78700707387179,"itl_p95_ms":114.84688988421112,"itl_p99_ms":116.2341053551063,"validation_errors":[],"timestamp":"2026-07-22T11:51:22+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":14.245734,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c112_i1024_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c112_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":640,"failed":0,"duration_s":595.4203029889613,"request_tps":1.0748709722984793,"input_tps":1100.6678756336428,"output_tps":1100.6678756336428,"total_tps":2201.3357512672856,"mean_input_tokens":1024.0,"mean_output_tokens":1024.0,"ttft_p50_ms":2600.2102604834363,"ttft_p95_ms":4463.483180751791,"ttft_p99_ms":5327.597370402655,"tpot_p50_ms":113.90593018375277,"tpot_p95_ms":116.32653706593149,"tpot_p99_ms":117.56499459549492,"e2e_p50_ms":119196.08418550342,"e2e_p95_ms":120925.74072624557,"e2e_p99_ms":120940.96202988061,"itl_p50_ms":112.34435392543674,"itl_p95_ms":115.09799619670957,"itl_p99_ms":118.18784192204477,"validation_errors":[],"timestamp":"2026-07-22T12:01:59+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":128,"num_prompts":640,"warmup_requests":128,"attempt":1,"gain_pct":12.931681,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c128_i1024_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c128_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":784.3171388680348,"request_tps":0.10199955609214398,"input_tps":104.44754543835543,"output_tps":208.89509087671087,"total_tps":313.34263631506633,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":647.4065009970218,"ttft_p95_ms":1013.166410929989,"ttft_p99_ms":1069.228255158523,"tpot_p50_ms":76.04851655884539,"tpot_p95_ms":77.50511485444038,"tpot_p99_ms":77.53782623779264,"e2e_p50_ms":156350.01297696726,"e2e_p95_ms":159372.4140102975,"e2e_p99_ms":159426.10840613954,"itl_p50_ms":76.20464696083218,"itl_p95_ms":78.21892402134836,"itl_p99_ms":79.03465735260397,"validation_errors":[],"timestamp":"2026-07-22T12:15:47+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":934.3937273499323,"request_tps":0.17123402621053735,"input_tps":175.34364283959025,"output_tps":350.6872856791805,"total_tps":526.0309285187708,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":977.8251114767045,"ttft_p95_ms":1290.5446410994023,"ttft_p99_ms":1426.3530042255297,"tpot_p50_ms":90.97675536076744,"tpot_p95_ms":92.0002269944053,"tpot_p99_ms":92.18749054829064,"e2e_p50_ms":187165.13015399687,"e2e_p95_ms":189489.15034800302,"e2e_p99_ms":189494.93819196243,"itl_p50_ms":90.83973400993273,"itl_p95_ms":92.95165599905886,"itl_p99_ms":93.82738563814198,"validation_errors":[],"timestamp":"2026-07-22T12:32:09+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":67.877227,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":978.7830082740402,"request_tps":0.24520245853389874,"input_tps":251.0873175387123,"output_tps":502.1746350774246,"total_tps":753.2619526161369,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":1300.4444009857252,"ttft_p95_ms":1844.4834600784816,"ttft_p99_ms":1987.816799179418,"tpot_p50_ms":94.85032523109369,"tpot_p95_ms":97.37002397148652,"tpot_p99_ms":97.37218030496251,"e2e_p50_ms":195477.61286405148,"e2e_p95_ms":200129.3688242964,"e2e_p99_ms":200147.38655565656,"itl_p50_ms":94.7448950028047,"itl_p95_ms":97.93993306811899,"itl_p99_ms":99.25523796118796,"validation_errors":[],"timestamp":"2026-07-22T12:49:08+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":43.197274,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c48_i1024_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c48_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":1078.9417045660084,"request_tps":0.2965869227649479,"input_tps":303.70500891130666,"output_tps":607.4100178226133,"total_tps":911.11502673392,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":2010.0743054645136,"ttft_p95_ms":2417.9770519258454,"ttft_p99_ms":2453.3929923188407,"tpot_p50_ms":107.24304267341789,"tpot_p95_ms":108.16247001702978,"tpot_p99_ms":108.3081485144646,"e2e_p50_ms":221053.33935900126,"e2e_p95_ms":222603.57491533505,"e2e_p99_ms":222615.94897501986,"itl_p50_ms":106.66407051030546,"itl_p95_ms":108.71329526416957,"itl_p99_ms":109.7187170421239,"validation_errors":[],"timestamp":"2026-07-22T13:07:50+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":20.955934,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c64_i1024_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c64_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":1131.3380788279464,"request_tps":0.3535636318494605,"input_tps":362.04915901384754,"output_tps":724.0983180276951,"total_tps":1086.1474770415427,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":2084.4462675158866,"ttft_p95_ms":3403.142456099158,"ttft_p99_ms":3415.090579481327,"tpot_p50_ms":109.0542897332674,"tpot_p95_ms":112.14133604847325,"tpot_p99_ms":112.77302049991856,"e2e_p50_ms":225279.02198198717,"e2e_p95_ms":231759.82493066695,"e2e_p99_ms":231766.82265097857,"itl_p50_ms":109.14539702935144,"itl_p95_ms":112.30930815800093,"itl_p99_ms":113.30504634068348,"validation_errors":[],"timestamp":"2026-07-22T13:27:32+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":19.210796,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c80_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":1151.3901135979686,"request_tps":0.41688737321189284,"input_tps":426.89267016897827,"output_tps":853.7853403379565,"total_tps":1280.6780105069347,"mean_input_tokens":1024.0,"mean_output_tokens":2048.0,"ttft_p50_ms":2609.664688061457,"ttft_p95_ms":4145.358256303007,"ttft_p99_ms":4921.978658180451,"tpot_p50_ms":111.07689807986891,"tpot_p95_ms":113.34880685358846,"tpot_p99_ms":113.69751149809086,"e2e_p50_ms":229926.48969194852,"e2e_p95_ms":233467.8463754768,"e2e_p99_ms":233592.9404158576,"itl_p50_ms":110.69605499505997,"itl_p95_ms":113.1438875803724,"itl_p99_ms":114.60410898085684,"validation_errors":[],"timestamp":"2026-07-22T13:47:31+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":17.91014,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c96_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1568.687286911998,"request_tps":0.05099805465847951,"input_tps":52.222007970283016,"output_tps":208.88803188113206,"total_tps":261.1100398514151,"mean_input_tokens":1024.0,"mean_output_tokens":4096.0,"ttft_p50_ms":689.4311624928378,"ttft_p95_ms":1059.8639070638455,"ttft_p99_ms":1133.0788673192728,"tpot_p50_ms":76.46017711879959,"tpot_p95_ms":77.62713979114635,"tpot_p99_ms":77.64527561319468,"e2e_p50_ms":313720.1397625613,"e2e_p95_ms":318662.498210941,"e2e_p99_ms":318716.93921104074,"itl_p50_ms":76.4597415109165,"itl_p95_ms":78.77037590369582,"itl_p99_ms":79.73776805913076,"validation_errors":[],"timestamp":"2026-07-22T14:14:22+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o4096_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i1024_o4096_a1.log"} +{"timestamp":"2026-07-22T15:21:04+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":2,"status":"FAILED","completed":0,"failed":160,"error_type":"benchmark rc=124","validation_errors":[],"gain_pct":null,"plateau_streak":0,"raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o4096_a2.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i1024_o4096_a2.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":70.1714427509578,"request_tps":1.1400649162070713,"input_tps":4669.705896784164,"output_tps":145.92830927450512,"total_tps":4815.634206058669,"mean_input_tokens":4096.0,"mean_output_tokens":128.0,"ttft_p50_ms":2719.8881005169824,"ttft_p95_ms":4153.263504902134,"ttft_p99_ms":4371.949667572044,"tpot_p50_ms":87.16765722474376,"tpot_p95_ms":102.97594817788854,"tpot_p99_ms":107.35202158980921,"e2e_p50_ms":14058.65088448627,"e2e_p95_ms":14950.036883819848,"e2e_p99_ms":15057.886625736719,"itl_p50_ms":75.79882303252816,"itl_p95_ms":78.27955649700016,"itl_p99_ms":850.8944046963006,"validation_errors":[],"timestamp":"2026-07-22T15:29:19+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i4096_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":50.10760599502828,"request_tps":0.7982820014184839,"input_tps":3269.76307781011,"output_tps":102.18009618156594,"total_tps":3371.9431739916763,"mean_input_tokens":4096.0,"mean_output_tokens":128.0,"ttft_p50_ms":1483.0816420144401,"ttft_p95_ms":2610.20162794739,"ttft_p99_ms":2642.093107792316,"tpot_p50_ms":66.78857510647273,"tpot_p95_ms":79.36093136416125,"tpot_p99_ms":83.13325850967698,"e2e_p50_ms":10297.572901064996,"e2e_p95_ms":11193.540374934673,"e2e_p99_ms":11198.173419741215,"itl_p50_ms":61.21738493675366,"itl_p95_ms":71.81573032867163,"itl_p99_ms":72.50719220028259,"validation_errors":[],"timestamp":"2026-07-22T15:30:51+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i4096_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i4096_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":111.64253633795306,"request_tps":0.7165727564432256,"input_tps":2935.082010391452,"output_tps":183.44262564946575,"total_tps":3118.524636040918,"mean_input_tokens":4096.0,"mean_output_tokens":256.0,"ttft_p50_ms":1862.3742060735822,"ttft_p95_ms":3421.6164269135334,"ttft_p99_ms":3873.057030111085,"tpot_p50_ms":78.03079821374816,"tpot_p95_ms":85.53662629348372,"tpot_p99_ms":87.28448097116095,"e2e_p50_ms":22468.99163600756,"e2e_p95_ms":23171.188393374905,"e2e_p99_ms":23408.368235799713,"itl_p50_ms":75.12174302246422,"itl_p95_ms":79.07615164294839,"itl_p99_ms":80.38425606675446,"validation_errors":[],"timestamp":"2026-07-22T15:33:22+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i4096_o256_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":144.92976135702338,"request_tps":1.1039830501469758,"input_tps":4521.914573402013,"output_tps":282.6196608376258,"total_tps":4804.53423423964,"mean_input_tokens":4096.0,"mean_output_tokens":256.0,"ttft_p50_ms":3704.925291473046,"ttft_p95_ms":5902.076709311223,"ttft_p99_ms":6816.154367570997,"tpot_p50_ms":99.63794201384208,"tpot_p95_ms":111.11449596152512,"tpot_p99_ms":115.6742643184695,"e2e_p50_ms":29396.56875305809,"e2e_p95_ms":31882.640700414777,"e2e_p99_ms":32629.044190520646,"itl_p50_ms":89.29755998542532,"itl_p95_ms":92.08525097346865,"itl_p99_ms":485.9807970758994,"validation_errors":[],"timestamp":"2026-07-22T15:36:24+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":256,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":54.064335,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i4096_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":204.09347839094698,"request_tps":0.3919772480272872,"input_tps":1605.5388079197683,"output_tps":200.69235098997103,"total_tps":1806.2311589097392,"mean_input_tokens":4096.0,"mean_output_tokens":512.0,"ttft_p50_ms":1614.55061705783,"ttft_p95_ms":2678.689720894908,"ttft_p99_ms":3133.8617387670092,"tpot_p50_ms":76.32985176020094,"tpot_p95_ms":80.06816529908077,"tpot_p99_ms":81.68059098313987,"e2e_p50_ms":40768.65813345648,"e2e_p95_ms":42686.032499722205,"e2e_p99_ms":43690.48365992727,"itl_p50_ms":75.28371352236718,"itl_p95_ms":78.16215605707839,"itl_p99_ms":79.32297390536404,"validation_errors":[],"timestamp":"2026-07-22T15:40:27+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":253.6633088820381,"request_tps":0.6307573637873081,"input_tps":2583.582162072814,"output_tps":322.94777025910173,"total_tps":2906.5299323319155,"mean_input_tokens":4096.0,"mean_output_tokens":512.0,"ttft_p50_ms":2571.81734603364,"ttft_p95_ms":5121.034596726531,"ttft_p99_ms":5482.0641031314135,"tpot_p50_ms":93.02792283856728,"tpot_p95_ms":99.59929229189513,"tpot_p99_ms":101.65720929981505,"e2e_p50_ms":49885.97165804822,"e2e_p95_ms":52690.3599616955,"e2e_p99_ms":52768.585482526105,"itl_p50_ms":89.64612550335005,"itl_p95_ms":92.22582574002445,"itl_p99_ms":93.8876695476938,"validation_errors":[],"timestamp":"2026-07-22T15:45:20+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":60.91683,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":392.5261165959528,"request_tps":0.20380809484416573,"input_tps":834.7979564817028,"output_tps":208.6994891204257,"total_tps":1043.4974456021287,"mean_input_tokens":4096.0,"mean_output_tokens":1024.0,"ttft_p50_ms":894.4826324586757,"ttft_p95_ms":1770.5690847535152,"ttft_p99_ms":1921.1237181560136,"tpot_p50_ms":75.71208073703194,"tpot_p95_ms":78.07329096818864,"tpot_p99_ms":78.53492406551145,"e2e_p50_ms":78724.40412145806,"e2e_p95_ms":80713.97086211364,"e2e_p99_ms":80881.92751282942,"itl_p50_ms":75.16288291662931,"itl_p95_ms":77.88682021200657,"itl_p99_ms":78.81514808163047,"validation_errors":[],"timestamp":"2026-07-22T15:52:36+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":481.449953553034,"request_tps":0.33232945359994776,"input_tps":1361.221441945386,"output_tps":340.3053604863465,"total_tps":1701.5268024317324,"mean_input_tokens":4096.0,"mean_output_tokens":1024.0,"ttft_p50_ms":2111.089213518426,"ttft_p95_ms":4176.813643838977,"ttft_p99_ms":4458.734513003146,"tpot_p50_ms":91.85031863049633,"tpot_p95_ms":94.21061297402038,"tpot_p99_ms":94.71221014331887,"e2e_p50_ms":96010.46120253159,"e2e_p95_ms":99033.49485744839,"e2e_p99_ms":99112.23364639329,"itl_p50_ms":90.1532219722867,"itl_p95_ms":92.39473997149616,"itl_p99_ms":93.4872226556763,"validation_errors":[],"timestamp":"2026-07-22T16:01:22+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":63.059987,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":808.2887427610112,"request_tps":0.0989745319559075,"input_tps":405.3996828913971,"output_tps":202.69984144569855,"total_tps":608.0995243370957,"mean_input_tokens":4096.0,"mean_output_tokens":2048.0,"ttft_p50_ms":1022.5364145589992,"ttft_p95_ms":2005.4678575659634,"ttft_p99_ms":2140.9124239021908,"tpot_p50_ms":79.21184836077546,"tpot_p95_ms":82.06618750060073,"tpot_p99_ms":82.14563771427342,"e2e_p50_ms":163231.2090964988,"e2e_p95_ms":169187.75821728632,"e2e_p99_ms":169197.71439048694,"itl_p50_ms":78.84798053419217,"itl_p95_ms":82.50283435336314,"itl_p99_ms":83.4025753417518,"validation_errors":[],"timestamp":"2026-07-22T16:15:36+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":906.7116708529647,"request_tps":0.17646182920473968,"input_tps":722.7876524226137,"output_tps":361.3938262113069,"total_tps":1084.1814786339205,"mean_input_tokens":4096.0,"mean_output_tokens":2048.0,"ttft_p50_ms":2042.5662884954363,"ttft_p95_ms":3785.110361117404,"ttft_p99_ms":4255.727169085293,"tpot_p50_ms":89.95643447069216,"tpot_p95_ms":92.22286094482986,"tpot_p99_ms":92.23329009812447,"e2e_p50_ms":186119.39821543638,"e2e_p95_ms":189912.50659273937,"e2e_p99_ms":189922.54467678024,"itl_p50_ms":89.55569704994559,"itl_p95_ms":92.51312862616032,"itl_p99_ms":93.6052941577509,"validation_errors":[],"timestamp":"2026-07-22T16:31:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":78.290138,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":1015.6513869259506,"request_tps":0.23630155296336733,"input_tps":967.8911609379526,"output_tps":483.9455804689763,"total_tps":1451.8367414069287,"mean_input_tokens":4096.0,"mean_output_tokens":2048.0,"ttft_p50_ms":2689.30905952584,"ttft_p95_ms":7299.072919989703,"ttft_p99_ms":7852.952230469091,"tpot_p50_ms":96.19561621788925,"tpot_p95_ms":106.10464789299431,"tpot_p99_ms":106.27637753282794,"e2e_p50_ms":199284.25612102728,"e2e_p95_ms":218408.41324480134,"e2e_p99_ms":219166.1641972768,"itl_p50_ms":94.6597830625251,"itl_p95_ms":106.21352911111899,"itl_p99_ms":107.68367234850301,"validation_errors":[],"timestamp":"2026-07-22T16:49:09+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":33.91086,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c48_i4096_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c48_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1587.1689622419653,"request_tps":0.05040421146277678,"input_tps":206.4556501515337,"output_tps":206.4556501515337,"total_tps":412.9113003030674,"mean_input_tokens":4096.0,"mean_output_tokens":4096.0,"ttft_p50_ms":860.7760275481269,"ttft_p95_ms":1392.4716420646291,"ttft_p99_ms":1432.3574046452993,"tpot_p50_ms":75.8284482509222,"tpot_p95_ms":81.49119488273489,"tpot_p99_ms":81.60051827410109,"e2e_p50_ms":311360.3704459383,"e2e_p95_ms":334503.08132686187,"e2e_p99_ms":334559.00491850567,"itl_p50_ms":76.02121704258025,"itl_p95_ms":82.34823623206466,"itl_p99_ms":83.32733014365658,"validation_errors":[],"timestamp":"2026-07-22T17:16:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o4096_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i4096_o4096_a1.log"} +{"timestamp":"2026-07-22T18:22:58+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":2,"status":"FAILED","completed":0,"failed":160,"error_type":"benchmark rc=124","validation_errors":[],"gain_pct":null,"plateau_streak":0,"raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o4096_a2.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i4096_o4096_a2.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":96.36851366201881,"request_tps":0.8301466626389399,"input_tps":6800.5614603381955,"output_tps":106.2587728177843,"total_tps":6906.82023315598,"mean_input_tokens":8192.0,"mean_output_tokens":128.0,"ttft_p50_ms":4616.379614977632,"ttft_p95_ms":6779.2717946053,"ttft_p99_ms":7796.650803452354,"tpot_p50_ms":122.99874923641127,"tpot_p95_ms":143.55263539647692,"tpot_p99_ms":151.70017837647595,"e2e_p50_ms":19991.168014414143,"e2e_p95_ms":22055.42305885465,"e2e_p99_ms":23253.154755790012,"itl_p50_ms":75.7559819612652,"itl_p95_ms":78.74409896321592,"itl_p99_ms":1221.9013671623545,"validation_errors":[],"timestamp":"2026-07-22T18:31:37+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i8192_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i8192_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":59.59401975211222,"request_tps":0.6712082884555252,"input_tps":5498.538299027662,"output_tps":85.91466092230722,"total_tps":5584.452959949969,"mean_input_tokens":8192.0,"mean_output_tokens":128.0,"ttft_p50_ms":2586.9538324768655,"ttft_p95_ms":4640.047149494057,"ttft_p99_ms":5291.139317123452,"tpot_p50_ms":72.68566230731818,"tpot_p95_ms":87.58289683912697,"tpot_p99_ms":92.43291318685175,"e2e_p50_ms":12007.312975998502,"e2e_p95_ms":13999.434245546581,"e2e_p99_ms":14765.589517641347,"itl_p50_ms":59.36828750418499,"itl_p95_ms":61.42948223860003,"itl_p99_ms":931.8136646668428,"validation_errors":[],"timestamp":"2026-07-22T18:33:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i8192_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i8192_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":20.49409785901662,"request_tps":0.24397268103217293,"input_tps":1998.6242030155606,"output_tps":31.228503172118135,"total_tps":2029.852706187679,"mean_input_tokens":8192.0,"mean_output_tokens":128.0,"ttft_p50_ms":1061.1285719787702,"ttft_p95_ms":1073.2807493768632,"ttft_p99_ms":1075.0835434533656,"tpot_p50_ms":25.102876449429143,"tpot_p95_ms":25.711215033533243,"tpot_p99_ms":25.81016031742888,"e2e_p50_ms":4263.599551049992,"e2e_p95_ms":4325.1825956162065,"e2e_p99_ms":4337.494589546695,"itl_p50_ms":25.253435946069658,"itl_p95_ms":25.983121409080923,"itl_p99_ms":26.27969579072669,"validation_errors":[],"timestamp":"2026-07-22T18:34:13+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i8192_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i8192_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":135.56818615505472,"request_tps":0.5901089501079614,"input_tps":4834.17251928442,"output_tps":151.0678912276381,"total_tps":4985.240410512058,"mean_input_tokens":8192.0,"mean_output_tokens":256.0,"ttft_p50_ms":3712.155805027578,"ttft_p95_ms":5529.782033787342,"ttft_p99_ms":6875.880905248922,"tpot_p50_ms":92.58666171158171,"tpot_p95_ms":103.35217525267187,"tpot_p99_ms":107.05739523677666,"e2e_p50_ms":27222.904000489507,"e2e_p95_ms":29542.051456909394,"e2e_p99_ms":30851.14542481722,"itl_p50_ms":76.23058051103726,"itl_p95_ms":78.69554529315792,"itl_p99_ms":943.9475631946698,"validation_errors":[],"timestamp":"2026-07-22T18:37:09+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i8192_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i8192_o256_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":90.88856101292185,"request_tps":0.44009938714194297,"input_tps":3605.294179466797,"output_tps":112.6654431083374,"total_tps":3717.9596225751343,"mean_input_tokens":8192.0,"mean_output_tokens":256.0,"ttft_p50_ms":1921.5318015194498,"ttft_p95_ms":3342.204241425497,"ttft_p99_ms":3696.0857983806636,"tpot_p50_ms":63.9850913629155,"tpot_p95_ms":71.86538405136625,"tpot_p99_ms":71.91616144865368,"e2e_p50_ms":18390.8772150171,"e2e_p95_ms":19590.47735191998,"e2e_p99_ms":20168.15899345558,"itl_p50_ms":60.10080047417432,"itl_p95_ms":64.33249299880117,"itl_p99_ms":65.72977091534995,"validation_errors":[],"timestamp":"2026-07-22T18:39:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i8192_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i8192_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":217.70777697500307,"request_tps":0.36746505389738787,"input_tps":3010.2737215274014,"output_tps":188.1421075954626,"total_tps":3198.415829122864,"mean_input_tokens":8192.0,"mean_output_tokens":512.0,"ttft_p50_ms":2588.536791037768,"ttft_p95_ms":4166.588377655717,"ttft_p99_ms":4562.831237260252,"tpot_p50_ms":79.99444148722405,"tpot_p95_ms":86.02221064512625,"tpot_p99_ms":86.63271500385017,"e2e_p50_ms":43376.19098147843,"e2e_p95_ms":45426.707432884716,"e2e_p99_ms":46181.83115239488,"itl_p50_ms":77.0906760590151,"itl_p95_ms":80.55677398806438,"itl_p99_ms":81.79810459259899,"validation_errors":[],"timestamp":"2026-07-22T18:43:48+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i8192_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i8192_o512_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":158.17554600001313,"request_tps":0.2528835904887389,"input_tps":2071.622373283749,"output_tps":129.4763983302343,"total_tps":2201.0987716139834,"mean_input_tokens":8192.0,"mean_output_tokens":512.0,"ttft_p50_ms":1443.1584720150568,"ttft_p95_ms":2563.5417400219008,"ttft_p99_ms":2709.771884668153,"tpot_p50_ms":60.06682871926554,"tpot_p95_ms":71.2232996729465,"tpot_p99_ms":72.94286882198405,"e2e_p50_ms":32206.854028976522,"e2e_p95_ms":37716.517192852916,"e2e_p99_ms":37743.93099553301,"itl_p50_ms":58.93414845922962,"itl_p95_ms":71.33412163821049,"itl_p99_ms":72.1114836493507,"validation_errors":[],"timestamp":"2026-07-22T18:47:04+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i8192_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i8192_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":411.41942564898636,"request_tps":0.1944487669093539,"input_tps":1592.924298521427,"output_tps":199.1155373151784,"total_tps":1792.0398358366053,"mean_input_tokens":8192.0,"mean_output_tokens":1024.0,"ttft_p50_ms":1821.8501105438918,"ttft_p95_ms":4445.312119938898,"ttft_p99_ms":4728.198262103832,"tpot_p50_ms":78.28133990808334,"tpot_p95_ms":80.30841988362309,"tpot_p99_ms":81.51844104213225,"e2e_p50_ms":82350.53083050298,"e2e_p95_ms":84510.51414076355,"e2e_p99_ms":84881.58685678149,"itl_p50_ms":76.71058003325015,"itl_p95_ms":78.99297920521349,"itl_p99_ms":79.98417927883565,"validation_errors":[],"timestamp":"2026-07-22T18:54:35+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i8192_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i8192_o1024_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":324.6368852739688,"request_tps":0.12321458778857815,"input_tps":1009.3739031640322,"output_tps":126.17173789550402,"total_tps":1135.5456410595361,"mean_input_tokens":8192.0,"mean_output_tokens":1024.0,"ttft_p50_ms":647.6456999662332,"ttft_p95_ms":1466.7398457124364,"ttft_p99_ms":1510.4055220563896,"tpot_p50_ms":61.503918705265036,"tpot_p95_ms":71.72676882975601,"tpot_p99_ms":71.81623347762145,"e2e_p50_ms":63510.10489551118,"e2e_p95_ms":73866.76847617491,"e2e_p99_ms":73909.99065396376,"itl_p50_ms":61.2029365147464,"itl_p95_ms":72.03168839332648,"itl_p99_ms":72.72949185222387,"validation_errors":[],"timestamp":"2026-07-22T19:00:38+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":1024,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i8192_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i8192_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":804.0681018510368,"request_tps":0.09949406003774162,"input_tps":815.0553398291794,"output_tps":203.76383495729485,"total_tps":1018.8191747864743,"mean_input_tokens":8192.0,"mean_output_tokens":2048.0,"ttft_p50_ms":1062.2408664785326,"ttft_p95_ms":2809.8898826690856,"ttft_p99_ms":2897.4033243174194,"tpot_p50_ms":77.83257750929418,"tpot_p95_ms":80.11583087915345,"tpot_p99_ms":80.26925650076862,"e2e_p50_ms":162034.85967952292,"e2e_p95_ms":164812.287680601,"e2e_p99_ms":164832.01820794493,"itl_p50_ms":77.55219400860369,"itl_p95_ms":80.47275985591114,"itl_p99_ms":81.47998804808594,"validation_errors":[],"timestamp":"2026-07-22T19:14:48+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i8192_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i8192_o2048_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":1000.8758942920249,"request_tps":0.15985997955638334,"input_tps":1309.5729525258923,"output_tps":327.3932381314731,"total_tps":1636.9661906573656,"mean_input_tokens":8192.0,"mean_output_tokens":2048.0,"ttft_p50_ms":4155.190574994776,"ttft_p95_ms":10120.331422821617,"ttft_p99_ms":11858.213511677457,"tpot_p50_ms":95.19103976113527,"tpot_p95_ms":98.82701551680167,"tpot_p99_ms":99.68580165473414,"e2e_p50_ms":199192.74586753454,"e2e_p95_ms":207727.64267942982,"e2e_p99_ms":214621.6531449859,"itl_p50_ms":92.94612391386181,"itl_p95_ms":95.86258397903292,"itl_p99_ms":97.17849837616087,"validation_errors":[],"timestamp":"2026-07-22T19:32:12+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":2048,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":60.672888,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i8192_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i8192_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1520.2255089710234,"request_tps":0.05262377162329596,"input_tps":431.09393713804053,"output_tps":215.54696856902027,"total_tps":646.6409057070608,"mean_input_tokens":8192.0,"mean_output_tokens":4096.0,"ttft_p50_ms":994.0135960350744,"ttft_p95_ms":2476.090034638762,"ttft_p99_ms":3367.8496101777996,"tpot_p50_ms":76.35606971868714,"tpot_p95_ms":80.1220021431473,"tpot_p99_ms":80.13669016258166,"e2e_p50_ms":313368.0764324963,"e2e_p95_ms":328726.80867983145,"e2e_p99_ms":329101.1374201963,"itl_p50_ms":76.0038010776043,"itl_p95_ms":80.35729501862079,"itl_p99_ms":81.52575802057981,"validation_errors":[],"timestamp":"2026-07-22T19:58:14+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i8192_o4096_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i8192_o4096_a1.log"} +{"timestamp":"2026-07-22T21:04:40+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":4096,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":2,"status":"FAILED","completed":0,"failed":160,"error_type":"benchmark rc=124","validation_errors":[],"gain_pct":null,"plateau_streak":0,"raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c32_i8192_o4096_a2.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c32_i8192_o4096_a2.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":152.14894604403526,"request_tps":0.5258005532082111,"input_tps":8614.71626376333,"output_tps":67.30247081065102,"total_tps":8682.018734573981,"mean_input_tokens":16384.0,"mean_output_tokens":128.0,"ttft_p50_ms":8930.64839503495,"ttft_p95_ms":14768.681913177714,"ttft_p99_ms":16999.57406088477,"tpot_p50_ms":167.9793164802268,"tpot_p95_ms":209.58889008991923,"tpot_p99_ms":227.62914901487449,"e2e_p50_ms":30281.566550431307,"e2e_p95_ms":35201.92647461663,"e2e_p99_ms":38441.21146804652,"itl_p50_ms":79.54272604547441,"itl_p95_ms":1053.9019158808514,"itl_p99_ms":1848.2122915424407,"validation_errors":[],"timestamp":"2026-07-22T21:14:28+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":85.96913352701813,"request_tps":0.46528327504230244,"input_tps":7623.201178293083,"output_tps":59.55625920541471,"total_tps":7682.757437498498,"mean_input_tokens":16384.0,"mean_output_tokens":128.0,"ttft_p50_ms":4595.128995017149,"ttft_p95_ms":11363.914758071765,"ttft_p99_ms":12680.35467678681,"tpot_p50_ms":97.88505525573882,"tpot_p95_ms":132.18067766421134,"tpot_p99_ms":142.84177842096236,"e2e_p50_ms":16511.405663506594,"e2e_p95_ms":26731.15985441836,"e2e_p99_ms":28975.950179346833,"itl_p50_ms":60.15623750863597,"itl_p95_ms":72.36015482340008,"itl_p99_ms":1290.250467308797,"validation_errors":[],"timestamp":"2026-07-22T21:16:33+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i16384_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":24.015481730923057,"request_tps":0.20819902994333234,"input_tps":3411.132906591557,"output_tps":26.64947583274654,"total_tps":3437.7823824243033,"mean_input_tokens":16384.0,"mean_output_tokens":128.0,"ttft_p50_ms":2350.7743020309135,"ttft_p95_ms":2379.2813392123207,"ttft_p99_ms":2381.40694945585,"tpot_p50_ms":26.014181268291445,"tpot_p95_ms":26.11499430802394,"tpot_p99_ms":26.115789588570127,"e2e_p50_ms":5561.865342082456,"e2e_p95_ms":5695.885616331361,"e2e_p99_ms":5698.112227204256,"itl_p50_ms":26.084523065946996,"itl_p95_ms":26.382693357300013,"itl_p99_ms":27.272247327491634,"validation_errors":[],"timestamp":"2026-07-22T21:17:36+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i16384_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":183.32123939308804,"request_tps":0.43639242383944044,"input_tps":7149.853472185392,"output_tps":111.71646050289675,"total_tps":7261.569932688289,"mean_input_tokens":16384.0,"mean_output_tokens":256.0,"ttft_p50_ms":6019.28992557805,"ttft_p95_ms":11963.147473795105,"ttft_p99_ms":13563.278930385122,"tpot_p50_ms":124.50609861590041,"tpot_p95_ms":142.360413944093,"tpot_p99_ms":148.92814680024082,"e2e_p50_ms":37011.94945897441,"e2e_p95_ms":42886.32667664788,"e2e_p99_ms":45173.82331278872,"itl_p50_ms":80.11842099949718,"itl_p95_ms":86.84101296239533,"itl_p99_ms":1477.776997112669,"validation_errors":[],"timestamp":"2026-07-22T21:21:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":104.86609260295518,"request_tps":0.38143883315504384,"input_tps":6249.493842412238,"output_tps":97.64834128769122,"total_tps":6347.14218369993,"mean_input_tokens":16384.0,"mean_output_tokens":256.0,"ttft_p50_ms":2714.707928011194,"ttft_p95_ms":6291.392291383815,"ttft_p99_ms":7381.1270218132995,"tpot_p50_ms":64.52171442181091,"tpot_p95_ms":98.81134190980126,"tpot_p99_ms":99.49650147035443,"e2e_p50_ms":22391.589315026067,"e2e_p95_ms":25955.0052329374,"e2e_p99_ms":26040.732813572977,"itl_p50_ms":60.81248092232272,"itl_p95_ms":74.73444222123362,"itl_p99_ms":76.02488879812884,"validation_errors":[],"timestamp":"2026-07-22T21:23:55+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i16384_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":34.65528558695223,"request_tps":0.1442781357970545,"input_tps":2363.852976898941,"output_tps":36.935202764045954,"total_tps":2400.788179662987,"mean_input_tokens":16384.0,"mean_output_tokens":256.0,"ttft_p50_ms":282.2947649983689,"ttft_p95_ms":309.6236903918907,"ttft_p99_ms":312.2021372569725,"tpot_p50_ms":25.828365086322172,"tpot_p95_ms":26.696593282726027,"tpot_p99_ms":26.711118725895442,"e2e_p50_ms":6850.338249001652,"e2e_p95_ms":7114.359220885672,"e2e_p99_ms":7122.958261040039,"itl_p50_ms":25.92746599111706,"itl_p95_ms":26.900527719408274,"itl_p99_ms":27.07159960642457,"validation_errors":[],"timestamp":"2026-07-22T21:25:11+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":256,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i16384_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":257.05899314698763,"request_tps":0.31121260929492395,"input_tps":5098.907390688034,"output_tps":159.34085595900106,"total_tps":5258.248246647036,"mean_input_tokens":16384.0,"mean_output_tokens":512.0,"ttft_p50_ms":3257.5230360380374,"ttft_p95_ms":10286.28223366686,"ttft_p99_ms":12287.076267036608,"tpot_p50_ms":91.40865246964886,"tpot_p95_ms":101.40167090735615,"tpot_p99_ms":102.43453131460105,"e2e_p50_ms":51518.65177351283,"e2e_p95_ms":57256.40635866438,"e2e_p99_ms":60635.87321437546,"itl_p50_ms":81.04736206587404,"itl_p95_ms":84.0959909837693,"itl_p99_ms":209.14175660582308,"validation_errors":[],"timestamp":"2026-07-22T21:30:09+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":176.08719991100952,"request_tps":0.2271601798439358,"input_tps":3721.792386563044,"output_tps":116.30601208009513,"total_tps":3838.0983986431393,"mean_input_tokens":16384.0,"mean_output_tokens":512.0,"ttft_p50_ms":2957.4136214796454,"ttft_p95_ms":6196.125268546166,"ttft_p99_ms":6968.152716426411,"tpot_p50_ms":65.80627199313413,"tpot_p95_ms":73.96782673677792,"tpot_p99_ms":74.0924035757894,"e2e_p50_ms":38161.60569654312,"e2e_p95_ms":38934.03083109879,"e2e_p99_ms":40232.19222860177,"itl_p50_ms":60.60073204571381,"itl_p95_ms":65.97969695576467,"itl_p99_ms":67.06546737113968,"validation_errors":[],"timestamp":"2026-07-22T21:33:45+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i16384_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":68.62545018608216,"request_tps":0.07285926702764339,"input_tps":1193.7262309809093,"output_tps":37.30394471815342,"total_tps":1231.0301756990627,"mean_input_tokens":16384.0,"mean_output_tokens":512.0,"ttft_p50_ms":315.92171895317733,"ttft_p95_ms":323.10447501949966,"ttft_p99_ms":324.16507822461426,"tpot_p50_ms":26.239310872855867,"tpot_p95_ms":26.448842497139715,"tpot_p99_ms":26.453789196181276,"e2e_p50_ms":13726.089315023273,"e2e_p95_ms":13832.981937006116,"e2e_p99_ms":13834.148338604718,"itl_p50_ms":26.318517047911882,"itl_p95_ms":26.5985599020496,"itl_p99_ms":26.717948708683252,"validation_errors":[],"timestamp":"2026-07-22T21:35:33+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":512,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i16384_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":461.0305955449585,"request_tps":0.17352427533673004,"input_tps":2843.021727116985,"output_tps":177.68885794481156,"total_tps":3020.7105850617963,"mean_input_tokens":16384.0,"mean_output_tokens":1024.0,"ttft_p50_ms":4333.275114011485,"ttft_p95_ms":10494.679190590978,"ttft_p99_ms":11740.18245950806,"tpot_p50_ms":86.14549763832525,"tpot_p95_ms":92.48261324917821,"tpot_p99_ms":94.54915040380429,"e2e_p50_ms":90015.53192199208,"e2e_p95_ms":102330.17868882162,"e2e_p99_ms":104476.66450355666,"itl_p50_ms":82.53799099475145,"itl_p95_ms":86.244410648942,"itl_p99_ms":87.36306346487254,"validation_errors":[],"timestamp":"2026-07-22T21:43:54+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i16384_o1024_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":335.37097579892725,"request_tps":0.11927090561343666,"input_tps":1954.1345175705462,"output_tps":122.13340734815914,"total_tps":2076.2679249187054,"mean_input_tokens":16384.0,"mean_output_tokens":1024.0,"ttft_p50_ms":874.616723973304,"ttft_p95_ms":3430.5481604242177,"ttft_p99_ms":5314.674343651859,"tpot_p50_ms":64.25959147164235,"tpot_p95_ms":67.71913338735365,"tpot_p99_ms":69.17382468570152,"e2e_p50_ms":66650.51912900526,"e2e_p95_ms":71496.29078928847,"e2e_p99_ms":71678.0972598528,"itl_p50_ms":64.15185943478718,"itl_p95_ms":65.84921926259995,"itl_p99_ms":66.49885277845897,"validation_errors":[],"timestamp":"2026-07-22T21:50:10+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":1024,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i16384_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i16384_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":878.2558397840476,"request_tps":0.09108963057925243,"input_tps":1492.4125074104718,"output_tps":186.55156342630897,"total_tps":1678.9640708367808,"mean_input_tokens":16384.0,"mean_output_tokens":2048.0,"ttft_p50_ms":4242.505369475111,"ttft_p95_ms":10269.977670686785,"ttft_p99_ms":12835.008265402867,"tpot_p50_ms":82.28961341813276,"tpot_p95_ms":88.66383706850593,"tpot_p99_ms":89.09228175807435,"e2e_p50_ms":174247.51013499917,"e2e_p95_ms":182866.0599557741,"e2e_p99_ms":187003.0920605431,"itl_p50_ms":80.35815448965877,"itl_p95_ms":85.91891000978649,"itl_p99_ms":86.98616176960059,"validation_errors":[],"timestamp":"2026-07-22T22:05:29+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":651.3175416319864,"request_tps":0.061413976199340845,"input_tps":1006.2065860500004,"output_tps":125.77582325625005,"total_tps":1131.9824093062505,"mean_input_tokens":16384.0,"mean_output_tokens":2048.0,"ttft_p50_ms":2698.1677320436575,"ttft_p95_ms":4531.052401958732,"ttft_p99_ms":4655.3382462018635,"tpot_p50_ms":61.376528254481286,"tpot_p95_ms":65.57403422004505,"tpot_p99_ms":66.52941480059218,"e2e_p50_ms":127086.29324403591,"e2e_p95_ms":136885.88085551746,"e2e_p99_ms":137049.35106788413,"itl_p50_ms":61.028828960843384,"itl_p95_ms":65.67398469196633,"itl_p99_ms":66.46701183286496,"validation_errors":[],"timestamp":"2026-07-22T22:16:57+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":2048,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i16384_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":269.9440217630472,"request_tps":0.018522358699941593,"input_tps":303.47032493984307,"output_tps":37.93379061748038,"total_tps":341.40411555732345,"mean_input_tokens":16384.0,"mean_output_tokens":2048.0,"ttft_p50_ms":320.6515049096197,"ttft_p95_ms":1929.4331216486166,"ttft_p99_ms":2250.98537156824,"tpot_p50_ms":26.10909627897471,"tpot_p95_ms":26.38850710689148,"tpot_p99_ms":26.431286142047988,"e2e_p50_ms":53747.753055999056,"e2e_p95_ms":55618.120527407154,"e2e_p99_ms":55852.26735110395,"itl_p50_ms":26.075390982441604,"itl_p95_ms":26.549331622663885,"itl_p99_ms":26.76485584583133,"validation_errors":[],"timestamp":"2026-07-22T22:22:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":2048,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i16384_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":251.6305991950212,"request_tps":0.3179263581453288,"input_tps":10417.810903706135,"output_tps":40.69457384260209,"total_tps":10458.505477548737,"mean_input_tokens":32768.0,"mean_output_tokens":128.0,"ttft_p50_ms":13290.765161043964,"ttft_p95_ms":26118.876439298034,"ttft_p99_ms":30842.52740347988,"tpot_p50_ms":280.3995917751967,"tpot_p95_ms":351.41655613134765,"tpot_p99_ms":384.6924748546792,"e2e_p50_ms":49856.00823297864,"e2e_p95_ms":61870.40506798657,"e2e_p99_ms":68195.25385075594,"itl_p50_ms":86.5422049537301,"itl_p95_ms":2142.5647311378207,"itl_p99_ms":2569.6078925300408,"validation_errors":[],"timestamp":"2026-07-22T22:27:07+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i32768_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i32768_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":145.12928141199518,"request_tps":0.2756163305628683,"input_tps":9031.395919884068,"output_tps":35.27889031204714,"total_tps":9066.674810196115,"mean_input_tokens":32768.0,"mean_output_tokens":128.0,"ttft_p50_ms":10151.981936069205,"ttft_p95_ms":20585.27990888687,"ttft_p99_ms":23533.07879790431,"tpot_p50_ms":147.52407989365702,"tpot_p95_ms":197.6478013071125,"tpot_p99_ms":226.66445618979336,"e2e_p50_ms":29529.69190949807,"e2e_p95_ms":37971.92853669985,"e2e_p99_ms":42321.51983953896,"itl_p50_ms":62.61512544006109,"itl_p95_ms":72.26468421868049,"itl_p99_ms":2215.470617420506,"validation_errors":[],"timestamp":"2026-07-22T22:30:14+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i32768_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i32768_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":33.14747340697795,"request_tps":0.15084105924487862,"input_tps":4942.7598293361825,"output_tps":19.307655583344463,"total_tps":4962.0674849195275,"mean_input_tokens":32768.0,"mean_output_tokens":128.0,"ttft_p50_ms":3483.0551599152386,"ttft_p95_ms":6017.307216976769,"ttft_p99_ms":6018.419136987068,"tpot_p50_ms":26.796044330809295,"tpot_p95_ms":26.91129839403452,"tpot_p99_ms":26.91965950905954,"e2e_p50_ms":6902.117382967845,"e2e_p95_ms":9406.63605616428,"e2e_p99_ms":9413.203408783302,"itl_p50_ms":27.017473010346293,"itl_p95_ms":27.333528629969805,"itl_p99_ms":27.895204978995025,"validation_errors":[],"timestamp":"2026-07-22T22:31:23+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i32768_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i32768_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":289.8359683640301,"request_tps":0.27601819212279777,"input_tps":9044.564119479837,"output_tps":70.66065718343623,"total_tps":9115.224776663274,"mean_input_tokens":32768.0,"mean_output_tokens":256.0,"ttft_p50_ms":12942.16159998905,"ttft_p95_ms":27984.303534514038,"ttft_p99_ms":36095.26163545784,"tpot_p50_ms":182.56656457076542,"tpot_p95_ms":233.11230910085962,"tpot_p99_ms":256.2320246398677,"e2e_p50_ms":56715.26775805978,"e2e_p95_ms":76209.34526897616,"e2e_p99_ms":88998.17507414958,"itl_p50_ms":87.69033552380279,"itl_p95_ms":149.319258442847,"itl_p99_ms":2278.4392145299353,"validation_errors":[],"timestamp":"2026-07-22T22:37:03+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i32768_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i32768_o256_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":199.1597462639911,"request_tps":0.20084379876131708,"input_tps":6581.249597810838,"output_tps":51.41601248289717,"total_tps":6632.665610293735,"mean_input_tokens":32768.0,"mean_output_tokens":256.0,"ttft_p50_ms":8475.281495484523,"ttft_p95_ms":20099.412193516022,"ttft_p99_ms":28631.00056842202,"tpot_p50_ms":115.58317266468543,"tpot_p95_ms":167.6707908403877,"tpot_p99_ms":176.8612443380898,"e2e_p50_ms":36181.63466954138,"e2e_p95_ms":60353.09472696388,"e2e_p99_ms":71048.37405076947,"itl_p50_ms":69.18587005930021,"itl_p95_ms":78.64517325069754,"itl_p99_ms":1977.9912414611317,"validation_errors":[],"timestamp":"2026-07-22T22:41:00+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i32768_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i32768_o256_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":46.97131344303489,"request_tps":0.1064479494716242,"input_tps":3488.0864082861817,"output_tps":27.250675064735795,"total_tps":3515.3370833509175,"mean_input_tokens":32768.0,"mean_output_tokens":256.0,"ttft_p50_ms":401.9349740119651,"ttft_p95_ms":6085.346337221563,"ttft_p99_ms":6105.484023466706,"tpot_p50_ms":26.50156015685449,"tpot_p95_ms":26.567292910014444,"tpot_p99_ms":26.570727724161948,"e2e_p50_ms":7177.689513075165,"e2e_p95_ms":12838.474808237515,"e2e_p99_ms":12872.333960882388,"itl_p50_ms":26.61031298339367,"itl_p95_ms":26.86186645878479,"itl_p99_ms":27.01719551347196,"validation_errors":[],"timestamp":"2026-07-22T22:42:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":256,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i32768_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i32768_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":389.8900619589258,"request_tps":0.20518604551769226,"input_tps":6723.53633952374,"output_tps":105.05525530505844,"total_tps":6828.591594828798,"mean_input_tokens":32768.0,"mean_output_tokens":512.0,"ttft_p50_ms":16550.17402151134,"ttft_p95_ms":29292.055011849152,"ttft_p99_ms":33573.35275775752,"tpot_p50_ms":121.31984492364283,"tpot_p95_ms":147.02211885868869,"tpot_p99_ms":150.94870738281426,"e2e_p50_ms":77740.93291250756,"e2e_p95_ms":96493.6448461609,"e2e_p99_ms":100914.91184239625,"itl_p50_ms":86.31842699833214,"itl_p95_ms":89.24683055374771,"itl_p99_ms":2189.0342302853264,"validation_errors":[],"timestamp":"2026-07-22T22:49:37+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i32768_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i32768_o512_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":286.918702392024,"request_tps":0.13941231319716144,"input_tps":4568.262678844586,"output_tps":71.37910435694666,"total_tps":4639.641783201532,"mean_input_tokens":32768.0,"mean_output_tokens":512.0,"ttft_p50_ms":8227.453268540557,"ttft_p95_ms":24807.397105713604,"ttft_p99_ms":27055.376205944922,"tpot_p50_ms":93.17175104987211,"tpot_p95_ms":117.19474970381762,"tpot_p99_ms":124.26452839882394,"e2e_p50_ms":60135.26890298817,"e2e_p95_ms":72376.33647950133,"e2e_p99_ms":74950.31652775127,"itl_p50_ms":69.83639346435666,"itl_p95_ms":81.93503119400702,"itl_p99_ms":1307.6532075740397,"validation_errors":[],"timestamp":"2026-07-22T22:55:05+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i32768_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i32768_o512_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":80.82389019103721,"request_tps":0.06186289707389591,"input_tps":2027.1234113174212,"output_tps":31.673803301834706,"total_tps":2058.797214619256,"mean_input_tokens":32768.0,"mean_output_tokens":512.0,"ttft_p50_ms":428.804399096407,"ttft_p95_ms":6021.562439762056,"ttft_p99_ms":6028.755957521498,"tpot_p50_ms":26.121873878545827,"tpot_p95_ms":27.30702476707194,"tpot_p99_ms":27.328918752189427,"e2e_p50_ms":14355.769739951938,"e2e_p95_ms":19272.42521396838,"e2e_p99_ms":19312.725651571527,"itl_p50_ms":26.26544947270304,"itl_p95_ms":27.489012631122023,"itl_p99_ms":27.658809248823673,"validation_errors":[],"timestamp":"2026-07-22T22:57:03+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":512,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i32768_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i32768_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":577.4338035809342,"request_tps":0.1385440192518744,"input_tps":4539.81042284542,"output_tps":141.8690757139194,"total_tps":4681.67949855934,"mean_input_tokens":32768.0,"mean_output_tokens":1024.0,"ttft_p50_ms":15170.622567995451,"ttft_p95_ms":27841.32018307573,"ttft_p99_ms":33022.26603679823,"tpot_p50_ms":93.4342387575476,"tpot_p95_ms":111.31350052575873,"tpot_p99_ms":114.96811384988925,"e2e_p50_ms":112511.02925202576,"e2e_p95_ms":130216.1091115733,"e2e_p99_ms":134943.78475562664,"itl_p50_ms":70.4409530153498,"itl_p95_ms":89.1938341781497,"itl_p99_ms":951.0676693916321,"validation_errors":[],"timestamp":"2026-07-22T23:07:28+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i32768_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i32768_o1024_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":429.3920534180943,"request_tps":0.0931549610235857,"input_tps":3052.501762820856,"output_tps":95.39068008815175,"total_tps":3147.892442909008,"mean_input_tokens":32768.0,"mean_output_tokens":1024.0,"ttft_p50_ms":9378.182704967912,"ttft_p95_ms":22657.615452288876,"ttft_p99_ms":26928.17785229534,"tpot_p50_ms":75.24616758995494,"tpot_p95_ms":88.05348021736167,"tpot_p99_ms":92.09336932968982,"e2e_p50_ms":87491.7528219521,"e2e_p95_ms":108219.97572283144,"e2e_p99_ms":114512.34613837092,"itl_p50_ms":63.79989755805582,"itl_p95_ms":71.3233457528986,"itl_p99_ms":72.90060254279524,"validation_errors":[],"timestamp":"2026-07-22T23:15:18+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":1024,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i32768_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i32768_o1024_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":157.58758033602498,"request_tps":0.03172838867973268,"input_tps":1039.6758402574806,"output_tps":32.48987000804627,"total_tps":1072.1657102655267,"mean_input_tokens":32768.0,"mean_output_tokens":1024.0,"ttft_p50_ms":6003.1426378991455,"ttft_p95_ms":6056.393590779044,"ttft_p99_ms":6060.642982930876,"tpot_p50_ms":27.085386071434264,"tpot_p95_ms":27.197785476629576,"tpot_p99_ms":27.199235149661035,"e2e_p50_ms":33690.94874302391,"e2e_p95_ms":33844.39835459925,"e2e_p99_ms":33870.97950772382,"itl_p50_ms":27.135262032970786,"itl_p95_ms":27.374596137087792,"itl_p99_ms":27.560830253642052,"validation_errors":[],"timestamp":"2026-07-22T23:18:34+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":1024,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i32768_o1024_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i32768_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1049.319787114975,"request_tps":0.07623986603736303,"input_tps":2498.227930312312,"output_tps":156.1392456445195,"total_tps":2654.3671759568315,"mean_input_tokens":32768.0,"mean_output_tokens":2048.0,"ttft_p50_ms":12598.86947600171,"ttft_p95_ms":25754.183080128852,"ttft_p99_ms":34338.4104972775,"tpot_p50_ms":101.6727683397577,"tpot_p95_ms":108.70993985757177,"tpot_p99_ms":109.69215612051703,"e2e_p50_ms":216177.34529951122,"e2e_p95_ms":236198.01819474084,"e2e_p99_ms":252498.66901825648,"itl_p50_ms":87.80708746053278,"itl_p95_ms":95.22842805017717,"itl_p99_ms":97.75970649323418,"validation_errors":[],"timestamp":"2026-07-22T23:36:46+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i32768_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i32768_o2048_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":752.5763154129963,"request_tps":0.05315075585131713,"input_tps":1741.6439677359597,"output_tps":108.85274798349748,"total_tps":1850.4967157194571,"mean_input_tokens":32768.0,"mean_output_tokens":2048.0,"ttft_p50_ms":8210.53161896998,"ttft_p95_ms":22150.38943065446,"ttft_p99_ms":28208.5629260214,"tpot_p50_ms":68.71709677674656,"tpot_p95_ms":90.40034104001951,"tpot_p99_ms":94.99942484920679,"e2e_p50_ms":148338.84903800208,"e2e_p95_ms":198235.59242666815,"e2e_p99_ms":205907.76084323414,"itl_p50_ms":61.89525354420766,"itl_p95_ms":82.65664269565605,"itl_p99_ms":83.62127898377366,"validation_errors":[],"timestamp":"2026-07-22T23:50:01+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":2048,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i32768_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i32768_o2048_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":296.31990192306694,"request_tps":0.01687365569288742,"input_tps":552.9159497445349,"output_tps":34.557246859033434,"total_tps":587.4731966035683,"mean_input_tokens":32768.0,"mean_output_tokens":2048.0,"ttft_p50_ms":4555.9156680246815,"ttft_p95_ms":6039.396488503553,"ttft_p99_ms":6041.268515386619,"tpot_p50_ms":27.242493868553417,"tpot_p95_ms":27.291186915291544,"tpot_p99_ms":27.298651098739583,"e2e_p50_ms":60243.64867596887,"e2e_p95_ms":61883.05334667675,"e2e_p99_ms":61907.9666286055,"itl_p50_ms":27.26531506050378,"itl_p95_ms":27.464679756667465,"itl_p99_ms":27.67756721121259,"validation_errors":[],"timestamp":"2026-07-22T23:55:37+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":2048,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i32768_o2048_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i32768_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":762.7487404310377,"request_tps":0.10488381790678686,"input_tps":6873.6658903391835,"output_tps":13.425128692068718,"total_tps":6887.0910190312525,"mean_input_tokens":65536.0,"mean_output_tokens":128.0,"ttft_p50_ms":40100.48328898847,"ttft_p95_ms":72883.65917775658,"ttft_p99_ms":115248.27543771581,"tpot_p50_ms":904.2289407165744,"tpot_p95_ms":1082.2992204270035,"tpot_p99_ms":1133.203141234364,"e2e_p50_ms":154677.09750350332,"e2e_p95_ms":190670.53921719771,"e2e_p99_ms":249435.94695679948,"itl_p50_ms":103.40386896859854,"itl_p95_ms":3789.5884490106255,"itl_p99_ms":4201.93060528487,"validation_errors":[],"timestamp":"2026-07-23T00:09:24+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c16_i65536_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c16_i65536_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":434.32164914906025,"request_tps":0.09209764256138174,"input_tps":6035.711102902714,"output_tps":11.788498247856863,"total_tps":6047.499601150571,"mean_input_tokens":65536.0,"mean_output_tokens":128.0,"ttft_p50_ms":38818.221221969,"ttft_p95_ms":76978.34239983931,"ttft_p99_ms":86220.37316633154,"tpot_p50_ms":376.71108279509224,"tpot_p95_ms":529.6431666294636,"tpot_p99_ms":599.631088042584,"e2e_p50_ms":85188.20616602898,"e2e_p95_ms":111972.06442226883,"e2e_p99_ms":148730.04269028548,"itl_p50_ms":68.17380792926997,"itl_p95_ms":3030.2434626268223,"itl_p99_ms":3855.509727266617,"validation_errors":[],"timestamp":"2026-07-23T00:17:48+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c8_i65536_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c8_i65536_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":71.41779580002185,"request_tps":0.07001056170930538,"input_tps":4588.212172181037,"output_tps":8.961351898791088,"total_tps":4597.173524079828,"mean_input_tokens":65536.0,"mean_output_tokens":128.0,"ttft_p50_ms":17408.01959799137,"ttft_p95_ms":17662.359986361116,"ttft_p99_ms":17678.296818844974,"tpot_p50_ms":27.606362055283128,"tpot_p95_ms":28.067865795052544,"tpot_p99_ms":28.06993041238858,"e2e_p50_ms":20971.327521954663,"e2e_p95_ms":21180.155713972636,"e2e_p99_ms":21186.662349184044,"itl_p50_ms":28.22552004363388,"itl_p95_ms":28.75272782985121,"itl_p99_ms":29.038289727177474,"validation_errors":[],"timestamp":"2026-07-23T00:19:44+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/raw_outputs/vllm_adaptive_c1_i65536_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp2_dp4/logs/vllm_c1_i65536_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":44.40719663898926,"request_tps":1.8015098014487243,"input_tps":1844.7460366834937,"output_tps":230.5932545854367,"total_tps":2075.3392912689305,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1005.9468819526955,"ttft_p95_ms":1074.713952158345,"ttft_p99_ms":1079.2205419240054,"tpot_p50_ms":61.3667477558433,"tpot_p95_ms":66.86363791163247,"tpot_p99_ms":69.15889790394402,"e2e_p50_ms":8860.960246936884,"e2e_p95_ms":9109.096292313188,"e2e_p99_ms":9111.173996197758,"itl_p50_ms":61.856393003836274,"itl_p95_ms":64.88115987740457,"itl_p99_ms":65.88571431813762,"validation_errors":[],"timestamp":"2026-07-23T00:27:56+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c16_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":66.94320556393359,"request_tps":2.39008572493878,"input_tps":2447.447782337311,"output_tps":305.93097279216386,"total_tps":2753.378755129475,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1279.5342245372012,"ttft_p95_ms":1907.6965985295828,"ttft_p99_ms":1909.568356674863,"tpot_p50_ms":95.32844811800764,"tpot_p95_ms":105.05584735197829,"tpot_p99_ms":106.25885317178157,"e2e_p50_ms":13117.860729980748,"e2e_p95_ms":14153.041057894006,"e2e_p99_ms":14163.864497826435,"itl_p50_ms":90.64350748667493,"itl_p95_ms":98.03166088531725,"itl_p99_ms":133.15003629770686,"validation_errors":[],"timestamp":"2026-07-23T00:29:40+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":32.671258,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c32_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c32_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":73.31772922794335,"request_tps":3.273423802500004,"input_tps":3351.985973760004,"output_tps":418.9982467200005,"total_tps":3770.9842204800048,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1209.7833154839464,"ttft_p95_ms":2362.9576735547726,"ttft_p99_ms":2467.7785141789354,"tpot_p50_ms":106.28649112163478,"tpot_p95_ms":114.68083644418712,"tpot_p99_ms":116.25784468838916,"e2e_p50_ms":15148.305448528845,"e2e_p95_ms":15778.819729801035,"e2e_p99_ms":16112.2722271597,"itl_p50_ms":101.01440898142755,"itl_p95_ms":102.79461299069226,"itl_p99_ms":288.1781766819768,"validation_errors":[],"timestamp":"2026-07-23T00:31:30+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":36.958427,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c48_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c48_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":77.71671831805725,"request_tps":4.117518172735929,"input_tps":4216.338608881591,"output_tps":527.0423261101989,"total_tps":4743.38093499179,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1541.571965557523,"ttft_p95_ms":2973.194576759124,"ttft_p99_ms":3355.183034266811,"tpot_p50_ms":108.81937849592036,"tpot_p95_ms":123.23788346774971,"tpot_p99_ms":129.52345561391058,"e2e_p50_ms":15222.109164053109,"e2e_p95_ms":17127.17312244349,"e2e_p99_ms":17134.239162297454,"itl_p50_ms":104.25966395996511,"itl_p95_ms":107.29315697681159,"itl_p99_ms":325.2980307349935,"validation_errors":[],"timestamp":"2026-07-23T00:33:24+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":25.786284,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c64_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c64_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":82.91905850404873,"request_tps":4.8239814490955535,"input_tps":4939.757003873847,"output_tps":617.4696254842308,"total_tps":5557.226629358078,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1721.3420600164682,"ttft_p95_ms":3531.2187138246372,"ttft_p99_ms":4163.57716984232,"tpot_p50_ms":114.93293692515502,"tpot_p95_ms":135.40843139662897,"tpot_p99_ms":141.25728253901232,"e2e_p50_ms":16314.259518054314,"e2e_p95_ms":18598.02438748884,"e2e_p99_ms":18628.951506328303,"itl_p50_ms":110.39953900035471,"itl_p95_ms":114.46549538522959,"itl_p99_ms":444.8370685754344,"validation_errors":[],"timestamp":"2026-07-23T00:35:27+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":17.157502,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c80_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c80_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":85.13241609802935,"request_tps":5.638275312746716,"input_tps":5773.593920252637,"output_tps":721.6992400315796,"total_tps":6495.293160284216,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1603.032307990361,"ttft_p95_ms":3563.508887437638,"ttft_p99_ms":4370.562668723287,"tpot_p50_ms":117.72647111788464,"tpot_p95_ms":135.60391064539849,"tpot_p99_ms":137.8363488640408,"e2e_p50_ms":16355.641119007487,"e2e_p95_ms":18534.29522922961,"e2e_p99_ms":18550.03452652716,"itl_p50_ms":110.01877998933196,"itl_p95_ms":115.49648403306492,"itl_p99_ms":622.202554200775,"validation_errors":[],"timestamp":"2026-07-23T00:37:33+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":16.88012,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c96_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c96_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":560,"failed":0,"duration_s":86.79016929701902,"request_tps":6.452343675970157,"input_tps":6607.199924193441,"output_tps":825.8999905241801,"total_tps":7433.099914717622,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1707.2073600138538,"ttft_p95_ms":3495.4696796194185,"ttft_p99_ms":4662.740297575946,"tpot_p50_ms":120.47115057078005,"tpot_p95_ms":149.35551934172472,"tpot_p99_ms":153.17628147921113,"e2e_p50_ms":16285.670631972607,"e2e_p95_ms":20442.440698377322,"e2e_p99_ms":20467.823019960197,"itl_p50_ms":111.84241797309369,"itl_p95_ms":120.91306851943953,"itl_p99_ms":624.5386045239866,"validation_errors":[],"timestamp":"2026-07-23T00:39:41+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":14.438251,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c112_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c112_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":640,"failed":0,"duration_s":91.11920318298507,"request_tps":7.023766425116291,"input_tps":7192.336819319082,"output_tps":899.0421024148852,"total_tps":8091.378921733966,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1774.1853999905288,"ttft_p95_ms":3482.3102760477923,"ttft_p99_ms":4927.2171049704775,"tpot_p50_ms":125.79494998762824,"tpot_p95_ms":151.34507266755992,"tpot_p99_ms":156.7388365654858,"e2e_p50_ms":17414.89270393504,"e2e_p95_ms":20863.278109609382,"e2e_p99_ms":20929.274403997697,"itl_p50_ms":116.44666595384479,"itl_p95_ms":122.20438427757472,"itl_p99_ms":719.9294293159619,"validation_errors":[],"timestamp":"2026-07-23T00:41:53+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":128,"num_prompts":640,"warmup_requests":128,"attempt":1,"gain_pct":8.856049,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c128_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c128_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":720,"failed":0,"duration_s":92.2181025010068,"request_tps":7.80757769324238,"input_tps":7994.959557880197,"output_tps":999.3699447350247,"total_tps":8994.329502615221,"mean_input_tokens":1024.0,"mean_output_tokens":128.0,"ttft_p50_ms":1931.1170189757831,"ttft_p95_ms":4479.106376477284,"ttft_p99_ms":5383.928186239209,"tpot_p50_ms":127.13536829504324,"tpot_p95_ms":151.7455794639504,"tpot_p99_ms":156.95134611568398,"e2e_p50_ms":18086.432086478453,"e2e_p95_ms":20988.543751242105,"e2e_p99_ms":21075.163445842918,"itl_p50_ms":115.6650580232963,"itl_p95_ms":126.78809182252735,"itl_p99_ms":801.1834052240015,"validation_errors":[],"timestamp":"2026-07-23T00:44:08+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"concurrency":144,"num_prompts":720,"warmup_requests":144,"attempt":1,"gain_pct":11.159415,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c144_i1024_o128_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c144_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":108.61439983802848,"request_tps":0.7365505873926498,"input_tps":754.2278014900734,"output_tps":188.55695037251834,"total_tps":942.7847518625917,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":501.36691849911585,"ttft_p95_ms":698.2993290643208,"ttft_p99_ms":722.4401124182618,"tpot_p50_ms":83.41608631958746,"tpot_p95_ms":85.24892140695754,"tpot_p99_ms":85.34223880568592,"e2e_p50_ms":21824.81415948132,"e2e_p95_ms":22127.113521198044,"e2e_p99_ms":22127.97656976967,"itl_p50_ms":83.04204104933888,"itl_p95_ms":85.24561722297221,"itl_p99_ms":86.50979544501752,"validation_errors":[],"timestamp":"2026-07-23T00:46:34+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c16_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c16_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":115.79425556096248,"request_tps":1.38176111781093,"input_tps":1414.9233846383922,"output_tps":353.73084615959806,"total_tps":1768.6542307979903,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":553.4031150164083,"ttft_p95_ms":742.4477303342428,"ttft_p99_ms":805.1197216962464,"tpot_p50_ms":95.19693892946759,"tpot_p95_ms":95.9978383072588,"tpot_p99_ms":96.78827319203364,"e2e_p50_ms":24837.384668469895,"e2e_p95_ms":25006.177042907802,"e2e_p99_ms":25011.209703838686,"itl_p50_ms":94.05417751986533,"itl_p95_ms":96.53609097586013,"itl_p99_ms":97.97159155714327,"validation_errors":[],"timestamp":"2026-07-23T00:49:09+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":87.598943,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c32_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c32_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":129.07391834794544,"request_tps":1.8593996608441867,"input_tps":1904.0252527044472,"output_tps":476.0063131761118,"total_tps":2380.031565880559,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":970.1211859937757,"ttft_p95_ms":1202.3288815573324,"ttft_p99_ms":1219.9526215123478,"tpot_p50_ms":101.19262043724531,"tpot_p95_ms":103.48327788460415,"tpot_p99_ms":103.78909427222484,"e2e_p50_ms":26813.58839350287,"e2e_p95_ms":26862.83846984734,"e2e_p99_ms":26872.07451156224,"itl_p50_ms":100.52085248753428,"itl_p95_ms":102.58559971116483,"itl_p99_ms":104.62224620045158,"validation_errors":[],"timestamp":"2026-07-23T00:51:57+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":34.567375,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c48_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c48_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":137.37284643296152,"request_tps":2.3294268722615517,"input_tps":2385.333117195829,"output_tps":596.3332792989572,"total_tps":2981.666396494786,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":931.4560675411485,"ttft_p95_ms":1231.7633512779141,"ttft_p99_ms":1299.321782257175,"tpot_p50_ms":104.49927640215986,"tpot_p95_ms":107.45930945235945,"tpot_p99_ms":108.10227546701208,"e2e_p50_ms":27655.851563438773,"e2e_p95_ms":28066.758056537947,"e2e_p99_ms":28130.54605856538,"itl_p50_ms":103.78814802970737,"itl_p95_ms":106.70563101302832,"itl_p99_ms":110.72339110542089,"validation_errors":[],"timestamp":"2026-07-23T00:54:58+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":25.278439,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c64_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c64_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":149.09916729596443,"request_tps":2.682778229109711,"input_tps":2747.164906608344,"output_tps":686.791226652086,"total_tps":3433.95613326043,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1210.738035500981,"ttft_p95_ms":1585.6146395090036,"ttft_p99_ms":1616.0292647394817,"tpot_p50_ms":112.28147955684392,"tpot_p95_ms":115.43985034035538,"tpot_p99_ms":116.14153316402442,"e2e_p50_ms":29873.060641461052,"e2e_p95_ms":30127.641169016715,"e2e_p99_ms":30154.465005820384,"itl_p50_ms":111.31411395035684,"itl_p95_ms":113.59066078439355,"itl_p99_ms":117.26110428571701,"validation_errors":[],"timestamp":"2026-07-23T00:58:12+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":15.169026,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c80_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c80_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":148.1221665260382,"request_tps":3.2405683177448084,"input_tps":3318.3419573706838,"output_tps":829.5854893426709,"total_tps":4147.927446713355,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1127.111411478836,"ttft_p95_ms":1695.9994666802231,"ttft_p99_ms":1844.0605269174557,"tpot_p50_ms":111.16537677840896,"tpot_p95_ms":115.07484113151554,"tpot_p99_ms":115.86536781461544,"e2e_p50_ms":29524.47735151509,"e2e_p95_ms":30036.0296443454,"e2e_p99_ms":30050.413801866816,"itl_p50_ms":109.62403006851673,"itl_p95_ms":112.32327178586274,"itl_p99_ms":116.88451471738522,"validation_errors":[],"timestamp":"2026-07-23T01:01:19+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":20.791509,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c96_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c96_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":560,"failed":0,"duration_s":152.28788569802418,"request_tps":3.6772458783126014,"input_tps":3765.499779392104,"output_tps":941.374944848026,"total_tps":4706.87472424013,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1458.6557929869741,"ttft_p95_ms":1804.1540066536982,"ttft_p99_ms":1866.7548648861703,"tpot_p50_ms":114.18696821977694,"tpot_p95_ms":117.94835005793264,"tpot_p99_ms":118.27713989624844,"e2e_p50_ms":30445.957715972327,"e2e_p95_ms":30741.47924106219,"e2e_p99_ms":30943.298781815683,"itl_p50_ms":112.21128300530836,"itl_p95_ms":115.17703911522403,"itl_p99_ms":123.66167776286619,"validation_errors":[],"timestamp":"2026-07-23T01:04:35+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":13.475339,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c112_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c112_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":640,"failed":0,"duration_s":160.09789549699053,"request_tps":3.997554109085903,"input_tps":4093.4954077039647,"output_tps":1023.3738519259912,"total_tps":5116.869259629956,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1661.581659980584,"ttft_p95_ms":2283.770077803638,"ttft_p99_ms":2455.9397031890694,"tpot_p50_ms":118.9470321668184,"tpot_p95_ms":123.39290103642271,"tpot_p99_ms":124.33732080280178,"e2e_p50_ms":31976.319596986286,"e2e_p95_ms":32279.766251757977,"e2e_p99_ms":32297.847480613273,"itl_p50_ms":116.51917599374428,"itl_p95_ms":119.54121307353489,"itl_p99_ms":144.30194522370587,"validation_errors":[],"timestamp":"2026-07-23T01:07:57+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":128,"num_prompts":640,"warmup_requests":128,"attempt":1,"gain_pct":8.710547,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c128_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c128_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":720,"failed":0,"duration_s":162.97565648006275,"request_tps":4.417837703805043,"input_tps":4523.865808696364,"output_tps":1130.966452174091,"total_tps":5654.832260870456,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1650.7194820442237,"ttft_p95_ms":3314.660829550121,"ttft_p99_ms":3943.820036307443,"tpot_p50_ms":119.6380508020429,"tpot_p95_ms":131.84232741697033,"tpot_p99_ms":134.12020944468404,"e2e_p50_ms":32064.860058017075,"e2e_p95_ms":34854.52993239742,"e2e_p99_ms":34895.79141958384,"itl_p50_ms":116.30661389790475,"itl_p95_ms":124.1098301950842,"itl_p99_ms":164.06555980909633,"validation_errors":[],"timestamp":"2026-07-23T01:11:22+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":144,"num_prompts":720,"warmup_requests":144,"attempt":1,"gain_pct":10.513519,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c144_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c144_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":800,"failed":0,"duration_s":169.45548730297014,"request_tps":4.721003803020417,"input_tps":4834.307894292907,"output_tps":1208.5769735732267,"total_tps":6042.884867866133,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1751.3434045831673,"ttft_p95_ms":3905.036666453816,"ttft_p99_ms":5314.02985720546,"tpot_p50_ms":124.19041375077202,"tpot_p95_ms":135.14831245520756,"tpot_p99_ms":139.11448915524628,"e2e_p50_ms":33201.032530516386,"e2e_p95_ms":36204.430650611175,"e2e_p99_ms":36260.13615369797,"itl_p50_ms":120.24135491810739,"itl_p95_ms":125.69208833156154,"itl_p99_ms":220.49296644283456,"validation_errors":[],"timestamp":"2026-07-23T01:14:59+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":160,"num_prompts":800,"warmup_requests":160,"attempt":1,"gain_pct":6.862319,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c160_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c160_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":880,"failed":0,"duration_s":170.62969895696733,"request_tps":5.157367125297076,"input_tps":5281.143936304205,"output_tps":1320.2859840760514,"total_tps":6601.429920380257,"mean_input_tokens":1024.0,"mean_output_tokens":256.0,"ttft_p50_ms":1945.2701334957965,"ttft_p95_ms":5343.910970742581,"ttft_p99_ms":6312.216381983607,"tpot_p50_ms":123.64396909822473,"tpot_p95_ms":137.08349655131244,"tpot_p99_ms":143.16010421727253,"e2e_p50_ms":33257.54008057993,"e2e_p95_ms":37319.53025209368,"e2e_p99_ms":37368.234107329044,"itl_p50_ms":118.80167806521058,"itl_p95_ms":124.655477469787,"itl_p99_ms":294.7731074877081,"validation_errors":[],"timestamp":"2026-07-23T01:18:34+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"concurrency":176,"num_prompts":880,"warmup_requests":176,"attempt":1,"gain_pct":9.24302,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c176_i1024_o256_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c176_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":160.02539056702517,"request_tps":0.49992066706747224,"input_tps":511.91876307709157,"output_tps":255.95938153854578,"total_tps":767.8781446156373,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":410.2638910408132,"ttft_p95_ms":427.63494712417014,"ttft_p99_ms":431.19447772740386,"tpot_p50_ms":61.59523493437636,"tpot_p95_ms":63.70277809863774,"tpot_p99_ms":64.014158456312,"e2e_p50_ms":31897.490329050925,"e2e_p95_ms":32963.00349935773,"e2e_p99_ms":32965.062390364474,"itl_p50_ms":61.822288029361516,"itl_p95_ms":64.68662861734629,"itl_p99_ms":65.7200977334287,"validation_errors":[],"timestamp":"2026-07-23T01:21:52+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c16_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c16_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":222.556726446026,"request_tps":0.7189178352639136,"input_tps":736.1718633102475,"output_tps":368.0859316551238,"total_tps":1104.2577949653712,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":632.1121985092759,"ttft_p95_ms":785.1605885778553,"ttft_p99_ms":789.6364544820972,"tpot_p50_ms":88.66255554115881,"tpot_p95_ms":95.89418757765202,"tpot_p99_ms":95.99700372289992,"e2e_p50_ms":45937.07379500847,"e2e_p95_ms":49456.24339070055,"e2e_p99_ms":49461.27653849078,"itl_p50_ms":87.81167201232165,"itl_p95_ms":96.0932505549863,"itl_p99_ms":97.09955179132521,"validation_errors":[],"timestamp":"2026-07-23T01:26:12+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":43.806384,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c32_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c32_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":264.71396346297115,"request_tps":0.9066389882132976,"input_tps":928.3983239304167,"output_tps":464.19916196520836,"total_tps":1392.5974858956251,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":964.9323470075615,"ttft_p95_ms":1221.9252238748595,"ttft_p99_ms":1240.2217320876662,"tpot_p50_ms":101.30938005300126,"tpot_p95_ms":103.89200894160008,"tpot_p99_ms":104.71317925374251,"e2e_p50_ms":52718.08856452117,"e2e_p95_ms":54053.23392060236,"e2e_p99_ms":54073.47771329689,"itl_p50_ms":101.32965806405991,"itl_p95_ms":104.51437833253294,"itl_p99_ms":106.37578951660544,"validation_errors":[],"timestamp":"2026-07-23T01:31:17+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":26.111628,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c48_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c48_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":273.4915737150004,"request_tps":1.17005432984003,"input_tps":1198.1356337561908,"output_tps":599.0678168780954,"total_tps":1797.2034506342861,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":1016.1562144639902,"ttft_p95_ms":1265.4756680713035,"ttft_p99_ms":1272.4853331770282,"tpot_p50_ms":105.18224471043195,"tpot_p95_ms":107.03515330095567,"tpot_p99_ms":107.27208556062652,"e2e_p50_ms":54675.60680198949,"e2e_p95_ms":55279.26374865929,"e2e_p99_ms":55332.57855539094,"itl_p50_ms":104.82980299275368,"itl_p95_ms":107.36952442675829,"itl_p99_ms":109.66756303329021,"validation_errors":[],"timestamp":"2026-07-23T01:36:27+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":29.05405,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c64_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c64_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":286.6468812410021,"request_tps":1.3954451493358295,"input_tps":1428.9358329198894,"output_tps":714.4679164599447,"total_tps":2143.403749379834,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":1278.7064270232804,"ttft_p95_ms":1602.201730379602,"ttft_p99_ms":1632.2937768895645,"tpot_p50_ms":111.01857787672496,"tpot_p95_ms":113.85904396877308,"tpot_p99_ms":114.30346369218225,"e2e_p50_ms":57741.44902906846,"e2e_p95_ms":58910.990245314315,"e2e_p99_ms":59101.83050505933,"itl_p50_ms":110.43362598866224,"itl_p95_ms":113.56170702492818,"itl_p99_ms":116.38755345484242,"validation_errors":[],"timestamp":"2026-07-23T01:41:56+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":19.263278,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c80_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c80_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":291.0035195989767,"request_tps":1.649464586069178,"input_tps":1689.0517361348382,"output_tps":844.5258680674191,"total_tps":2533.577604202257,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":1213.4021369274706,"ttft_p95_ms":1664.2729850602336,"ttft_p99_ms":1845.0450551777612,"tpot_p50_ms":110.90761146391058,"tpot_p95_ms":115.59953097436976,"tpot_p99_ms":116.09297388468809,"e2e_p50_ms":58069.17218101444,"e2e_p95_ms":59833.35679273005,"e2e_p99_ms":59844.78946550982,"itl_p50_ms":110.18450406845659,"itl_p95_ms":114.58867364563048,"itl_p99_ms":118.4574611741118,"validation_errors":[],"timestamp":"2026-07-23T01:47:31+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":18.20347,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c96_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c96_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":560,"failed":0,"duration_s":295.41844289994333,"request_tps":1.895616246916815,"input_tps":1941.1110368428185,"output_tps":970.5555184214093,"total_tps":2911.666555264228,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":1476.903619011864,"ttft_p95_ms":1741.1559896427207,"ttft_p99_ms":1891.4167065476067,"tpot_p50_ms":113.01586586686304,"tpot_p95_ms":115.23916551780863,"tpot_p99_ms":115.76414850658242,"e2e_p50_ms":59152.80058601638,"e2e_p95_ms":59671.71411286108,"e2e_p99_ms":59789.349250178784,"itl_p50_ms":112.2535599861294,"itl_p95_ms":115.00405773986131,"itl_p99_ms":120.01903203083202,"validation_errors":[],"timestamp":"2026-07-23T01:53:16+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":14.923125,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c112_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c112_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":640,"failed":0,"duration_s":308.64544958795886,"request_tps":2.0735766584422315,"input_tps":2123.342498244845,"output_tps":1061.6712491224225,"total_tps":3185.0137473672676,"mean_input_tokens":1024.0,"mean_output_tokens":512.0,"ttft_p50_ms":1639.3986864713952,"ttft_p95_ms":2276.824323134497,"ttft_p99_ms":2298.7781248858664,"tpot_p50_ms":117.84748917713456,"tpot_p95_ms":119.6933357496705,"tpot_p99_ms":120.5755311049295,"e2e_p50_ms":61583.21982098278,"e2e_p95_ms":62242.855491722,"e2e_p99_ms":62391.29308728385,"itl_p50_ms":116.52753595262766,"itl_p95_ms":120.08594442158937,"itl_p99_ms":125.61383717227726,"validation_errors":[],"timestamp":"2026-07-23T01:59:12+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":512,"concurrency":128,"num_prompts":640,"warmup_requests":128,"attempt":1,"gain_pct":9.387998,"plateau_streak":0,"error_type":"","raw_file":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/raw_outputs/vllm_adaptive_c128_i1024_o512_a1.jsonl","detail_log":"/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/tp4_dp2/logs/vllm_c128_i1024_o512_a1.log"} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/adaptive_shapes.jsonl b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/adaptive_shapes.jsonl new file mode 100644 index 0000000..25ca3a1 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/adaptive_shapes.jsonl @@ -0,0 +1,31 @@ +{"timestamp":"2026-07-22T09:43:41+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":6,"search_cap":256,"max_successful_concurrency":96,"saturation_concurrency":null,"stop_probe_concurrency":96,"best_tps_concurrency":96,"best_total_tps":6223.701501879885,"last_total_tps":6223.701501879885} +{"timestamp":"2026-07-22T10:04:20+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":7,"search_cap":256,"max_successful_concurrency":112,"saturation_concurrency":null,"stop_probe_concurrency":112,"best_tps_concurrency":112,"best_total_tps":4555.979139556733,"last_total_tps":4555.979139556733} +{"timestamp":"2026-07-22T10:45:57+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":8,"search_cap":256,"max_successful_concurrency":128,"saturation_concurrency":null,"stop_probe_concurrency":128,"best_tps_concurrency":128,"best_total_tps":3243.6752265172054,"last_total_tps":3243.6752265172054} +{"timestamp":"2026-07-22T12:01:59+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":8,"search_cap":256,"max_successful_concurrency":128,"saturation_concurrency":null,"stop_probe_concurrency":128,"best_tps_concurrency":128,"best_total_tps":2201.3357512672856,"last_total_tps":2201.3357512672856} +{"timestamp":"2026-07-22T13:47:31+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":6,"search_cap":256,"max_successful_concurrency":96,"saturation_concurrency":null,"stop_probe_concurrency":96,"best_tps_concurrency":96,"best_total_tps":1280.6780105069347,"last_total_tps":1280.6780105069347} +{"timestamp":"2026-07-22T15:21:04+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"status":"FAILED","stop_reason":"BENCHMARK_FAILED","tested_points":2,"search_cap":256,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":261.1100398514151,"last_total_tps":261.1100398514151} +{"timestamp":"2026-07-22T15:30:51+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":256,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":16,"best_total_tps":4815.634206058669,"last_total_tps":3371.9431739916763} +{"timestamp":"2026-07-22T15:36:24+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":256,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":32,"best_total_tps":4804.53423423964,"last_total_tps":4804.53423423964} +{"timestamp":"2026-07-22T15:45:20+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":256,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":32,"best_total_tps":2906.5299323319155,"last_total_tps":2906.5299323319155} +{"timestamp":"2026-07-22T16:01:22+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":256,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":32,"best_total_tps":1701.5268024317324,"last_total_tps":1701.5268024317324} +{"timestamp":"2026-07-22T16:49:09+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":256,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":1451.8367414069287,"last_total_tps":1451.8367414069287} +{"timestamp":"2026-07-22T18:22:58+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"status":"FAILED","stop_reason":"BENCHMARK_FAILED","tested_points":2,"search_cap":256,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":412.9113003030674,"last_total_tps":412.9113003030674} +{"timestamp":"2026-07-22T18:34:13+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":6906.82023315598,"last_total_tps":2029.852706187679} +{"timestamp":"2026-07-22T18:39:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":256,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":16,"best_total_tps":4985.240410512058,"last_total_tps":3717.9596225751343} +{"timestamp":"2026-07-22T18:47:04+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":256,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":16,"best_total_tps":3198.415829122864,"last_total_tps":2201.0987716139834} +{"timestamp":"2026-07-22T19:00:39+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":256,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":16,"best_total_tps":1792.0398358366053,"last_total_tps":1135.5456410595361} +{"timestamp":"2026-07-22T19:32:12+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":256,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":32,"best_total_tps":1636.9661906573656,"last_total_tps":1636.9661906573656} +{"timestamp":"2026-07-22T21:04:41+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":8192,"osl":4096,"status":"FAILED","stop_reason":"BENCHMARK_FAILED","tested_points":2,"search_cap":256,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":646.6409057070608,"last_total_tps":646.6409057070608} +{"timestamp":"2026-07-22T21:17:36+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":8682.018734573981,"last_total_tps":3437.7823824243033} +{"timestamp":"2026-07-22T21:25:11+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":7261.569932688289,"last_total_tps":2400.788179662987} +{"timestamp":"2026-07-22T21:35:33+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":5258.248246647036,"last_total_tps":1231.0301756990627} +{"timestamp":"2026-07-22T21:50:10+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":2,"search_cap":256,"max_successful_concurrency":8,"saturation_concurrency":null,"stop_probe_concurrency":8,"best_tps_concurrency":16,"best_total_tps":3020.7105850617963,"last_total_tps":2076.2679249187054} +{"timestamp":"2026-07-22T22:22:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_BACKOFF_FOUND","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":1678.9640708367808,"last_total_tps":341.40411555732345} +{"timestamp":"2026-07-22T22:31:23+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":10458.505477548737,"last_total_tps":4962.0674849195275} +{"timestamp":"2026-07-22T22:42:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":9115.224776663274,"last_total_tps":3515.3370833509175} +{"timestamp":"2026-07-22T22:57:03+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":6828.591594828798,"last_total_tps":2058.797214619256} +{"timestamp":"2026-07-22T23:18:34+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":4681.67949855934,"last_total_tps":1072.1657102655267} +{"timestamp":"2026-07-22T23:55:37+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":32768,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":2654.3671759568315,"last_total_tps":587.4731966035683} +{"timestamp":"2026-07-23T00:19:44+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_GROUP_SKIP_THRESHOLD","tested_points":3,"search_cap":256,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":6887.0910190312525,"last_total_tps":4597.173524079828} +{"timestamp":"2026-07-23T00:44:08+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":9,"search_cap":256,"max_successful_concurrency":144,"saturation_concurrency":null,"stop_probe_concurrency":144,"best_tps_concurrency":144,"best_total_tps":8994.329502615221,"last_total_tps":8994.329502615221} +{"timestamp":"2026-07-23T01:18:34+00:00","engine":"vllm","tp":4,"dp":2,"mark":"Y","isl":1024,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":11,"search_cap":256,"max_successful_concurrency":176,"saturation_concurrency":null,"stop_probe_concurrency":176,"best_tps_concurrency":176,"best_total_tps":6601.429920380257,"last_total_tps":6601.429920380257} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/run_manifest.json b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/run_manifest.json new file mode 100644 index 0000000..62151ce --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/run_manifest.json @@ -0,0 +1,27 @@ +{ + "experiment": "hy3_h20_vllm_tp_dp_matrix", + "engine": "vllm", + "run_id": "adaptive_20260722-092437", + "model": "/data2/models/Hy3-FP8", + "hardware": "8x NVIDIA H20 96GB", + "matrix": "/data1/qqt/sskj/experiments/h20/hy3_h20_vllm_tp_dp_matrix/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1.0, + "search": { + "start_concurrency": 16, + "max_concurrency": 256, + "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2.0, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 + } +} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/shapes.tsv b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/shapes.tsv new file mode 100644 index 0000000..3c03092 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260722-092437/shapes.tsv @@ -0,0 +1,36 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 8192 128 +Y 8192 256 +Y 8192 512 +Y 8192 1024 +Y 8192 2048 +Y 8192 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 32768 128 +Y 32768 256 +Y 32768 512 +Y 32768 1024 +Y 32768 2048 +Y 65536 128 +Y 65536 256 +Y 65536 512 +Y 65536 1024 +Y 131072 128 +Y 131072 256 +Y 131072 512 diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/compare.py b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/compare.py new file mode 100644 index 0000000..002fa0c --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/compare.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Cross TP×DP configuration comparison for dsv4_h20_vllm_tp_dp_matrix. + +Usage: + python3 compare.py --run-root results/ [--output comparison.md] +""" +import argparse +import json +import re +from collections import defaultdict +from pathlib import Path + + +def load_result(result_root: Path) -> dict: + path = result_root / "results.json" + with open(path, "r", encoding="utf-8") as f: + return json.load(f) + + +def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_ok = ttft_p95_ms < ttft_limit_ms + tpot_ok = tpot_mean_ms < tpot_limit_ms + if ttft_ok and tpot_ok: + return "PASS" + if ttft_ok or tpot_ok: + return "PARTIAL" + return "FAIL" + + +def gpu_memory_str(gpu: dict | None) -> str: + if not gpu: + return "-" + peak = gpu.get("peak_used_mb", 0) + total = gpu.get("memory_total_mb", 0) + if total: + return f"{peak:.0f}/{total:.0f} ({100*peak/total:.1f}%)" + return f"{peak:.0f}" + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--run-root", type=Path, required=True) + parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) + parser.add_argument("--ttft-limit", type=float, default=3000.0) + parser.add_argument("--tpot-limit", type=float, default=50.0) + args = parser.parse_args() + + # Discover configurations: tp*_dp* directories. + configs = [] + for subdir in sorted(args.run_root.iterdir()): + if not subdir.is_dir(): + continue + name = subdir.name + if not (name.startswith("tp") and "_dp" in name): + continue + results_json = subdir / "results.json" + if not results_json.exists(): + continue + configs.append((name, load_result(subdir))) + + if not configs: + print(f"No tp*_dp* results found under {args.run_root}") + return + + model = configs[0][1].get("metadata", {}).get("model", "unknown") + hardware = configs[0][1].get("metadata", {}).get("hardware", "unknown") + + # Group by scenario name. + by_scenario: dict[str, dict[str, dict]] = defaultdict(dict) + skipped: dict[str, dict[str, str]] = defaultdict(dict) + for label, data in configs: + for s in data.get("scenarios", []): + key = s["name"] + if s.get("status") == "skipped_oom": + skipped[key][label] = s.get("note", "skipped") + else: + by_scenario[key][label] = s + + with open(args.output, "w", encoding="utf-8") as f: + f.write(f"# vLLM TP×DP matrix comparison ({hardware})\n\n") + f.write("## Summary\n\n") + f.write(f"- Model: `{model}`\n") + f.write(f"- Hardware: {hardware}\n") + f.write("- Backend: vLLM (Docker)\n") + f.write("- Benchmark client: `sglang.bench_serving`\n") + f.write(f"- SLO reference: TTFT P95 < {args.ttft_limit}ms, TPOT mean < {args.tpot_limit}ms\n\n") + + # Configuration overview. + f.write("### Configurations\n\n") + f.write("| Config | TP | DP | GPUs/replica | Notes |\n") + f.write("|---|---:|---:|---:|---|\n") + for label, data in configs: + cfg = data.get("config", {}) + tp = cfg.get("tp", "?") + dp = cfg.get("dp", "?") + f.write(f"| {label} | {tp} | {dp} | {tp} | server args recorded per ISL in results.json |\n") + f.write("\n") + + # Side-by-side table. + f.write("## Side-by-side results\n\n") + headers = [ + "Scenario", "ISL", "OSL", "Config", "Conc", "Req/s", "OutTok/s", + "TTFT P95(ms)", "TTFT P99(ms)", "TPOT Mean(ms)", "TPOT P95(ms)", + "TPOT P99(ms)", "E2E P99(ms)", "Peak GPU mem", "SLO" + ] + f.write("| " + " | ".join(headers) + " |\n") + f.write("|" + "|".join(["---"] * len(headers)) + "|\n") + + for scenario_name in sorted(by_scenario.keys(), key=lambda x: tuple(map(int, re.findall(r"\d+", x)))): + _, isl, osl = re.findall(r"\d+", scenario_name) + # cfg_part not used; just for readability. + for label, data in configs: + s = by_scenario[scenario_name].get(label) + if s is None: + if scenario_name in skipped and label in skipped[scenario_name]: + note = skipped[scenario_name][label] + f.write(f"| {scenario_name} | {isl} | {osl} | {label} | - | - | - | - | - | - | - | - | - | - | {note} |\n") + continue + cfg = s["config"] + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + gpu = m.get("gpu_memory") + f.write( + f"| {scenario_name} | {isl} | {osl} | {label} | {cfg['concurrency']} | " + f"{m['request_throughput']:.2f} | {m['output_token_throughput']:.2f} | " + f"{m['ttft_ms']['p95']:.2f} | {m['ttft_ms']['p99']:.2f} | " + f"{m['tpot_ms']['mean']:.2f} | {m['tpot_ms']['p95']:.2f} | {m['tpot_ms']['p99']:.2f} | " + f"{m['e2e_ms']['p99']:.2f} | {gpu_memory_str(gpu)} | {status} |\n" + ) + + # Best throughput per ISL/OSL. + f.write("\n## Best throughput per (ISL, OSL)\n\n") + f.write("| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO |\n") + f.write("|---:|---:|---|---:|---:|---:|---:|---:|\n") + best_by_shape: dict[tuple[int, int], tuple[float, str, dict]] = {} + for scenario_name, backends in by_scenario.items(): + _, isl, osl = re.findall(r"\d+", scenario_name) + isl_i, osl_i = int(isl), int(osl) + for label, s in backends.items(): + m = s["metrics"] + out_tok = m["output_token_throughput"] + if (isl_i, osl_i) not in best_by_shape or out_tok > best_by_shape[(isl_i, osl_i)][0]: + best_by_shape[(isl_i, osl_i)] = (out_tok, label, s) + for (isl_i, osl_i), (out_tok, label, s) in sorted(best_by_shape.items()): + m = s["metrics"] + status = slo_status(m["ttft_ms"]["p95"], m["tpot_ms"]["mean"], args.ttft_limit, args.tpot_limit) + f.write( + f"| {isl_i} | {osl_i} | {label} | {s['config']['concurrency']} | " + f"{out_tok:.2f} | {m['ttft_ms']['p95']:.2f} | {m['tpot_ms']['mean']:.2f} | {status} |\n" + ) + + f.write("\n## Notes\n\n") + f.write("- SLO check uses TTFT P95 and TPOT mean.\n") + f.write("- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out.\n") + f.write("- `Peak GPU mem` shows peak used / total MB and utilization percentage.\n") + f.write("- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run.\n") + + print(f"Wrote comparison to {args.output}") + + +if __name__ == "__main__": + main() diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/config.env b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/config.env new file mode 100644 index 0000000..4d8610c --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/config.env @@ -0,0 +1,76 @@ +# TP×DP matrix experiment for Hy3-FP8 on H20 (8 GPUs) using vLLM. +# Tests vLLM with two memory-feasible parallel configurations: +# TP=4, DP=2 -> 4 GPUs per replica, 2 replicas +# TP=8, DP=1 -> 8 GPUs, no data parallelism + +EXPERIMENT="hy3_h20_vllm_tp_dp_matrix" +MODEL_NAME="Hy3-FP8" +MODEL_PATH="/data2/models/Hy3-FP8" +SERVED_MODEL_NAME="hy3" + +VLLM_PORT="${VLLM_PORT:-30030}" + +# Python interpreter for orchestration scripts (parse_backend.py, compare.py, etc.) +# and the benchmark client. Defaults to the system python3 if the sglang venv +# does not exist on the host. +VENV_CLIENT="${VENV_CLIENT:-${ROOT_DIR}/envs/sglang}" + +# Run the benchmark client natively (0) or inside Docker (1). +USE_DOCKER_CLIENT="${USE_DOCKER_CLIENT:-1}" + +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7}" +export VLLM_FLASHINFER_ALLREDUCE_BACKEND="${VLLM_FLASHINFER_ALLREDUCE_BACKEND:-trtllm}" + +# Runtime working directory for logs, pid files, and tmp. Defaults to a local +# directory under this experiment so the benchmark is self-contained. +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" + +# Parallel configurations to test. Format: "TP DP" +declare -a PARALLEL_CONFIGS=( + "2 4" + "4 2" + "8 1" +) + +# vLLM server settings adapted for Hy3-FP8 on H20. +GPU_MEMORY_UTILIZATION="${GPU_MEMORY_UTILIZATION:-0.9}" +KV_CACHE_DTYPE="${KV_CACHE_DTYPE:-fp8}" +BLOCK_SIZE="${BLOCK_SIZE:-256}" +MAX_MODEL_LEN="${MAX_MODEL_LEN:-262144}" +MAX_NUM_SEQS="${MAX_NUM_SEQS:-256}" + +# Deployment switch. 0 = native vllm venv, 1 = Docker. +USE_DOCKER="${USE_DOCKER:-1}" +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" + +# Benchmark client Docker image. vLLM's image does not include sglang.bench_serving, +# so the client runs inside the SGLang image and targets the vLLM backend via the +# OpenAI-compatible API. +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +# Dataset used by sglang.bench_serving --dataset-name random. +# The random sampler needs a ShareGPT-style JSON file locally; it falls back to +# downloading from HuggingFace, which usually fails on offline H20 nodes. +DATASET_PATH="${DATASET_PATH:-${ROOT_DIR}/datasets/ShareGPT_V3_unfiltered_cleaned_split.json}" + +# Matrix and concurrency rules are defined in matrix.json by default. +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR:-.}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" + +# Sampling density for concurrency. +# 0 = use the default heuristic in generate_scenarios.py (6-8 points). +# 2 = only test the low and high endpoints. +export CONCURRENCY_SAMPLES="${CONCURRENCY_SAMPLES:-2}" + +# Per-scenario timeout to avoid hangs (seconds). +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" + +# GPU memory sampling interval (seconds). +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" + +# Dry-run mode: if 1, only log the server args and scenario plan without starting +# any server or sending requests. +DRY_RUN="${DRY_RUN:-0}" + +# Per-config scenario limit for quick smoke tests. 0 = run all generated scenarios. +GRID_LIMIT="${GRID_LIMIT:-0}" diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/generate_scenarios.py b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/generate_scenarios.py new file mode 100644 index 0000000..2bfb152 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/generate_scenarios.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +"""Generate the scenario list for the TP×DP matrix experiment. + +Reads matrix.json and prints TSV lines: + + mark input_len output_len concurrency num_prompts + +mark is one of Y/P/N. The caller (run_bench.sh) decides how to treat each. +""" +import argparse +import json +import math +import os +from pathlib import Path + + +def sample_concurrency(low: int, high: int, target: int) -> list[int]: + """Return only the low and high concurrency values in [low, high]. + + For the TP×DP matrix we only need the two endpoints of the concurrency + range (e.g. 1 and 128 for ISL=1024). The `target` argument is kept for + API compatibility but is ignored. + """ + assert 1 <= low <= high, f"invalid concurrency range: {low}-{high}" + if low == high: + return [low] + return [low, high] + + +def generate_scenarios(matrix_path: Path, mode: str, target_samples: int) -> list[dict]: + with open(matrix_path, "r", encoding="utf-8") as f: + data = json.load(f) + + matrix = data["matrix"] + concurrency_cfg = data["concurrency"] + + scenarios = [] + for isl_str in sorted(matrix.keys(), key=int): + osl_map = matrix[isl_str] + low = concurrency_cfg[isl_str]["low"] + high = concurrency_cfg[isl_str]["high"] + concurrencies = sample_concurrency(low, high, target_samples) + + for osl_str in sorted(osl_map.keys(), key=int): + mark = osl_map[osl_str] + if mode == "Y" and mark != "Y": + continue + if mode == "Y+P" and mark not in ("Y", "P"): + continue + # mode == "all" keeps everything, including N. + + for conc in concurrencies: + scenarios.append( + { + "mark": mark, + "input_len": int(isl_str), + "output_len": int(osl_str), + "concurrency": conc, + "num_prompts": conc * 5, + } + ) + + return scenarios + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--matrix", type=Path, default=Path("matrix.json")) + parser.add_argument("--mode", choices=["Y", "Y+P", "all"], default=None, + help="Scenario selection mode. Defaults to matrix.mode.") + parser.add_argument("--target-samples", type=int, default=0, + help="Target number of concurrency samples. 0 = heuristic (6-8).") + args = parser.parse_args() + + with open(args.matrix, "r", encoding="utf-8") as f: + data = json.load(f) + + mode = args.mode if args.mode else data.get("mode", "Y+P") + + target_samples = args.target_samples + if target_samples <= 0: + env_samples = os.getenv("CONCURRENCY_SAMPLES", "0") + try: + target_samples = int(env_samples) + except ValueError: + target_samples = 0 + if target_samples <= 0: + target_samples = 7 + + scenarios = generate_scenarios(args.matrix, mode, target_samples) + + print("mark\tinput_len\toutput_len\tconcurrency\tnum_prompts") + for s in scenarios: + print(f"{s['mark']}\t{s['input_len']}\t{s['output_len']}\t{s['concurrency']}\t{s['num_prompts']}") + + +if __name__ == "__main__": + main() diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/matrix.json b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/matrix.json new file mode 100644 index 0000000..2e1f10f --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/matrix.json @@ -0,0 +1,92 @@ +{ + "comment": "Hy3 supports a 256K total context window. ISL >= 262144 is excluded because input plus output must remain within 262144 tokens.", + "mode": "Y", + "matrix": { + "1024": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "4096": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "8192": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "Y" + }, + "16384": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "32768": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "Y", + "4096": "P" + }, + "65536": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "Y", + "2048": "P", + "4096": "N" + }, + "131072": { + "128": "Y", + "256": "Y", + "512": "Y", + "1024": "P", + "2048": "N", + "4096": "N" + } + }, + "concurrency": { + "1024": { + "low": 1, + "high": 128 + }, + "4096": { + "low": 1, + "high": 64 + }, + "8192": { + "low": 1, + "high": 64 + }, + "16384": { + "low": 1, + "high": 32 + }, + "32768": { + "low": 1, + "high": 16 + }, + "65536": { + "low": 1, + "high": 8 + }, + "131072": { + "low": 1, + "high": 4 + } + } +} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/comparison.md b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/comparison.md new file mode 100644 index 0000000..5934b39 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/comparison.md @@ -0,0 +1,34 @@ +# vLLM TP×DP matrix comparison (8x NVIDIA H20 96GB) + +## Summary + +- Model: `/data2/models/Hy3-FP8` +- Hardware: 8x NVIDIA H20 96GB +- Backend: vLLM (Docker) +- Benchmark client: `sglang.bench_serving` +- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms + +### Configurations + +| Config | TP | DP | GPUs/replica | Notes | +|---|---:|---:|---:|---| +| tp2_dp4 | 2 | 4 | 2 | server args recorded per ISL in results.json | +| tp4_dp2 | 4 | 2 | 4 | server args recorded per ISL in results.json | +| tp8_dp1 | 8 | 1 | 8 | server args recorded per ISL in results.json | + +## Side-by-side results + +| Scenario | ISL | OSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| + +## Best throughput per (ISL, OSL) + +| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO | +|---:|---:|---|---:|---:|---:|---:|---:| + +## Notes + +- SLO check uses TTFT P95 and TPOT mean. +- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out. +- `Peak GPU mem` shows peak used / total MB and utilization percentage. +- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run. diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp2_dp4/results.json b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp2_dp4/results.json new file mode 100644 index 0000000..a0efc88 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp2_dp4/results.json @@ -0,0 +1,27 @@ +{ + "metadata": { + "experiment": "hy3_h20_vllm_tp_dp_matrix_tp2_dp4", + "run_id": "20260722-084347", + "timestamp": "2026-07-22T08:43:54+00:00", + "model": "/data2/models/Hy3-FP8", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H20 96GB", + "accelerator": "NVIDIA H20", + "chip": "nvidia_h20", + "script": "experiments/hy3_h20_vllm_tp_dp_matrix/run_bench.sh", + "env": "vllm/vllm-openai:latest", + "git_commit": "630a102", + "git_dirty": "dirty", + "description": "H20 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 2, + "dp": 4, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data2/models/Hy3-FP8 --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 2 --gpu-memory-utilization 0.9 --max-model-len 1048576 --max-num-seqs 256 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 --data-parallel-size 4 " + }, + "scenarios": [] +} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp2_dp4/scenarios.tsv b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp2_dp4/scenarios.tsv new file mode 100644 index 0000000..70b2ee8 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp2_dp4/scenarios.tsv @@ -0,0 +1,79 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 8192 128 1 5 +Y 8192 128 64 320 +Y 8192 256 1 5 +Y 8192 256 64 320 +Y 8192 512 1 5 +Y 8192 512 64 320 +Y 8192 1024 1 5 +Y 8192 1024 64 320 +Y 8192 2048 1 5 +Y 8192 2048 64 320 +Y 8192 4096 1 5 +Y 8192 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 32768 128 1 5 +Y 32768 128 16 80 +Y 32768 256 1 5 +Y 32768 256 16 80 +Y 32768 512 1 5 +Y 32768 512 16 80 +Y 32768 1024 1 5 +Y 32768 1024 16 80 +Y 32768 2048 1 5 +Y 32768 2048 16 80 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 +Y 262144 128 1 5 +Y 262144 128 2 10 +Y 262144 256 1 5 +Y 262144 256 2 10 +Y 524288 128 1 5 +Y 524288 128 2 10 +Y 1048576 128 1 5 +Y 1048576 128 2 10 diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp4_dp2/results.json b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp4_dp2/results.json new file mode 100644 index 0000000..4bfa4df --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp4_dp2/results.json @@ -0,0 +1,27 @@ +{ + "metadata": { + "experiment": "hy3_h20_vllm_tp_dp_matrix_tp4_dp2", + "run_id": "20260722-084347", + "timestamp": "2026-07-22T08:43:54+00:00", + "model": "/data2/models/Hy3-FP8", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H20 96GB", + "accelerator": "NVIDIA H20", + "chip": "nvidia_h20", + "script": "experiments/hy3_h20_vllm_tp_dp_matrix/run_bench.sh", + "env": "vllm/vllm-openai:latest", + "git_commit": "630a102", + "git_dirty": "dirty", + "description": "H20 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 4, + "dp": 2, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data2/models/Hy3-FP8 --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 4 --gpu-memory-utilization 0.9 --max-model-len 1048576 --max-num-seqs 256 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 --data-parallel-size 2 " + }, + "scenarios": [] +} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp4_dp2/scenarios.tsv b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp4_dp2/scenarios.tsv new file mode 100644 index 0000000..70b2ee8 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp4_dp2/scenarios.tsv @@ -0,0 +1,79 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 8192 128 1 5 +Y 8192 128 64 320 +Y 8192 256 1 5 +Y 8192 256 64 320 +Y 8192 512 1 5 +Y 8192 512 64 320 +Y 8192 1024 1 5 +Y 8192 1024 64 320 +Y 8192 2048 1 5 +Y 8192 2048 64 320 +Y 8192 4096 1 5 +Y 8192 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 32768 128 1 5 +Y 32768 128 16 80 +Y 32768 256 1 5 +Y 32768 256 16 80 +Y 32768 512 1 5 +Y 32768 512 16 80 +Y 32768 1024 1 5 +Y 32768 1024 16 80 +Y 32768 2048 1 5 +Y 32768 2048 16 80 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 +Y 262144 128 1 5 +Y 262144 128 2 10 +Y 262144 256 1 5 +Y 262144 256 2 10 +Y 524288 128 1 5 +Y 524288 128 2 10 +Y 1048576 128 1 5 +Y 1048576 128 2 10 diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp8_dp1/results.json b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp8_dp1/results.json new file mode 100644 index 0000000..e44c34c --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp8_dp1/results.json @@ -0,0 +1,27 @@ +{ + "metadata": { + "experiment": "hy3_h20_vllm_tp_dp_matrix_tp8_dp1", + "run_id": "20260722-084347", + "timestamp": "2026-07-22T08:43:54+00:00", + "model": "/data2/models/Hy3-FP8", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H20 96GB", + "accelerator": "NVIDIA H20", + "chip": "nvidia_h20", + "script": "experiments/hy3_h20_vllm_tp_dp_matrix/run_bench.sh", + "env": "vllm/vllm-openai:latest", + "git_commit": "630a102", + "git_dirty": "dirty", + "description": "H20 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "dp": 1, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data2/models/Hy3-FP8 --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 8 --gpu-memory-utilization 0.9 --max-model-len 1048576 --max-num-seqs 256 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 " + }, + "scenarios": [] +} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp8_dp1/scenarios.tsv b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp8_dp1/scenarios.tsv new file mode 100644 index 0000000..70b2ee8 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-084347/tp8_dp1/scenarios.tsv @@ -0,0 +1,79 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 8192 128 1 5 +Y 8192 128 64 320 +Y 8192 256 1 5 +Y 8192 256 64 320 +Y 8192 512 1 5 +Y 8192 512 64 320 +Y 8192 1024 1 5 +Y 8192 1024 64 320 +Y 8192 2048 1 5 +Y 8192 2048 64 320 +Y 8192 4096 1 5 +Y 8192 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 32768 128 1 5 +Y 32768 128 16 80 +Y 32768 256 1 5 +Y 32768 256 16 80 +Y 32768 512 1 5 +Y 32768 512 16 80 +Y 32768 1024 1 5 +Y 32768 1024 16 80 +Y 32768 2048 1 5 +Y 32768 2048 16 80 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 +Y 262144 128 1 5 +Y 262144 128 2 10 +Y 262144 256 1 5 +Y 262144 256 2 10 +Y 524288 128 1 5 +Y 524288 128 2 10 +Y 1048576 128 1 5 +Y 1048576 128 2 10 diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/comparison.md b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/comparison.md new file mode 100644 index 0000000..6072b2b --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/comparison.md @@ -0,0 +1,33 @@ +# vLLM TP×DP matrix comparison (8x NVIDIA H20 96GB) + +## Summary + +- Model: `/data2/models/Hy3-FP8` +- Hardware: 8x NVIDIA H20 96GB +- Backend: vLLM (Docker) +- Benchmark client: `sglang.bench_serving` +- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms + +### Configurations + +| Config | TP | DP | GPUs/replica | Notes | +|---|---:|---:|---:|---| +| tp4_dp2 | 4 | 2 | 4 | server args recorded per ISL in results.json | +| tp8_dp1 | 8 | 1 | 8 | server args recorded per ISL in results.json | + +## Side-by-side results + +| Scenario | ISL | OSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| + +## Best throughput per (ISL, OSL) + +| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO | +|---:|---:|---|---:|---:|---:|---:|---:| + +## Notes + +- SLO check uses TTFT P95 and TPOT mean. +- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out. +- `Peak GPU mem` shows peak used / total MB and utilization percentage. +- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run. diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp4_dp2/results.json b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp4_dp2/results.json new file mode 100644 index 0000000..ba37745 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp4_dp2/results.json @@ -0,0 +1,27 @@ +{ + "metadata": { + "experiment": "hy3_h20_vllm_tp_dp_matrix_tp4_dp2", + "run_id": "20260722-085021", + "timestamp": "2026-07-22T08:50:26+00:00", + "model": "/data2/models/Hy3-FP8", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H20 96GB", + "accelerator": "NVIDIA H20", + "chip": "nvidia_h20", + "script": "experiments/hy3_h20_vllm_tp_dp_matrix/run_bench.sh", + "env": "vllm/vllm-openai:latest", + "git_commit": "630a102", + "git_dirty": "dirty", + "description": "H20 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 4, + "dp": 2, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data2/models/Hy3-FP8 --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 4 --gpu-memory-utilization 0.9 --max-model-len 262144 --max-num-seqs 256 --speculative-config.method mtp --speculative-config.num_speculative_tokens 2 --tool-call-parser hy_v3 --reasoning-parser hy_v3 --enable-auto-tool-choice --served-model-name Hy3-FP8 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 --data-parallel-size 2 " + }, + "scenarios": [] +} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp4_dp2/scenarios.tsv b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp4_dp2/scenarios.tsv new file mode 100644 index 0000000..9e3c6c4 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp4_dp2/scenarios.tsv @@ -0,0 +1,71 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 8192 128 1 5 +Y 8192 128 64 320 +Y 8192 256 1 5 +Y 8192 256 64 320 +Y 8192 512 1 5 +Y 8192 512 64 320 +Y 8192 1024 1 5 +Y 8192 1024 64 320 +Y 8192 2048 1 5 +Y 8192 2048 64 320 +Y 8192 4096 1 5 +Y 8192 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 32768 128 1 5 +Y 32768 128 16 80 +Y 32768 256 1 5 +Y 32768 256 16 80 +Y 32768 512 1 5 +Y 32768 512 16 80 +Y 32768 1024 1 5 +Y 32768 1024 16 80 +Y 32768 2048 1 5 +Y 32768 2048 16 80 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp8_dp1/results.json b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp8_dp1/results.json new file mode 100644 index 0000000..57a42f5 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp8_dp1/results.json @@ -0,0 +1,27 @@ +{ + "metadata": { + "experiment": "hy3_h20_vllm_tp_dp_matrix_tp8_dp1", + "run_id": "20260722-085021", + "timestamp": "2026-07-22T08:50:26+00:00", + "model": "/data2/models/Hy3-FP8", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H20 96GB", + "accelerator": "NVIDIA H20", + "chip": "nvidia_h20", + "script": "experiments/hy3_h20_vllm_tp_dp_matrix/run_bench.sh", + "env": "vllm/vllm-openai:latest", + "git_commit": "630a102", + "git_dirty": "dirty", + "description": "H20 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "dp": 1, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data2/models/Hy3-FP8 --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 8 --gpu-memory-utilization 0.9 --max-model-len 262144 --max-num-seqs 256 --speculative-config.method mtp --speculative-config.num_speculative_tokens 2 --tool-call-parser hy_v3 --reasoning-parser hy_v3 --enable-auto-tool-choice --served-model-name Hy3-FP8 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 " + }, + "scenarios": [] +} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp8_dp1/scenarios.tsv b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp8_dp1/scenarios.tsv new file mode 100644 index 0000000..9e3c6c4 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085021/tp8_dp1/scenarios.tsv @@ -0,0 +1,71 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 8192 128 1 5 +Y 8192 128 64 320 +Y 8192 256 1 5 +Y 8192 256 64 320 +Y 8192 512 1 5 +Y 8192 512 64 320 +Y 8192 1024 1 5 +Y 8192 1024 64 320 +Y 8192 2048 1 5 +Y 8192 2048 64 320 +Y 8192 4096 1 5 +Y 8192 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 32768 128 1 5 +Y 32768 128 16 80 +Y 32768 256 1 5 +Y 32768 256 16 80 +Y 32768 512 1 5 +Y 32768 512 16 80 +Y 32768 1024 1 5 +Y 32768 1024 16 80 +Y 32768 2048 1 5 +Y 32768 2048 16 80 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/comparison.md b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/comparison.md new file mode 100644 index 0000000..6072b2b --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/comparison.md @@ -0,0 +1,33 @@ +# vLLM TP×DP matrix comparison (8x NVIDIA H20 96GB) + +## Summary + +- Model: `/data2/models/Hy3-FP8` +- Hardware: 8x NVIDIA H20 96GB +- Backend: vLLM (Docker) +- Benchmark client: `sglang.bench_serving` +- SLO reference: TTFT P95 < 3000.0ms, TPOT mean < 50.0ms + +### Configurations + +| Config | TP | DP | GPUs/replica | Notes | +|---|---:|---:|---:|---| +| tp4_dp2 | 4 | 2 | 4 | server args recorded per ISL in results.json | +| tp8_dp1 | 8 | 1 | 8 | server args recorded per ISL in results.json | + +## Side-by-side results + +| Scenario | ISL | OSL | Config | Conc | Req/s | OutTok/s | TTFT P95(ms) | TTFT P99(ms) | TPOT Mean(ms) | TPOT P95(ms) | TPOT P99(ms) | E2E P99(ms) | Peak GPU mem | SLO | +|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| + +## Best throughput per (ISL, OSL) + +| ISL | OSL | Best Config | Concurrency | OutTok/s | TTFT P95(ms) | TPOT Mean(ms) | SLO | +|---:|---:|---|---:|---:|---:|---:|---:| + +## Notes + +- SLO check uses TTFT P95 and TPOT mean. +- A PARTIAL indicates one of the two metrics is out of target; FAIL indicates both are out. +- `Peak GPU mem` shows peak used / total MB and utilization percentage. +- Optional (P) combinations that failed are marked as skipped/OOM and do not break the run. diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp4_dp2/results.json b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp4_dp2/results.json new file mode 100644 index 0000000..7097ae7 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp4_dp2/results.json @@ -0,0 +1,27 @@ +{ + "metadata": { + "experiment": "hy3_h20_vllm_tp_dp_matrix_tp4_dp2", + "run_id": "20260722-085141", + "timestamp": "2026-07-22T08:51:46+00:00", + "model": "/data2/models/Hy3-FP8", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H20 96GB", + "accelerator": "NVIDIA H20", + "chip": "nvidia_h20", + "script": "experiments/hy3_h20_vllm_tp_dp_matrix/run_bench.sh", + "env": "vllm/vllm-openai:latest", + "git_commit": "630a102", + "git_dirty": "dirty", + "description": "H20 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 4, + "dp": 2, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data2/models/Hy3-FP8 --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 4 --gpu-memory-utilization 0.9 --max-model-len 262144 --max-num-seqs 256 --speculative-config.method mtp --speculative-config.num_speculative_tokens 2 --tool-call-parser hy_v3 --reasoning-parser hy_v3 --enable-auto-tool-choice --served-model-name hy3 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 --data-parallel-size 2 " + }, + "scenarios": [] +} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp4_dp2/scenarios.tsv b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp4_dp2/scenarios.tsv new file mode 100644 index 0000000..9e3c6c4 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp4_dp2/scenarios.tsv @@ -0,0 +1,71 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 8192 128 1 5 +Y 8192 128 64 320 +Y 8192 256 1 5 +Y 8192 256 64 320 +Y 8192 512 1 5 +Y 8192 512 64 320 +Y 8192 1024 1 5 +Y 8192 1024 64 320 +Y 8192 2048 1 5 +Y 8192 2048 64 320 +Y 8192 4096 1 5 +Y 8192 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 32768 128 1 5 +Y 32768 128 16 80 +Y 32768 256 1 5 +Y 32768 256 16 80 +Y 32768 512 1 5 +Y 32768 512 16 80 +Y 32768 1024 1 5 +Y 32768 1024 16 80 +Y 32768 2048 1 5 +Y 32768 2048 16 80 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp8_dp1/results.json b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp8_dp1/results.json new file mode 100644 index 0000000..05ab5fd --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp8_dp1/results.json @@ -0,0 +1,27 @@ +{ + "metadata": { + "experiment": "hy3_h20_vllm_tp_dp_matrix_tp8_dp1", + "run_id": "20260722-085141", + "timestamp": "2026-07-22T08:51:46+00:00", + "model": "/data2/models/Hy3-FP8", + "backend": "vllm", + "engine": "vllm", + "hardware": "8x NVIDIA H20 96GB", + "accelerator": "NVIDIA H20", + "chip": "nvidia_h20", + "script": "experiments/hy3_h20_vllm_tp_dp_matrix/run_bench.sh", + "env": "vllm/vllm-openai:latest", + "git_commit": "630a102", + "git_dirty": "dirty", + "description": "H20 vLLM TP×DP matrix for DeepSeek-V4-Flash" + }, + "config": { + "tp": 8, + "dp": 1, + "cuda_visible_devices": "0,1,2,3,4,5,6,7", + "backend": "vllm", + "server_start_script": "experiments/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh", + "server_args": "vllm serve /data2/models/Hy3-FP8 --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 8 --gpu-memory-utilization 0.9 --max-model-len 262144 --max-num-seqs 256 --speculative-config.method mtp --speculative-config.num_speculative_tokens 2 --tool-call-parser hy_v3 --reasoning-parser hy_v3 --enable-auto-tool-choice --served-model-name hy3 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 " + }, + "scenarios": [] +} diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp8_dp1/scenarios.tsv b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp8_dp1/scenarios.tsv new file mode 100644 index 0000000..9e3c6c4 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/results/20260722-085141/tp8_dp1/scenarios.tsv @@ -0,0 +1,71 @@ +mark input_len output_len concurrency num_prompts +Y 1024 128 1 5 +Y 1024 128 128 640 +Y 1024 256 1 5 +Y 1024 256 128 640 +Y 1024 512 1 5 +Y 1024 512 128 640 +Y 1024 1024 1 5 +Y 1024 1024 128 640 +Y 1024 2048 1 5 +Y 1024 2048 128 640 +Y 1024 4096 1 5 +Y 1024 4096 128 640 +Y 4096 128 1 5 +Y 4096 128 64 320 +Y 4096 256 1 5 +Y 4096 256 64 320 +Y 4096 512 1 5 +Y 4096 512 64 320 +Y 4096 1024 1 5 +Y 4096 1024 64 320 +Y 4096 2048 1 5 +Y 4096 2048 64 320 +Y 4096 4096 1 5 +Y 4096 4096 64 320 +Y 8192 128 1 5 +Y 8192 128 64 320 +Y 8192 256 1 5 +Y 8192 256 64 320 +Y 8192 512 1 5 +Y 8192 512 64 320 +Y 8192 1024 1 5 +Y 8192 1024 64 320 +Y 8192 2048 1 5 +Y 8192 2048 64 320 +Y 8192 4096 1 5 +Y 8192 4096 64 320 +Y 16384 128 1 5 +Y 16384 128 32 160 +Y 16384 256 1 5 +Y 16384 256 32 160 +Y 16384 512 1 5 +Y 16384 512 32 160 +Y 16384 1024 1 5 +Y 16384 1024 32 160 +Y 16384 2048 1 5 +Y 16384 2048 32 160 +Y 32768 128 1 5 +Y 32768 128 16 80 +Y 32768 256 1 5 +Y 32768 256 16 80 +Y 32768 512 1 5 +Y 32768 512 16 80 +Y 32768 1024 1 5 +Y 32768 1024 16 80 +Y 32768 2048 1 5 +Y 32768 2048 16 80 +Y 65536 128 1 5 +Y 65536 128 8 40 +Y 65536 256 1 5 +Y 65536 256 8 40 +Y 65536 512 1 5 +Y 65536 512 8 40 +Y 65536 1024 1 5 +Y 65536 1024 8 40 +Y 131072 128 1 5 +Y 131072 128 4 20 +Y 131072 256 1 5 +Y 131072 256 4 20 +Y 131072 512 1 5 +Y 131072 512 4 20 diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh new file mode 100755 index 0000000..f76a93d --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_adaptive_concurrency.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$DOCKER_CLIENT_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +adaptive_main "$@" diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh new file mode 100755 index 0000000..fb18812 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -0,0 +1,173 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$DOCKER_CLIENT_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 +# If the initial concurrency violates the TTFT SLO, search downward. Stop at +# the first acceptable value (16 -> 8; only try 1 when 8 still violates it). +export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1" +# When concurrency 1 still has a severely excessive TTFT, stop the remaining +# shapes in this TP/DP group. Zero disables this rule. +export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}" + +adaptive_main "$@" diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_bench.sh b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_bench.sh new file mode 100755 index 0000000..f5755af --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/run_bench.sh @@ -0,0 +1,548 @@ +#!/usr/bin/env bash +# TP×DP matrix benchmark for Hy3-FP8 on vLLM (Docker). +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +# Export variables used inside functions that are called via bash -c subshells. +export DATASET_PATH MODEL_PATH RESULT_BASE \ + DOCKER_IMAGE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT + +RUN_ID="${RUN_ID:-$(date '+%Y%m%d-%H%M%S')}" +RESULT_BASE="${SCRIPT_DIR}/results" +MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" +MATRIX_MODE="${MATRIX_MODE:-Y}" +SCENARIO_TIMEOUT_S="${SCENARIO_TIMEOUT_S:-1800}" +GPU_MEM_SAMPLE_INTERVAL_S="${GPU_MEM_SAMPLE_INTERVAL_S:-1}" +DRY_RUN="${DRY_RUN:-0}" +GRID_LIMIT="${GRID_LIMIT:-0}" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +log_dir_global="${RESULT_BASE}/${RUN_ID}/logs" +mkdir -p "$log_dir_global" +log_init "${log_dir_global}/orchestrator.log" + +log "experiment=${EXPERIMENT_NAME} run_id=${RUN_ID} platform=${PLATFORM} hardware=${HARDWARE}" +log "matrix_mode=${MATRIX_MODE} matrix_file=${MATRIX_FILE} dry_run=${DRY_RUN} grid_limit=${GRID_LIMIT}" + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +is_server_healthy() { + curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1 +} + +stop_server() { + local tp="$1" + local dp="$2" + + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} (tp=${tp}, dp=${dp})" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + + # Fallback: remove any Docker container started by this experiment. + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + + # Fallback: kill any vllm serve processes for this model. + pkill -9 -f "vllm serve.*${MODEL_NAME}" 2>/dev/null || true + pkill -9 -f "vllm serve.*${MODEL_PATH}" 2>/dev/null || true + sleep 2 +} + +build_server_args() { + local tp="$1" + local dp="$2" + + local args=( + "vllm serve" "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$VLLM_PORT" + ) + if [[ "$dp" -gt 1 ]]; then + args+=( + --data-parallel-size "$dp" + ) + fi + printf '%s ' "${args[@]}" +} + +start_server() { + local tp="$1" + local dp="$2" + + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" \ + >> "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log" 2>&1 + + if ! is_server_healthy; then + log "error: vllm server tp=${tp} dp=${dp} failed health check on port ${VLLM_PORT}" + return 1 + fi + log "vllm server tp=${tp} dp=${dp} is healthy on port ${VLLM_PORT}" +} + +restart_server() { + local tp="$1" + local dp="$2" + log "restarting vllm server tp=${tp} dp=${dp} after non-OOM failure" + stop_server "$tp" "$dp" + sleep 10 + start_server "$tp" "$dp" +} + +run_bench_serving() { + # Run sglang.bench_serving either natively or inside the SGLang Docker image. + if [[ "${USE_DOCKER_CLIENT:-1}" == "1" ]]; then + local vol_args=() + vol_args+=("-v" "${MODEL_PATH}:${MODEL_PATH}:ro") + if [[ -n "${DATASET_PATH:-}" && -f "${DATASET_PATH}" ]]; then + vol_args+=("-v" "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + vol_args+=("-v" "${RESULT_BASE}:${RESULT_BASE}") + docker run --rm \ + --network host \ + "${vol_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + -e HF_DATASETS_OFFLINE=1 \ + "${DOCKER_CLIENT_IMAGE}" \ + python -m sglang.bench_serving "$@" + else + "$PYTHON" -m sglang.bench_serving "$@" + fi +} +export -f run_bench_serving + +run_warmup() { + local input_len="$1" + local output_len="$2" + + log "warming up (input=${input_len}, output=${output_len}, num=1)" + bash -c ' + run_bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "'"$VLLM_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$input_len"'" \ + --random-output-len "'"$output_len"'" \ + --num-prompts 1 \ + --max-concurrency 1 \ + --request-rate 10000 \ + --output-file /dev/null \ + --output-details \ + >> "'"${log_dir_global}/warmup.log"'" 2>&1 + ' + log "warmup completed" +} + +scenario_already_completed() { + local output_file="$1" + local expected="$2" + [[ -s "$output_file" ]] || return 1 + local completed + completed="$("$PYTHON" -c " +import json, sys +path = sys.argv[1] +try: + with open(path, 'r', encoding='utf-8') as f: + for line in f: + line = line.strip() + if line: + data = json.loads(line) + print(data.get('completed', 0)) + break +except Exception: + print(0) +" "$output_file")" + [[ "${completed:-0}" -ge "$expected" ]] +} + +scenario_already_processed() { + local result_root="$1" + local scenario_name="$2" + local json_path="${result_root}/results.json" + [[ -f "$json_path" ]] || return 1 + "$PYTHON" -c " +import json, sys +path, name = sys.argv[1], sys.argv[2] +try: + with open(path, 'r', encoding='utf-8') as f: + data = json.load(f) + for s in data.get('scenarios', []): + if s.get('name') == name: + if s.get('status') or s.get('metrics', {}).get('success', 0) > 0: + sys.exit(0) +except Exception: + pass +sys.exit(1) +" "$json_path" "$scenario_name" +} + +detect_oom() { + local detail_log="$1" + local server_outer_log="$2" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + if grep -Eiq "$pattern" "$detail_log" "$server_outer_log" 2>/dev/null; then + return 0 + fi + return 1 +} + +start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + nvidia-smi \ + --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \ + --format=csv \ + -l "$GPU_MEM_SAMPLE_INTERVAL_S" \ + > "$csv_path" 2>/dev/null & + echo $! +} + +stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +append_scenario_record() { + local result_root="$1" + local json_path="$result_root/results.json" + shift + local scenario_json + scenario_json="$("$PYTHON" -c " +import json, sys +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +print(json.dumps(d, ensure_ascii=False)) +" "$@")" + PYTHON="$PYTHON" append_scenario_to_json "$json_path" "$scenario_json" +} + +record_skipped_csv() { + local csv_path="$1" + shift + # Args: key=value + local row + row="$("$PYTHON" -c " +import csv, json, sys, io +pairs = [a.split('=', 1) for a in sys.argv[1:]] +d = {} +for k, v in pairs: + try: + d[k] = json.loads(v) + except json.JSONDecodeError: + d[k] = v +buf = io.StringIO() +writer = csv.DictWriter(buf, fieldnames=['engine','tp','dp','mark','isl','osl','concurrency','status','reason','detail_log'], extrasaction='ignore') +writer.writerow(d) +print(buf.getvalue().strip()) +" "$@")" + echo "$row" >> "$csv_path" +} + +skip_remaining_scenarios() { + local result_root="$1" + local scenario_tsv="$2" + local start_index="$3" + local status="$4" + local reason="$5" + local tp="$6" + local dp="$7" + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + + local i=0 + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + if (( i < start_index )); then + i=$((i + 1)) + continue + fi + i=$((i + 1)) + local sname="c${conc}_i${isl}_o${osl}" + if scenario_already_processed "$result_root" "$sname"; then + continue + fi + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"${status}\"" \ + "note=\"${reason}\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=${status}" "reason=${reason}" + done +} + +# --------------------------------------------------------------------------- +# Per-configuration runner +# --------------------------------------------------------------------------- + +run_parallel_config() { + local tp="$1" + local dp="$2" + local config_label="tp${tp}_dp${dp}" + local result_root="${RESULT_BASE}/${RUN_ID}/${config_label}" + local raw_dir="${result_root}/raw_outputs" + local gpu_log_dir="${result_root}/gpu_logs" + local phase_log_dir="${result_root}/logs" + mkdir -p "$raw_dir" "$gpu_log_dir" "$phase_log_dir" + + log "===== ${config_label} START =====" + + # Generate scenario list for this config. + local scenario_tsv="${result_root}/scenarios.tsv" + "$PYTHON" "${SCRIPT_DIR}/generate_scenarios.py" \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$scenario_tsv" + local total_scenarios + total_scenarios="$(tail -n +2 "$scenario_tsv" | wc -l)" + log "generated ${total_scenarios} scenarios for ${config_label}" + + # Write metadata. + ensure_result_root "$result_root" + write_metadata_json \ + "${result_root}/results.json" \ + "${EXPERIMENT_NAME}_${config_label}" \ + "$RUN_ID" \ + "$MODEL_PATH" \ + "vllm" \ + "vllm" \ + "$HARDWARE" \ + "$ACCELERATOR" \ + "$CHIP" \ + "experiments/${EXPERIMENT_NAME}/run_bench.sh" \ + "$DOCKER_IMAGE" \ + "H20 vLLM TP×DP matrix for DeepSeek-V4-Flash" + + local server_args_str + server_args_str="$(build_server_args "$tp" "$dp")" + jq --arg tp "$tp" --arg dp "$dp" --arg cuda "$CUDA_VISIBLE_DEVICES" --arg args "$server_args_str" \ + '.config = { + "tp": ($tp | tonumber), + "dp": ($dp | tonumber), + "cuda_visible_devices": $cuda, + "backend": "vllm", + "server_start_script": "experiments/'${EXPERIMENT_NAME}'/start_vllm_dp.sh", + "server_args": $args + }' "${result_root}/results.json" > "${result_root}/results.json.tmp" && \ + mv "${result_root}/results.json.tmp" "${result_root}/results.json" + + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN: would start server with args: ${server_args_str}" + local line + tail -n +2 "$scenario_tsv" | while IFS=$'\t' read -r mark isl osl conc num; do + log "DRY_RUN: ${config_label} scenario mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + done + log "===== ${config_label} DONE (dry run) =====" + return 0 + fi + + # Initialize skipped_after_oom.csv for this run. + local skipped_csv="${RESULT_BASE}/${RUN_ID}/skipped_after_oom.csv" + if [[ ! -f "$skipped_csv" ]]; then + echo "engine,tp,dp,mark,isl,osl,concurrency,status,reason,detail_log" > "$skipped_csv" + fi + + # Start server once for this TP×DP config. + if ! start_server "$tp" "$dp"; then + log "ERROR: ${config_label} failed to start; skipping all scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" 0 "SKIPPED_SERVICE_START_FAILED" "service failed to start" "$tp" "$dp" + log "===== ${config_label} DONE =====" + return 0 + fi + + # Warmup with a small prompt before the first scenario. + run_warmup 1024 128 || true + + # Read scenarios into an array so we can skip remaining entries on failure. + local -a scenarios=() + while IFS= read -r line; do + scenarios+=("$line") + done < <(tail -n +2 "$scenario_tsv") + + local i mark isl osl conc num + local output_file detail_log gpu_csv sname bench_rc + for (( i = 0; i < ${#scenarios[@]}; i++ )); do + IFS=$'\t' read -r mark isl osl conc num <<< "${scenarios[$i]}" + + if [[ "$GRID_LIMIT" -gt 0 && "$i" -ge "$GRID_LIMIT" ]]; then + log "GRID_LIMIT=${GRID_LIMIT} reached; skipping remaining scenarios" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$i" "SKIPPED_GRID_LIMIT" "GRID_LIMIT reached" "$tp" "$dp" + break + fi + + sname="c${conc}_i${isl}_o${osl}" + output_file="${raw_dir}/vllm_main_${conc}_${isl}_${osl}.jsonl" + detail_log="${phase_log_dir}/vllm_${config_label}_${sname}.log" + gpu_csv="${gpu_log_dir}/gpu_mem_${conc}_${isl}_${osl}.csv" + + if scenario_already_completed "$output_file" "$num" || scenario_already_processed "$result_root" "$sname"; then + log "skipping already-processed ${config_label} scenario: ${sname}" + continue + fi + + log "running ${config_label} scenario: mark=${mark} c=${conc} i=${isl} o=${osl} n=${num}" + + local gpu_pid + gpu_pid="$(start_gpu_monitor "$gpu_csv")" + + bench_rc=0 + timeout "$SCENARIO_TIMEOUT_S" bash -c ' + run_bench_serving \ + --backend vllm \ + --host 127.0.0.1 \ + --port "'"$VLLM_PORT"'" \ + --dataset-name random \ + --dataset-path "'"$DATASET_PATH"'" \ + --random-input-len "'"$isl"'" \ + --random-output-len "'"$osl"'" \ + --num-prompts "'"$num"'" \ + --max-concurrency "'"$conc"'" \ + --request-rate 10000 \ + --output-file "'"$output_file"'" \ + --output-details \ + > "'"$detail_log"'" 2>&1 + ' || bench_rc=$? + + stop_gpu_monitor "$gpu_pid" + + if [[ "$bench_rc" -eq 0 ]]; then + log "finished ${config_label} scenario: output=${output_file}" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"completed\"" \ + "note=\"benchmark finished successfully\"" + continue + fi + + # Failure handling. + if detect_oom "$detail_log" "${log_dir_global}/vllm_tp${tp}_dp${dp}.server.outer.log"; then + log "ERROR: ${config_label} scenario ${sname} triggered OOM; stopping config" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"OOM\"" \ + "note=\"detected CUDA out-of-memory\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=OOM" "reason=detected CUDA out-of-memory" "detail_log=${detail_log}" + stop_server "$tp" "$dp" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_AFTER_OOM" "previous case OOM" "$tp" "$dp" + break + fi + + log "ERROR: ${config_label} scenario ${sname} failed (rc=${bench_rc}); see ${detail_log}" + if [[ "$mark" == "P" ]]; then + log "optional (P) scenario failed; recording as skipped and continuing" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"skipped_optional\"" \ + "note=\"optional scenario failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=skipped_optional" "reason=optional scenario failed (rc=${bench_rc})" "detail_log=${detail_log}" + continue + fi + + # Mandatory scenario failed but not OOM: try to restart the server. + if restart_server "$tp" "$dp"; then + run_warmup 1024 128 || true + log "resuming ${config_label} after server restart" + continue + fi + + log "ERROR: ${config_label} server restart failed; skipping remaining scenarios" + append_scenario_record "$result_root" \ + "name=${sname}" \ + "config=$(jq -n --arg phase main --argjson c "$conc" --argjson i "$isl" --argjson o "$osl" --arg dataset random --argjson n "$num" '{phase: $phase, concurrency: $c, input_len: $i, output_len: $o, dataset: $dataset, num_prompts: $n}')" \ + "status=\"FAILED\"" \ + "note=\"scenario failed and server restart failed (rc=${bench_rc})\"" + record_skipped_csv "$skipped_csv" \ + "engine=vllm" "tp=${tp}" "dp=${dp}" "mark=${mark}" "isl=${isl}" "osl=${osl}" "concurrency=${conc}" "status=FAILED" "reason=scenario failed and server restart failed" "detail_log=${detail_log}" + skip_remaining_scenarios "$result_root" "$scenario_tsv" "$((i + 1))" "SKIPPED_RESTART_FAILED" "server restart failed" "$tp" "$dp" + break + done + + stop_server "$tp" "$dp" + + # Parse results. + log "parsing ${config_label} results" + "$PYTHON" "${SCRIPT_DIR}/../../../scripts/common/parse_backend.py" "$result_root" --backend vllm \ + >> "${phase_log_dir}/parse.log" 2>&1 || { + log "WARNING: parser failed for ${config_label}; see ${phase_log_dir}/parse.log" + } + + log "===== ${config_label} DONE =====" +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +# Cleanup any leftovers. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + stop_server "$tp" "$dp" +done + +# Run each parallel configuration. +for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + run_parallel_config "$tp" "$dp" +done + +# Generate cross-configuration comparison. +log "generating comparison report" +"$PYTHON" "${SCRIPT_DIR}/compare.py" \ + --run-root "${RESULT_BASE}/${RUN_ID}" \ + --output "${RESULT_BASE}/${RUN_ID}/comparison.md" \ + >> "${log_dir_global}/compare.log" 2>&1 || { + log "WARNING: comparison script failed; see ${log_dir_global}/compare.log" + } + +log "all results saved to ${RESULT_BASE}/${RUN_ID}" diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/start_vllm_docker.sh b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/start_vllm_docker.sh new file mode 100755 index 0000000..bab8154 --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/start_vllm_docker.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +# Start vLLM server in Docker for a given TPxDP configuration. +# Usage: start_vllm_docker.sh +# +# Uses the vllm/vllm-openai image and the same argument set as the +# bare-metal start script. The container is removed automatically on stop. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +PORT="${VLLM_PORT:-30030}" +NAME="${EXPERIMENT}_vllm_tp${TP}_dp${DP}" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_docker_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +rm -f "$PID_FILE" + +# Clean up any stale container with the same name. +docker rm -f "$NAME" >/dev/null 2>&1 || true + +SERVER_ARGS=( + "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server in Docker (TP=${TP}, DP=${DP}) ===" +echo "Image: $IMAGE" +echo "Model: $MODEL_PATH" +echo "Container name: $NAME" +echo "Host port: $PORT" +echo "Command: vllm ${SERVER_ARGS_STR}" +echo "Log: $LOG" + +# Run docker in the foreground so that killing the host process stops the +# container (the --rm flag ensures cleanup). nohup lets us background it and +# capture the host PID in the same way as the bare-metal start script. +nohup docker run --rm \ + --name "$NAME" \ + --gpus all \ + --ipc host \ + --shm-size 16g \ + --ulimit memlock=-1 \ + -p "${PORT}:${PORT}" \ + -v "${MODEL_PATH}:${MODEL_PATH}:ro" \ + -v "${RUNTIME_BASE}/tmp:/tmp" \ + -e CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$IMAGE" \ + "${SERVER_ARGS[@]}" \ + > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: Docker vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: Docker vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1 diff --git a/experiments/h20/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh new file mode 100755 index 0000000..741ca9c --- /dev/null +++ b/experiments/h20/hy3_h20_vllm_tp_dp_matrix/start_vllm_dp.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Start vLLM server for a given TP×DP configuration. +# Usage: start_vllm_dp.sh +# +# By default this delegates to the Docker start script because the experiment +# is intended to run vLLM inside a container. Set USE_DOCKER=0 to use the +# local VENV_VLLM environment instead. +set -e + +TP="${1}" +DP="${2}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" + +if [[ "${USE_DOCKER:-1}" == "1" ]]; then + exec "${SCRIPT_DIR}/start_vllm_docker.sh" "$@" +fi + +RUNTIME_BASE="${RUNTIME_BASE:-${SCRIPT_DIR}/runtime}" +mkdir -p "${RUNTIME_BASE}/logs" "${RUNTIME_BASE}/tmp" + +VENV="${VENV_VLLM}" +export PATH="$VENV/bin:$PATH" +export PYTHONUNBUFFERED=1 +export TMPDIR="${RUNTIME_BASE}/tmp" +export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES}" + +LOG="${RUNTIME_BASE}/logs/${EXPERIMENT}_vllm_tp${TP}_dp${DP}_$(date +%Y%m%d_%H%M%S).log" +PID_FILE="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${TP}_dp${DP}.pid" + +rm -f "$PID_FILE" + +SERVER_ARGS=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$TP" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$VLLM_PORT" +) + +if [[ "$DP" -gt 1 ]]; then + SERVER_ARGS+=( + --data-parallel-size "$DP" + ) +fi + +SERVER_ARGS_STR="${SERVER_ARGS[*]}" + +echo "=== Starting vLLM server (TP=${TP}, DP=${DP}) ===" +echo "Model: $MODEL_PATH" +echo "Port: $VLLM_PORT" +echo "Command: $SERVER_ARGS_STR" +echo "Log: $LOG" + +nohup "${SERVER_ARGS[@]}" > "$LOG" 2>&1 & + +PID=$! +echo $PID > "$PID_FILE" +echo "PID: $PID" +echo "Waiting for health on port ${VLLM_PORT}..." + +for i in $(seq 1 240); do + if curl --fail --silent --show-error --max-time 5 "http://127.0.0.1:${VLLM_PORT}/health" >/dev/null 2>&1; then + echo "vLLM server is ready at http://127.0.0.1:${VLLM_PORT}" + echo "Log: $LOG" + exit 0 + fi + if ! kill -0 $PID 2>/dev/null; then + echo "ERROR: vLLM server exited early" + tail -200 "$LOG" + exit 1 + fi + echo "Waiting... ($i/240)" + sleep 5 +done + +echo "ERROR: vLLM server not healthy after 240 retries" +tail -200 "$LOG" +exit 1