diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_points.jsonl b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_points.jsonl index f86e45a..d15141e 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_points.jsonl +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_points.jsonl @@ -14,3 +14,47 @@ {"status":"COMPLETED","completed":400,"failed":0,"duration_s":235.45233723000274,"request_tps":1.6988576316796469,"input_tps":1739.6302148399584,"output_tps":3479.260429679917,"total_tps":5218.890644519875,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":1586.3785705005284,"ttft_p95_ms":3133.5033947099873,"ttft_p99_ms":3142.967526005523,"tpot_p50_ms":22.12632127210705,"tpot_p95_ms":22.91146984701974,"tpot_p99_ms":22.976990950927,"e2e_p50_ms":47054.46670300444,"e2e_p95_ms":48105.06032910416,"e2e_p99_ms":48135.96432496226,"itl_p50_ms":21.907794500293676,"itl_p95_ms":23.54792225087294,"itl_p99_ms":25.59503832002519,"validation_errors":[],"timestamp":"2026-07-16T06:11:01+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":22.791758,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c80_i1024_o2048_a1.log"} {"status":"COMPLETED","completed":480,"failed":0,"duration_s":261.48785026300175,"request_tps":1.8356493409434549,"input_tps":1879.7049251260978,"output_tps":3759.4098502521956,"total_tps":5639.114775378293,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":2183.7680815006024,"ttft_p95_ms":3805.7011090502783,"ttft_p99_ms":4558.676817088854,"tpot_p50_ms":24.876903371278917,"tpot_p95_ms":25.891116515634604,"tpot_p99_ms":26.181815688893472,"e2e_p50_ms":53079.94003850763,"e2e_p95_ms":54830.756290954014,"e2e_p99_ms":54843.78045572128,"itl_p50_ms":24.47853601188399,"itl_p95_ms":26.07234639581293,"itl_p99_ms":27.980867308797315,"validation_errors":[],"timestamp":"2026-07-16T06:16:08+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":8.051982,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c96_i1024_o2048_a1.log"} {"status":"COMPLETED","completed":560,"failed":0,"duration_s":271.93914603099984,"request_tps":2.059284248602305,"input_tps":2108.70707056876,"output_tps":4217.41414113752,"total_tps":6326.121211706281,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":2360.8657174918335,"ttft_p95_ms":4785.738604894141,"ttft_p99_ms":5083.200499021332,"tpot_p50_ms":25.078506466781775,"tpot_p95_ms":26.261017000906556,"tpot_p99_ms":26.82069243732855,"e2e_p50_ms":54802.16980399564,"e2e_p95_ms":55957.59202675035,"e2e_p99_ms":55983.62555622443,"itl_p50_ms":24.729957003728487,"itl_p95_ms":26.741590403253216,"itl_p99_ms":29.055896277423017,"validation_errors":[],"timestamp":"2026-07-16T06:21:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":12.182877,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c112_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c112_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":316.64033691798977,"request_tps":0.2526525861445129,"input_tps":258.7162482119812,"output_tps":1034.864992847925,"total_tps":1293.581241059906,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":575.7432910031639,"ttft_p95_ms":887.6251749068615,"ttft_p99_ms":907.8684282244649,"tpot_p50_ms":15.333096270205825,"tpot_p95_ms":15.443372411818606,"tpot_p99_ms":15.483845902724966,"e2e_p50_ms":63438.04139450367,"e2e_p95_ms":63812.25281149527,"e2e_p99_ms":63831.44774455199,"itl_p50_ms":15.270748001057655,"itl_p95_ms":15.853451890870927,"itl_p99_ms":16.49769142619334,"validation_errors":[],"timestamp":"2026-07-16T06:27:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i1024_o4096_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":390.9258493729867,"request_tps":0.40928477934275004,"input_tps":419.10761404697604,"output_tps":1676.4304561879042,"total_tps":2095.5380702348803,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":1059.6694034975371,"ttft_p95_ms":1531.6532846380142,"ttft_p99_ms":1558.6441262916196,"tpot_p50_ms":18.821365679975997,"tpot_p95_ms":19.016937038081775,"tpot_p99_ms":19.073814857857453,"e2e_p50_ms":77972.91494200181,"e2e_p95_ms":78736.66218614817,"e2e_p99_ms":78754.35213367702,"itl_p50_ms":18.73732249805471,"itl_p95_ms":19.672631401772378,"itl_p99_ms":20.632040495693218,"validation_errors":[],"timestamp":"2026-07-16T06:34:34+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":61.995088,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i1024_o4096_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":395.37849693700264,"request_tps":0.6070132843826361,"input_tps":621.5816032078194,"output_tps":2486.3264128312776,"total_tps":3107.9080160390968,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":1144.204484502552,"ttft_p95_ms":1999.596584997198,"ttft_p99_ms":2035.2585943651502,"tpot_p50_ms":18.99256894627527,"tpot_p95_ms":19.224950862944212,"tpot_p99_ms":19.230104005436754,"e2e_p50_ms":78907.89274750568,"e2e_p95_ms":79839.60575450546,"e2e_p99_ms":79862.91444651186,"itl_p50_ms":18.858423005440272,"itl_p95_ms":20.01599159266334,"itl_p99_ms":21.22574228560552,"validation_errors":[],"timestamp":"2026-07-16T06:41:52+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":48.31074,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i1024_o4096_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":443.57241661399894,"request_tps":0.7214154623110101,"input_tps":738.7294334064743,"output_tps":2954.917733625897,"total_tps":3693.6471670323713,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":1471.407580494997,"ttft_p95_ms":2597.0414660041592,"ttft_p99_ms":2976.737847158656,"tpot_p50_ms":21.723876868741236,"tpot_p95_ms":22.00444482439625,"tpot_p99_ms":22.07639913107326,"e2e_p50_ms":90742.18811100582,"e2e_p95_ms":91541.6523386979,"e2e_p99_ms":91559.50254334108,"itl_p50_ms":21.4493349994882,"itl_p95_ms":22.813913105346728,"itl_p99_ms":24.170049769309127,"validation_errors":[],"timestamp":"2026-07-16T06:50:08+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":18.846734,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c64_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c64_i1024_o4096_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":463.4503973050014,"request_tps":0.8630912872791345,"input_tps":883.8054781738338,"output_tps":3535.221912695335,"total_tps":4419.027390869169,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":2439.6097005082993,"ttft_p95_ms":3686.9959791976726,"ttft_p99_ms":3709.631283667256,"tpot_p50_ms":22.095354364345052,"tpot_p95_ms":22.518257637104984,"tpot_p99_ms":22.625587380977006,"e2e_p50_ms":92209.49949500209,"e2e_p95_ms":93844.80823813865,"e2e_p99_ms":93874.67774137796,"itl_p50_ms":21.87070349464193,"itl_p95_ms":23.618548751983326,"itl_p99_ms":25.773182249395177,"validation_errors":[],"timestamp":"2026-07-16T06:58:45+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":19.63859,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c80_i1024_o4096_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":524.4931549339963,"request_tps":0.9151692362132059,"input_tps":937.1332978823228,"output_tps":3748.5331915292913,"total_tps":4685.666489411614,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":2751.5591720002703,"ttft_p95_ms":4227.864060657878,"ttft_p99_ms":4536.041375702771,"tpot_p50_ms":24.893937553724363,"tpot_p95_ms":25.43370572488336,"tpot_p99_ms":25.74416057706036,"e2e_p50_ms":104575.3224170112,"e2e_p95_ms":106446.31684675842,"e2e_p99_ms":106478.32935052022,"itl_p50_ms":24.56568699562922,"itl_p95_ms":27.559315803227946,"itl_p99_ms":30.286136486683972,"validation_errors":[],"timestamp":"2026-07-16T07:08:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":6.033887,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c96_i1024_o4096_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":44.308171736003715,"request_tps":1.8055360188782963,"input_tps":7395.475533325502,"output_tps":231.10861041642193,"total_tps":7626.584143741924,"mean_input_tokens":4096,"mean_output_tokens":128,"ttft_p50_ms":1465.1815819961485,"ttft_p95_ms":23821.693100642005,"ttft_p99_ms":24259.730825575534,"tpot_p50_ms":22.75063312988532,"tpot_p95_ms":194.92663006346027,"tpot_p99_ms":197.04311093216822,"e2e_p50_ms":4635.849632992176,"e2e_p95_ms":26182.2721034383,"e2e_p99_ms":26191.62205229819,"itl_p50_ms":15.226024501316715,"itl_p95_ms":16.216387940949062,"itl_p99_ms":414.4044824814774,"validation_errors":[],"timestamp":"2026-07-16T07:09:51+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":32.527519440001925,"request_tps":2.4594559123256423,"input_tps":10073.931416885831,"output_tps":629.6207135553644,"total_tps":10703.552130441196,"mean_input_tokens":4096,"mean_output_tokens":256,"ttft_p50_ms":1533.099703505286,"ttft_p95_ms":2719.3758142995644,"ttft_p99_ms":2959.5566013005737,"tpot_p50_ms":19.392000529407206,"tpot_p95_ms":23.99643604938105,"tpot_p99_ms":24.379314330662144,"e2e_p50_ms":6627.143721503671,"e2e_p95_ms":6811.619238646381,"e2e_p99_ms":6835.494205219293,"itl_p50_ms":15.377331001218408,"itl_p95_ms":16.272442803892773,"itl_p99_ms":18.94523348833901,"validation_errors":[],"timestamp":"2026-07-16T07:11:08+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o256_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":66.26290708300075,"request_tps":2.414623913188481,"input_tps":9890.299548420018,"output_tps":618.1437217762511,"total_tps":10508.443270196269,"mean_input_tokens":4096,"mean_output_tokens":256,"ttft_p50_ms":3032.6789590035332,"ttft_p95_ms":5014.513322354456,"ttft_p99_ms":14411.20142712994,"tpot_p50_ms":25.094734256865696,"tpot_p95_ms":108.72940146214187,"tpot_p99_ms":110.79764089783636,"e2e_p50_ms":9434.836903004907,"e2e_p95_ms":29872.09643335227,"e2e_p99_ms":29884.739060596476,"itl_p50_ms":18.683340997085907,"itl_p95_ms":19.744293195253704,"itl_p99_ms":26.305865632602966,"validation_errors":[],"timestamp":"2026-07-16T07:12:56+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":256,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":-1.822842,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":48.17216287400515,"request_tps":1.6607101534809827,"input_tps":6802.268788658105,"output_tps":850.2835985822632,"total_tps":7652.552387240369,"mean_input_tokens":4096,"mean_output_tokens":512,"ttft_p50_ms":1226.4635695100878,"ttft_p95_ms":1866.224592289654,"ttft_p99_ms":2426.2945817218856,"tpot_p50_ms":16.140000863013167,"tpot_p95_ms":17.640392132493695,"tpot_p99_ms":18.638964293663246,"e2e_p50_ms":9504.232979503286,"e2e_p95_ms":10157.524074796675,"e2e_p99_ms":10178.896629959345,"itl_p50_ms":15.288406488252804,"itl_p95_ms":15.857895543740598,"itl_p99_ms":17.156079869600944,"validation_errors":[],"timestamp":"2026-07-16T07:14:23+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":63.387327321004705,"request_tps":2.52416384728972,"input_tps":10338.975118498693,"output_tps":1292.3718898123366,"total_tps":11631.347008311028,"mean_input_tokens":4096,"mean_output_tokens":512,"ttft_p50_ms":2139.6533224979066,"ttft_p95_ms":3791.57651143978,"ttft_p99_ms":4309.74766367377,"tpot_p50_ms":20.01632839431788,"tpot_p95_ms":24.519710790119547,"tpot_p99_ms":25.129288290261652,"e2e_p50_ms":12805.874345496704,"e2e_p95_ms":13800.694600059069,"e2e_p99_ms":13819.40531841843,"itl_p50_ms":18.705550508457236,"itl_p95_ms":20.301984495017674,"itl_p99_ms":21.65679321915376,"validation_errors":[],"timestamp":"2026-07-16T07:16:15+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":51.99304,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":77.56397197001206,"request_tps":3.0942200857479203,"input_tps":12673.925471223482,"output_tps":1584.2406839029352,"total_tps":14258.166155126417,"mean_input_tokens":4096,"mean_output_tokens":512,"ttft_p50_ms":2946.3751199946273,"ttft_p95_ms":6820.920404194476,"ttft_p99_ms":8050.301623321715,"tpot_p50_ms":23.094741272006033,"tpot_p95_ms":31.340273439141725,"tpot_p99_ms":33.989053702566764,"e2e_p50_ms":15109.904168501089,"e2e_p95_ms":19630.664032053755,"e2e_p99_ms":19649.577008009364,"itl_p50_ms":19.291372009320185,"itl_p95_ms":22.182088205590844,"itl_p99_ms":24.491042752633813,"validation_errors":[],"timestamp":"2026-07-16T07:18:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":22.583963,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i4096_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":84.46414080199611,"request_tps":0.9471475023648069,"input_tps":3879.516169686249,"output_tps":969.8790424215623,"total_tps":4849.395212107812,"mean_input_tokens":4096,"mean_output_tokens":1024,"ttft_p50_ms":755.6758849968901,"ttft_p95_ms":1184.9343558955297,"ttft_p99_ms":1330.5567739978123,"tpot_p50_ms":15.835003849462446,"tpot_p95_ms":16.285198607422213,"tpot_p99_ms":16.55195564271857,"e2e_p50_ms":16962.706995996996,"e2e_p95_ms":17168.894911993266,"e2e_p99_ms":17178.149301250814,"itl_p50_ms":15.242625500832219,"itl_p95_ms":16.113438254251378,"itl_p99_ms":16.882939507922856,"validation_errors":[],"timestamp":"2026-07-16T07:20:28+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":107.34333604600397,"request_tps":1.490544321553683,"input_tps":6105.269541083886,"output_tps":1526.3173852709715,"total_tps":7631.5869263548575,"mean_input_tokens":4096,"mean_output_tokens":1024,"ttft_p50_ms":1697.2974899981637,"ttft_p95_ms":2827.3090714494147,"ttft_p99_ms":3041.8824867265357,"tpot_p50_ms":19.041262989738744,"tpot_p95_ms":20.483581416088622,"tpot_p99_ms":20.796884204547236,"e2e_p50_ms":21322.126428000047,"e2e_p95_ms":21977.338839244476,"e2e_p99_ms":21997.641324478464,"itl_p50_ms":18.680188004509546,"itl_p95_ms":19.772065294091586,"itl_p99_ms":21.03478136268678,"validation_errors":[],"timestamp":"2026-07-16T07:23:01+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":57.371932,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":114.72205163900799,"request_tps":2.092012795893852,"input_tps":8568.884411981218,"output_tps":2142.2211029953046,"total_tps":10711.105514976523,"mean_input_tokens":4096,"mean_output_tokens":1024,"ttft_p50_ms":2002.436676499201,"ttft_p95_ms":5319.875290898926,"ttft_p99_ms":6160.708812107041,"tpot_p50_ms":19.824865686698722,"tpot_p95_ms":23.0216869557243,"tpot_p99_ms":23.20206612341515,"e2e_p50_ms":21923.11628299649,"e2e_p95_ms":25281.340099497902,"e2e_p99_ms":25297.796992440562,"itl_p50_ms":18.851815999369137,"itl_p95_ms":20.00413840287365,"itl_p99_ms":21.528728755656623,"validation_errors":[],"timestamp":"2026-07-16T07:25:36+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":40.35227,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i4096_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":161.3202662160038,"request_tps":0.4959079344245688,"input_tps":2031.2388994030339,"output_tps":1015.6194497015169,"total_tps":3046.858349104551,"mean_input_tokens":4096,"mean_output_tokens":2048,"ttft_p50_ms":766.6918885006453,"ttft_p95_ms":1151.3519862077374,"ttft_p99_ms":1174.3669241876341,"tpot_p50_ms":15.32661797704204,"tpot_p95_ms":15.672662826941613,"tpot_p99_ms":15.77215861052261,"e2e_p50_ms":32366.54036600521,"e2e_p95_ms":32729.988757940737,"e2e_p99_ms":32742.016700837994,"itl_p50_ms":15.24984300340293,"itl_p95_ms":15.832478402444394,"itl_p99_ms":16.448479092650818,"validation_errors":[],"timestamp":"2026-07-16T07:28:59+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":203.61964830600482,"request_tps":0.7857787857463927,"input_tps":3218.5499064172245,"output_tps":1609.2749532086123,"total_tps":4827.824859625837,"mean_input_tokens":4096,"mean_output_tokens":2048,"ttft_p50_ms":1823.4369839992723,"ttft_p95_ms":2880.811745747633,"ttft_p99_ms":2898.1182565601193,"tpot_p50_ms":18.92700290888939,"tpot_p95_ms":19.648448953663472,"tpot_p99_ms":19.892009308511824,"e2e_p50_ms":40591.624035492714,"e2e_p95_ms":41278.74980429624,"e2e_p99_ms":41301.732580115786,"itl_p50_ms":18.753395997919142,"itl_p95_ms":19.575604447891237,"itl_p99_ms":20.460386879567523,"validation_errors":[],"timestamp":"2026-07-16T07:33:01+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":58.452554,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":209.64498886499496,"request_tps":1.1447924479346976,"input_tps":4689.069866740521,"output_tps":2344.5349333702607,"total_tps":7033.604800110782,"mean_input_tokens":4096,"mean_output_tokens":2048,"ttft_p50_ms":1850.1262010031496,"ttft_p95_ms":5557.007933600835,"ttft_p99_ms":5728.514949967212,"tpot_p50_ms":19.032042742548224,"tpot_p95_ms":20.8632619918908,"tpot_p99_ms":21.040059630580483,"e2e_p50_ms":40867.48082550184,"e2e_p95_ms":44286.82900869899,"e2e_p99_ms":44307.41918329746,"itl_p50_ms":18.80523749423446,"itl_p95_ms":20.01578960043844,"itl_p99_ms":21.193549290037485,"validation_errors":[],"timestamp":"2026-07-16T07:37:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":45.688897,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i4096_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":316.8896808080026,"request_tps":0.252453787059322,"input_tps":1034.0507117949828,"output_tps":1034.0507117949828,"total_tps":2068.1014235899656,"mean_input_tokens":4096,"mean_output_tokens":4096,"ttft_p50_ms":658.9246715011541,"ttft_p95_ms":1005.5294943493209,"ttft_p99_ms":1026.091546237294,"tpot_p50_ms":15.320106219048359,"tpot_p95_ms":15.378604879511569,"tpot_p99_ms":15.45278130631852,"e2e_p50_ms":63453.26198550174,"e2e_p95_ms":63513.736396850436,"e2e_p99_ms":63521.74632662762,"itl_p50_ms":15.256563012371771,"itl_p95_ms":15.777264998177998,"itl_p99_ms":16.359419524669644,"validation_errors":[],"timestamp":"2026-07-16T07:43:10+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o4096_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":389.95699894400605,"request_tps":0.4103016497543987,"input_tps":1680.595557394017,"output_tps":1680.595557394017,"total_tps":3361.191114788034,"mean_input_tokens":4096,"mean_output_tokens":4096,"ttft_p50_ms":1123.4332419990096,"ttft_p95_ms":1676.0268465513946,"ttft_p99_ms":1717.2243972464637,"tpot_p50_ms":18.85827924127029,"tpot_p95_ms":19.032755866286788,"tpot_p99_ms":19.08965374972383,"e2e_p50_ms":77915.13783499977,"e2e_p95_ms":78629.53818804745,"e2e_p99_ms":78646.88334376202,"itl_p50_ms":18.702409994148184,"itl_p95_ms":19.55858354413067,"itl_p99_ms":20.443303814536193,"validation_errors":[],"timestamp":"2026-07-16T07:50:22+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":62.525449,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o4096_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":402.4469851330068,"request_tps":0.596351839785012,"input_tps":2442.6571357594094,"output_tps":2442.6571357594094,"total_tps":4885.314271518819,"mean_input_tokens":4096,"mean_output_tokens":4096,"ttft_p50_ms":2047.3789034949732,"ttft_p95_ms":4081.0472775985545,"ttft_p99_ms":4647.203718657256,"tpot_p50_ms":19.02283846043863,"tpot_p95_ms":19.43320571131982,"tpot_p99_ms":19.899421790027564,"e2e_p50_ms":80315.7795865045,"e2e_p95_ms":81844.74676285972,"e2e_p99_ms":81865.64722886324,"itl_p50_ms":18.825628008926287,"itl_p95_ms":19.975638002506457,"itl_p99_ms":21.221037310315303,"validation_errors":[],"timestamp":"2026-07-16T07:57:45+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":45.344734,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i4096_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i4096_o4096_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":56.001648912992096,"request_tps":1.4285293657030234,"input_tps":23405.025127678335,"output_tps":182.851758809987,"total_tps":23587.87688648832,"mean_input_tokens":16384,"mean_output_tokens":128,"ttft_p50_ms":5949.235703003069,"ttft_p95_ms":9227.944630107957,"ttft_p99_ms":11097.429403630083,"tpot_p50_ms":39.72050276768026,"tpot_p95_ms":72.95558840476615,"tpot_p99_ms":80.88457267013912,"e2e_p50_ms":11065.538848997676,"e2e_p95_ms":13142.54083404594,"e2e_p99_ms":14229.989901815272,"itl_p50_ms":15.273823504685424,"itl_p95_ms":17.06440234847832,"itl_p99_ms":1069.5194568477748,"validation_errors":[],"timestamp":"2026-07-16T07:59:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":64.32757946698985,"request_tps":1.2436345446676191,"input_tps":20375.708379834272,"output_tps":318.3704434349105,"total_tps":20694.07882326918,"mean_input_tokens":16384,"mean_output_tokens":256,"ttft_p50_ms":4847.543320000113,"ttft_p95_ms":8945.170914454502,"ttft_p99_ms":10200.10803171811,"tpot_p50_ms":30.697892317645408,"tpot_p95_ms":45.12670790233909,"tpot_p99_ms":46.21028031682417,"e2e_p50_ms":12845.963567502622,"e2e_p95_ms":15142.341455946007,"e2e_p99_ms":15201.770440481196,"itl_p50_ms":15.241901011904702,"itl_p95_ms":16.105174794211056,"itl_p99_ms":806.8968500866322,"validation_errors":[],"timestamp":"2026-07-16T08:01:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":72.41434827599733,"request_tps":1.1047534349835064,"input_tps":18100.28027876977,"output_tps":565.6337587115553,"total_tps":18665.914037481325,"mean_input_tokens":16384,"mean_output_tokens":512,"ttft_p50_ms":3977.608113505994,"ttft_p95_ms":6827.726796599017,"ttft_p99_ms":7064.793197970429,"tpot_p50_ms":20.6349563414749,"tpot_p95_ms":26.606408149222858,"tpot_p99_ms":27.689232515706706,"e2e_p50_ms":14308.850026493019,"e2e_p95_ms":14974.010983452172,"e2e_p99_ms":14998.68528622057,"itl_p50_ms":15.342150996730197,"itl_p95_ms":16.593942245526705,"itl_p99_ms":19.848780611646326,"validation_errors":[],"timestamp":"2026-07-16T08:03:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":111.50969841099868,"request_tps":0.7174263865833329,"input_tps":11754.313917781326,"output_tps":734.6446198613329,"total_tps":12488.958537642658,"mean_input_tokens":16384,"mean_output_tokens":1024,"ttft_p50_ms":4311.803800497728,"ttft_p95_ms":7437.313835108215,"ttft_p99_ms":9130.135508555395,"tpot_p50_ms":17.93213432404895,"tpot_p95_ms":22.04782108866086,"tpot_p99_ms":22.43329354472889,"e2e_p50_ms":21698.23511350114,"e2e_p95_ms":24644.28777230205,"e2e_p99_ms":26412.721827901405,"itl_p50_ms":15.224864997435361,"itl_p95_ms":15.731755192973651,"itl_p99_ms":16.860831878148,"validation_errors":[],"timestamp":"2026-07-16T08:05:39+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":188.87208026400185,"request_tps":0.4235671036617879,"input_tps":6939.723426394733,"output_tps":867.4654282993416,"total_tps":7807.1888546940745,"mean_input_tokens":16384,"mean_output_tokens":2048,"ttft_p50_ms":2707.821463998698,"ttft_p95_ms":6159.3108133580245,"ttft_p99_ms":7476.327504011569,"tpot_p50_ms":16.789645155595252,"tpot_p95_ms":19.235649654987466,"tpot_p99_ms":19.809279338319755,"e2e_p50_ms":37331.52540450101,"e2e_p95_ms":41633.905589940696,"e2e_p99_ms":41644.779238444025,"itl_p50_ms":15.280855994205922,"itl_p95_ms":18.66166050749598,"itl_p99_ms":19.038321707921565,"validation_errors":[],"timestamp":"2026-07-16T08:09:28+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o2048_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":263.11688939499436,"request_tps":0.30404737675316235,"input_tps":19926.048882895248,"output_tps":38.91806422440478,"total_tps":19964.966947119654,"mean_input_tokens":65536,"mean_output_tokens":128,"ttft_p50_ms":21783.06187000271,"ttft_p95_ms":48708.6885400975,"ttft_p99_ms":53530.08918488397,"tpot_p50_ms":208.85715699601343,"tpot_p95_ms":353.9625207657282,"tpot_p99_ms":368.66045906663794,"e2e_p50_ms":53675.07621049299,"e2e_p95_ms":64500.535845247214,"e2e_p99_ms":78019.4687606334,"itl_p50_ms":15.624218496668618,"itl_p95_ms":1377.797994745198,"itl_p99_ms":1593.279839480674,"validation_errors":[],"timestamp":"2026-07-16T08:14:46+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i65536_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i65536_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":318.6807930810028,"request_tps":0.2510348967898592,"input_tps":16451.82299602021,"output_tps":64.26493357820395,"total_tps":16516.087929598416,"mean_input_tokens":65536,"mean_output_tokens":256,"ttft_p50_ms":25208.387847495032,"ttft_p95_ms":57570.60906205442,"ttft_p99_ms":64559.6027293129,"tpot_p50_ms":139.20286317058046,"tpot_p95_ms":218.68997558943673,"tpot_p99_ms":237.28532163909728,"e2e_p50_ms":62259.9117714999,"e2e_p95_ms":93757.1761358522,"e2e_p99_ms":109198.36791741903,"itl_p50_ms":15.581318992190063,"itl_p95_ms":1060.9013718029018,"itl_p99_ms":1579.705025086296,"validation_errors":[],"timestamp":"2026-07-16T08:21:04+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i65536_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i65536_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":337.8231221129972,"request_tps":0.23681031511289238,"input_tps":15519.600811238515,"output_tps":121.2468813378009,"total_tps":15640.847692576317,"mean_input_tokens":65536,"mean_output_tokens":512,"ttft_p50_ms":24695.115143498697,"ttft_p95_ms":57546.68756374739,"ttft_p99_ms":64468.70614748111,"tpot_p50_ms":76.04405581409218,"tpot_p95_ms":120.77805700313024,"tpot_p99_ms":135.3599259530104,"e2e_p50_ms":66170.59861750022,"e2e_p95_ms":92553.88260725957,"e2e_p99_ms":109859.61571042174,"itl_p50_ms":15.599313002894633,"itl_p95_ms":824.2479222499242,"itl_p99_ms":1410.173195308016,"validation_errors":[],"timestamp":"2026-07-16T08:27:37+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i65536_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i65536_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":376.9979162859963,"request_tps":0.21220276437631763,"input_tps":13906.920366166352,"output_tps":217.29563072134926,"total_tps":14124.215996887702,"mean_input_tokens":65536,"mean_output_tokens":1024,"ttft_p50_ms":25524.604717495095,"ttft_p95_ms":56727.187289898575,"ttft_p99_ms":65385.280855649864,"tpot_p50_ms":45.977114398337235,"tpot_p95_ms":68.72763672496647,"tpot_p99_ms":71.65886099602294,"e2e_p50_ms":72648.60455600865,"e2e_p95_ms":102850.95250014929,"e2e_p99_ms":128267.6254233617,"itl_p50_ms":15.504026996495668,"itl_p95_ms":16.275252601190004,"itl_p99_ms":1222.4654011012171,"validation_errors":[],"timestamp":"2026-07-16T08:34:46+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i65536_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i65536_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":653.2824726870022,"request_tps":0.12245851273332609,"input_tps":16050.882180982517,"output_tps":15.674689629865739,"total_tps":16066.556870612381,"mean_input_tokens":131072,"mean_output_tokens":128,"ttft_p50_ms":55664.04897299799,"ttft_p95_ms":118594.41714475105,"ttft_p99_ms":139872.23895933607,"tpot_p50_ms":544.0846560747494,"tpot_p95_ms":902.7456552543442,"tpot_p99_ms":958.5754168834561,"e2e_p50_ms":129924.55754899856,"e2e_p95_ms":185606.83365028963,"e2e_p99_ms":243322.26077544419,"itl_p50_ms":16.123155495733954,"itl_p95_ms":1741.9904875576322,"itl_p99_ms":1853.2892793747305,"validation_errors":[],"timestamp":"2026-07-16T08:46:49+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i131072_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i131072_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":702.479465484008,"request_tps":0.11388233241078448,"input_tps":14926.785073746343,"output_tps":29.153877097160827,"total_tps":14955.938950843505,"mean_input_tokens":131072,"mean_output_tokens":256,"ttft_p50_ms":57824.69000250421,"ttft_p95_ms":127985.28046924763,"ttft_p99_ms":151725.3745295175,"tpot_p50_ms":253.09320623725628,"tpot_p95_ms":525.102048641203,"tpot_p99_ms":619.6555398026641,"e2e_p50_ms":138529.73486200062,"e2e_p95_ms":184438.2270416012,"e2e_p99_ms":185996.91346610835,"itl_p50_ms":15.797007494256832,"itl_p95_ms":1590.6224329526603,"itl_p99_ms":1839.2413294677688,"validation_errors":[],"timestamp":"2026-07-16T09:15:16+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i131072_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i131072_o256_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":341.0657499819936,"request_tps":0.11727943952775025,"input_tps":15372.05069778128,"output_tps":30.023536519104063,"total_tps":15402.074234300384,"mean_input_tokens":131072,"mean_output_tokens":256,"ttft_p50_ms":38767.44465551019,"ttft_p95_ms":67753.44985890988,"ttft_p99_ms":68613.70976785096,"tpot_p50_ms":99.49740432157162,"tpot_p95_ms":179.5039385709941,"tpot_p99_ms":227.61194875988735,"e2e_p50_ms":70759.12754750607,"e2e_p95_ms":71843.55354870277,"e2e_p99_ms":71887.88047623617,"itl_p50_ms":12.980383995454758,"itl_p95_ms":1125.97120024584,"itl_p99_ms":1602.8499752024072,"validation_errors":[],"timestamp":"2026-07-16T09:22:39+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c8_i131072_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c8_i131072_o256_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":97.25075286399806,"request_tps":0.051413483728936615,"input_tps":6738.86813931918,"output_tps":13.161851834607774,"total_tps":6752.029991153788,"mean_input_tokens":131072,"mean_output_tokens":256,"ttft_p50_ms":16969.752281991532,"ttft_p95_ms":17121.84501980082,"ttft_p99_ms":17125.887859960785,"tpot_p50_ms":9.60843350975683,"tpot_p95_ms":9.634889066653034,"tpot_p99_ms":9.637108200772976,"e2e_p50_ms":19427.356342988787,"e2e_p95_ms":19571.998772607185,"e2e_p99_ms":19575.258900129702,"itl_p50_ms":10.00273649697192,"itl_p95_ms":10.19823560709483,"itl_p99_ms":11.044782063545426,"validation_errors":[],"timestamp":"2026-07-16T09:25:11+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":256,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c1_i131072_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c1_i131072_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":667.9211692050012,"request_tps":0.1197746136646944,"input_tps":15699.098162258824,"output_tps":61.32460219632353,"total_tps":15760.422764455148,"mean_input_tokens":131072,"mean_output_tokens":512,"ttft_p50_ms":55981.32588399312,"ttft_p95_ms":121882.91273160065,"ttft_p99_ms":140760.88047753967,"tpot_p50_ms":141.8939732465815,"tpot_p95_ms":235.749314115342,"tpot_p99_ms":254.5075704539367,"e2e_p50_ms":132681.28659549984,"e2e_p95_ms":175247.86795944526,"e2e_p99_ms":234920.12947427476,"itl_p50_ms":15.690204003476538,"itl_p95_ms":1249.2242270018323,"itl_p99_ms":1797.7115144065467,"validation_errors":[],"timestamp":"2026-07-16T09:37:52+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i131072_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i131072_o512_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":368.12840555600997,"request_tps":0.10865773843120097,"input_tps":14241.987091654373,"output_tps":55.632762076774895,"total_tps":14297.619853731148,"mean_input_tokens":131072,"mean_output_tokens":512,"ttft_p50_ms":36897.75435599586,"ttft_p95_ms":68026.47409820274,"ttft_p99_ms":74667.961793141,"tpot_p50_ms":63.40141743053057,"tpot_p95_ms":116.81624726055898,"tpot_p99_ms":143.86918745584376,"e2e_p50_ms":74333.1425399956,"e2e_p95_ms":83933.02225654986,"e2e_p99_ms":104491.18695786949,"itl_p50_ms":12.98504300211789,"itl_p95_ms":18.927634652209232,"itl_p99_ms":1447.1019217558205,"validation_errors":[],"timestamp":"2026-07-16T09:45:12+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c8_i131072_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c8_i131072_o512_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":110.12223424699914,"request_tps":0.04540409150058859,"input_tps":5951.205081165148,"output_tps":23.24689484830136,"total_tps":5974.45197601345,"mean_input_tokens":131072,"mean_output_tokens":512,"ttft_p50_ms":16962.474314001156,"ttft_p95_ms":17155.541892390465,"ttft_p99_ms":17171.51858087047,"tpot_p50_ms":9.821654945213728,"tpot_p95_ms":9.835613834436566,"tpot_p99_ms":9.837956189974383,"e2e_p50_ms":21989.969162998023,"e2e_p95_ms":22173.37846419541,"e2e_p99_ms":22190.17843683483,"itl_p50_ms":9.989651007344946,"itl_p95_ms":10.151538599166088,"itl_p99_ms":10.469720991677605,"validation_errors":[],"timestamp":"2026-07-16T09:47:57+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":512,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c1_i131072_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c1_i131072_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1511.6757434350002,"request_tps":0.05292140219053523,"input_tps":13873.028055835668,"output_tps":6.77393948038851,"total_tps":13879.801995316056,"mean_input_tokens":262144,"mean_output_tokens":128,"ttft_p50_ms":70474.41060249548,"ttft_p95_ms":206680.64425829216,"ttft_p99_ms":263259.9471526309,"tpot_p50_ms":1770.9045697204808,"tpot_p95_ms":1859.1631874913205,"tpot_p99_ms":1883.4251564144938,"e2e_p50_ms":293508.3537885075,"e2e_p95_ms":418159.4987228506,"e2e_p99_ms":491266.18717678235,"itl_p50_ms":1741.0211320020608,"itl_p95_ms":2297.1252548493794,"itl_p99_ms":2395.285054906126,"validation_errors":[],"timestamp":"2026-07-16T10:15:00+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i262144_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i262144_o128_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":1143.3382095730049,"request_tps":0.03498527353068918,"input_tps":9171.179544428984,"output_tps":4.478115011928215,"total_tps":9175.65765944091,"mean_input_tokens":262144,"mean_output_tokens":128,"ttft_p50_ms":60434.31567850348,"ttft_p95_ms":171402.15240599928,"ttft_p99_ms":221100.73346630885,"tpot_p50_ms":1432.7522763937384,"tpot_p95_ms":1566.4298780566398,"tpot_p99_ms":1577.578224665822,"e2e_p50_ms":243043.1625314959,"e2e_p95_ms":344139.9401556495,"e2e_p99_ms":419677.4824362584,"itl_p50_ms":1394.7447990067303,"itl_p95_ms":1954.1919693932869,"itl_p99_ms":2118.3612401573937,"validation_errors":[],"timestamp":"2026-07-16T10:35:53+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c8_i262144_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c8_i262144_o128_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":215.86107532100868,"request_tps":0.02316304591999489,"input_tps":6072.05350965114,"output_tps":2.964869877759346,"total_tps":6075.0183795289,"mean_input_tokens":262144,"mean_output_tokens":128,"ttft_p50_ms":42189.58094799018,"ttft_p95_ms":42206.01177979843,"ttft_p99_ms":42207.67736715847,"tpot_p50_ms":9.026367795253979,"tpot_p95_ms":9.051597466134083,"tpot_p99_ms":9.053308627064942,"e2e_p50_ms":43335.3122970002,"e2e_p95_ms":43353.944078995846,"e2e_p99_ms":43357.12344699539,"itl_p50_ms":10.197708994382992,"itl_p95_ms":10.398621491185622,"itl_p99_ms":11.979914264811663,"validation_errors":[],"timestamp":"2026-07-16T10:40:50+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c1_i262144_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c1_i262144_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1557.1239152909984,"request_tps":0.0513767717613209,"input_tps":13468.112456599705,"output_tps":13.15245357089815,"total_tps":13481.264910170603,"mean_input_tokens":262144,"mean_output_tokens":256,"ttft_p50_ms":128086.70317550423,"ttft_p95_ms":293147.14115650055,"ttft_p99_ms":327246.22319525003,"tpot_p50_ms":694.1213535215974,"tpot_p95_ms":1110.7064755825186,"tpot_p99_ms":1121.9122485736023,"e2e_p50_ms":320114.9868900029,"e2e_p95_ms":442507.49014469614,"e2e_p99_ms":540593.0392589162,"itl_p50_ms":16.51875649986323,"itl_p95_ms":2135.694057152432,"itl_p99_ms":2348.4220405294036,"validation_errors":[],"timestamp":"2026-07-16T11:08:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i262144_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i262144_o256_a1.log"} +{"status":"COMPLETED","completed":40,"failed":0,"duration_s":818.7336148600007,"request_tps":0.04885593955591991,"input_tps":12807.291418947068,"output_tps":12.507120526315497,"total_tps":12819.798539473382,"mean_input_tokens":262144,"mean_output_tokens":256,"ttft_p50_ms":105032.74306450476,"ttft_p95_ms":162785.75602490891,"ttft_p99_ms":165083.89961342298,"tpot_p50_ms":214.14210347058685,"tpot_p95_ms":422.65357399257476,"tpot_p99_ms":440.83721461195177,"e2e_p50_ms":164185.14075000712,"e2e_p95_ms":168861.28058029353,"e2e_p99_ms":169246.668313704,"itl_p50_ms":13.376075992709957,"itl_p95_ms":1720.7928034986253,"itl_p99_ms":2045.3944211971263,"validation_errors":[],"timestamp":"2026-07-16T11:23:47+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c8_i262144_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c8_i262144_o256_a1.log"} +{"status":"COMPLETED","completed":5,"failed":0,"duration_s":222.6894240649999,"request_tps":0.022452795057481358,"input_tps":5885.865507548393,"output_tps":5.7479155347152275,"total_tps":5891.6134230831085,"mean_input_tokens":262144,"mean_output_tokens":256,"ttft_p50_ms":41992.54209799983,"ttft_p95_ms":42385.26861999417,"ttft_p99_ms":42405.3075527912,"tpot_p50_ms":9.564196976419884,"tpot_p95_ms":9.778543265853717,"tpot_p99_ms":9.797764013144736,"e2e_p50_ms":44431.4123269869,"e2e_p95_ms":44819.5239008026,"e2e_p99_ms":44832.609473761986,"itl_p50_ms":10.17798750399379,"itl_p95_ms":10.34693365872954,"itl_p99_ms":10.991512061009418,"validation_errors":[],"timestamp":"2026-07-16T11:28:54+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":256,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c1_i262144_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c1_i262144_o256_a1.log"} +{"timestamp":"2026-07-16T12:39:42+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":524288,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":2,"status":"FAILED","completed":0,"failed":80,"error_type":"benchmark rc=124","validation_errors":[],"gain_pct":null,"plateau_streak":0,"raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i524288_o128_a2.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i524288_o128_a2.log"} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_shapes.jsonl b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_shapes.jsonl index 945ee47..6850b0f 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_shapes.jsonl +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_shapes.jsonl @@ -3,3 +3,25 @@ {"timestamp":"2026-07-16T05:37:07+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":1933.4323295182799,"last_total_tps":1933.4323295182799} {"timestamp":"2026-07-16T05:50:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":5,"search_cap":512,"max_successful_concurrency":80,"saturation_concurrency":null,"stop_probe_concurrency":80,"best_tps_concurrency":80,"best_total_tps":5711.313811148789,"last_total_tps":5711.313811148789} {"timestamp":"2026-07-16T06:21:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":7,"search_cap":512,"max_successful_concurrency":112,"saturation_concurrency":null,"stop_probe_concurrency":112,"best_tps_concurrency":112,"best_total_tps":6326.121211706281,"last_total_tps":6326.121211706281} +{"timestamp":"2026-07-16T07:08:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":6,"search_cap":512,"max_successful_concurrency":96,"saturation_concurrency":null,"stop_probe_concurrency":96,"best_tps_concurrency":96,"best_total_tps":4685.666489411614,"last_total_tps":4685.666489411614} +{"timestamp":"2026-07-16T07:09:51+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":7626.584143741924,"last_total_tps":7626.584143741924} +{"timestamp":"2026-07-16T07:12:56+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":512,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":10703.552130441196,"last_total_tps":10508.443270196269} +{"timestamp":"2026-07-16T07:18:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":14258.166155126417,"last_total_tps":14258.166155126417} +{"timestamp":"2026-07-16T07:25:36+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":10711.105514976523,"last_total_tps":10711.105514976523} +{"timestamp":"2026-07-16T07:37:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":7033.604800110782,"last_total_tps":7033.604800110782} +{"timestamp":"2026-07-16T07:57:45+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":4885.314271518819,"last_total_tps":4885.314271518819} +{"timestamp":"2026-07-16T07:59:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":23587.87688648832,"last_total_tps":23587.87688648832} +{"timestamp":"2026-07-16T08:01:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":20694.07882326918,"last_total_tps":20694.07882326918} +{"timestamp":"2026-07-16T08:03:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":18665.914037481325,"last_total_tps":18665.914037481325} +{"timestamp":"2026-07-16T08:05:39+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":12488.958537642658,"last_total_tps":12488.958537642658} +{"timestamp":"2026-07-16T08:09:28+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":7807.1888546940745,"last_total_tps":7807.1888546940745} +{"timestamp":"2026-07-16T08:14:46+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":19964.966947119654,"last_total_tps":19964.966947119654} +{"timestamp":"2026-07-16T08:21:04+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":16516.087929598416,"last_total_tps":16516.087929598416} +{"timestamp":"2026-07-16T08:27:37+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":15640.847692576317,"last_total_tps":15640.847692576317} +{"timestamp":"2026-07-16T08:34:46+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":14124.215996887702,"last_total_tps":14124.215996887702} +{"timestamp":"2026-07-16T08:46:49+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":16066.556870612381,"last_total_tps":16066.556870612381} +{"timestamp":"2026-07-16T09:25:11+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":15402.074234300384,"last_total_tps":6752.029991153788} +{"timestamp":"2026-07-16T09:47:57+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":15760.422764455148,"last_total_tps":5974.45197601345} +{"timestamp":"2026-07-16T10:40:50+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":13879.801995316056,"last_total_tps":6075.0183795289} +{"timestamp":"2026-07-16T11:28:54+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":13481.264910170603,"last_total_tps":5891.6134230831085} +{"timestamp":"2026-07-16T12:39:42+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":524288,"osl":128,"status":"FAILED","stop_reason":"BENCHMARK_FAILED","tested_points":1,"search_cap":512,"max_successful_concurrency":0,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":null,"best_total_tps":null,"last_total_tps":null} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/run_manifest.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/run_manifest.json index ae2922b..8327a6b 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/run_manifest.json +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/run_manifest.json @@ -7,16 +7,21 @@ "matrix": "/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/matrix.json", "dataset": "random", "tokenize_prompt": false, - "random_range_ratio": 1, + "random_range_ratio": 1.0, "search": { "start_concurrency": 16, "max_concurrency": 512, "multiplier": 2, + "initial_backoff_concurrencies": [ + 8, + 1 + ], "num_prompts_multiplier": 5, - "min_tps_gain_pct": 2, + "min_tps_gain_pct": 2.0, "plateau_patience": 2, "warmup_max_requests": 0, - "ttft_slo_ms": 4000, - "enable_ttft_slo_stop": 1 + "ttft_slo_ms": 4000.0, + "enable_ttft_slo_stop": 1, + "ttft_group_skip_ms": 8000.0 } } diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o4096_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o4096_a1.json new file mode 100644 index 0000000..008271a --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o4096_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 316.64033691798977, + "request_tps": 0.2526525861445129, + "input_tps": 258.7162482119812, + "output_tps": 1034.864992847925, + "total_tps": 1293.581241059906, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 4096.0, + "ttft_p50_ms": 575.7432910031639, + "ttft_p95_ms": 887.6251749068615, + "ttft_p99_ms": 907.8684282244649, + "tpot_p50_ms": 15.333096270205825, + "tpot_p95_ms": 15.443372411818606, + "tpot_p99_ms": 15.483845902724966, + "e2e_p50_ms": 63438.04139450367, + "e2e_p95_ms": 63812.25281149527, + "e2e_p99_ms": 63831.44774455199, + "itl_p50_ms": 15.270748001057655, + "itl_p95_ms": 15.853451890870927, + "itl_p99_ms": 16.49769142619334, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i131072_o128_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i131072_o128_a1.json new file mode 100644 index 0000000..4c516eb --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i131072_o128_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 653.2824726870022, + "request_tps": 0.12245851273332609, + "input_tps": 16050.882180982517, + "output_tps": 15.674689629865739, + "total_tps": 16066.556870612381, + "mean_input_tokens": 131072.0, + "mean_output_tokens": 128.0, + "ttft_p50_ms": 55664.04897299799, + "ttft_p95_ms": 118594.41714475105, + "ttft_p99_ms": 139872.23895933607, + "tpot_p50_ms": 544.0846560747494, + "tpot_p95_ms": 902.7456552543442, + "tpot_p99_ms": 958.5754168834561, + "e2e_p50_ms": 129924.55754899856, + "e2e_p95_ms": 185606.83365028963, + "e2e_p99_ms": 243322.26077544419, + "itl_p50_ms": 16.123155495733954, + "itl_p95_ms": 1741.9904875576322, + "itl_p99_ms": 1853.2892793747305, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i131072_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i131072_o256_a1.json new file mode 100644 index 0000000..1130b45 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i131072_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 702.479465484008, + "request_tps": 0.11388233241078448, + "input_tps": 14926.785073746343, + "output_tps": 29.153877097160827, + "total_tps": 14955.938950843505, + "mean_input_tokens": 131072.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 57824.69000250421, + "ttft_p95_ms": 127985.28046924763, + "ttft_p99_ms": 151725.3745295175, + "tpot_p50_ms": 253.09320623725628, + "tpot_p95_ms": 525.102048641203, + "tpot_p99_ms": 619.6555398026641, + "e2e_p50_ms": 138529.73486200062, + "e2e_p95_ms": 184438.2270416012, + "e2e_p99_ms": 185996.91346610835, + "itl_p50_ms": 15.797007494256832, + "itl_p95_ms": 1590.6224329526603, + "itl_p99_ms": 1839.2413294677688, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i131072_o512_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i131072_o512_a1.json new file mode 100644 index 0000000..2740038 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i131072_o512_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 667.9211692050012, + "request_tps": 0.1197746136646944, + "input_tps": 15699.098162258824, + "output_tps": 61.32460219632353, + "total_tps": 15760.422764455148, + "mean_input_tokens": 131072.0, + "mean_output_tokens": 512.0, + "ttft_p50_ms": 55981.32588399312, + "ttft_p95_ms": 121882.91273160065, + "ttft_p99_ms": 140760.88047753967, + "tpot_p50_ms": 141.8939732465815, + "tpot_p95_ms": 235.749314115342, + "tpot_p99_ms": 254.5075704539367, + "e2e_p50_ms": 132681.28659549984, + "e2e_p95_ms": 175247.86795944526, + "e2e_p99_ms": 234920.12947427476, + "itl_p50_ms": 15.690204003476538, + "itl_p95_ms": 1249.2242270018323, + "itl_p99_ms": 1797.7115144065467, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o1024_a1.json new file mode 100644 index 0000000..de94686 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 111.50969841099868, + "request_tps": 0.7174263865833329, + "input_tps": 11754.313917781326, + "output_tps": 734.6446198613329, + "total_tps": 12488.958537642658, + "mean_input_tokens": 16384.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 4311.803800497728, + "ttft_p95_ms": 7437.313835108215, + "ttft_p99_ms": 9130.135508555395, + "tpot_p50_ms": 17.93213432404895, + "tpot_p95_ms": 22.04782108866086, + "tpot_p99_ms": 22.43329354472889, + "e2e_p50_ms": 21698.23511350114, + "e2e_p95_ms": 24644.28777230205, + "e2e_p99_ms": 26412.721827901405, + "itl_p50_ms": 15.224864997435361, + "itl_p95_ms": 15.731755192973651, + "itl_p99_ms": 16.860831878148, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o128_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o128_a1.json new file mode 100644 index 0000000..026dbc0 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o128_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 56.001648912992096, + "request_tps": 1.4285293657030234, + "input_tps": 23405.025127678335, + "output_tps": 182.851758809987, + "total_tps": 23587.87688648832, + "mean_input_tokens": 16384.0, + "mean_output_tokens": 128.0, + "ttft_p50_ms": 5949.235703003069, + "ttft_p95_ms": 9227.944630107957, + "ttft_p99_ms": 11097.429403630083, + "tpot_p50_ms": 39.72050276768026, + "tpot_p95_ms": 72.95558840476615, + "tpot_p99_ms": 80.88457267013912, + "e2e_p50_ms": 11065.538848997676, + "e2e_p95_ms": 13142.54083404594, + "e2e_p99_ms": 14229.989901815272, + "itl_p50_ms": 15.273823504685424, + "itl_p95_ms": 17.06440234847832, + "itl_p99_ms": 1069.5194568477748, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o2048_a1.json new file mode 100644 index 0000000..b709aff --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 188.87208026400185, + "request_tps": 0.4235671036617879, + "input_tps": 6939.723426394733, + "output_tps": 867.4654282993416, + "total_tps": 7807.1888546940745, + "mean_input_tokens": 16384.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 2707.821463998698, + "ttft_p95_ms": 6159.3108133580245, + "ttft_p99_ms": 7476.327504011569, + "tpot_p50_ms": 16.789645155595252, + "tpot_p95_ms": 19.235649654987466, + "tpot_p99_ms": 19.809279338319755, + "e2e_p50_ms": 37331.52540450101, + "e2e_p95_ms": 41633.905589940696, + "e2e_p99_ms": 41644.779238444025, + "itl_p50_ms": 15.280855994205922, + "itl_p95_ms": 18.66166050749598, + "itl_p99_ms": 19.038321707921565, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o256_a1.json new file mode 100644 index 0000000..2d5a40e --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 64.32757946698985, + "request_tps": 1.2436345446676191, + "input_tps": 20375.708379834272, + "output_tps": 318.3704434349105, + "total_tps": 20694.07882326918, + "mean_input_tokens": 16384.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 4847.543320000113, + "ttft_p95_ms": 8945.170914454502, + "ttft_p99_ms": 10200.10803171811, + "tpot_p50_ms": 30.697892317645408, + "tpot_p95_ms": 45.12670790233909, + "tpot_p99_ms": 46.21028031682417, + "e2e_p50_ms": 12845.963567502622, + "e2e_p95_ms": 15142.341455946007, + "e2e_p99_ms": 15201.770440481196, + "itl_p50_ms": 15.241901011904702, + "itl_p95_ms": 16.105174794211056, + "itl_p99_ms": 806.8968500866322, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o512_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o512_a1.json new file mode 100644 index 0000000..5b0ad35 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i16384_o512_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 72.41434827599733, + "request_tps": 1.1047534349835064, + "input_tps": 18100.28027876977, + "output_tps": 565.6337587115553, + "total_tps": 18665.914037481325, + "mean_input_tokens": 16384.0, + "mean_output_tokens": 512.0, + "ttft_p50_ms": 3977.608113505994, + "ttft_p95_ms": 6827.726796599017, + "ttft_p99_ms": 7064.793197970429, + "tpot_p50_ms": 20.6349563414749, + "tpot_p95_ms": 26.606408149222858, + "tpot_p99_ms": 27.689232515706706, + "e2e_p50_ms": 14308.850026493019, + "e2e_p95_ms": 14974.010983452172, + "e2e_p99_ms": 14998.68528622057, + "itl_p50_ms": 15.342150996730197, + "itl_p95_ms": 16.593942245526705, + "itl_p99_ms": 19.848780611646326, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i262144_o128_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i262144_o128_a1.json new file mode 100644 index 0000000..e07d4d4 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i262144_o128_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 1511.6757434350002, + "request_tps": 0.05292140219053523, + "input_tps": 13873.028055835668, + "output_tps": 6.77393948038851, + "total_tps": 13879.801995316056, + "mean_input_tokens": 262144.0, + "mean_output_tokens": 128.0, + "ttft_p50_ms": 70474.41060249548, + "ttft_p95_ms": 206680.64425829216, + "ttft_p99_ms": 263259.9471526309, + "tpot_p50_ms": 1770.9045697204808, + "tpot_p95_ms": 1859.1631874913205, + "tpot_p99_ms": 1883.4251564144938, + "e2e_p50_ms": 293508.3537885075, + "e2e_p95_ms": 418159.4987228506, + "e2e_p99_ms": 491266.18717678235, + "itl_p50_ms": 1741.0211320020608, + "itl_p95_ms": 2297.1252548493794, + "itl_p99_ms": 2395.285054906126, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i262144_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i262144_o256_a1.json new file mode 100644 index 0000000..e595b4f --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i262144_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 1557.1239152909984, + "request_tps": 0.0513767717613209, + "input_tps": 13468.112456599705, + "output_tps": 13.15245357089815, + "total_tps": 13481.264910170603, + "mean_input_tokens": 262144.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 128086.70317550423, + "ttft_p95_ms": 293147.14115650055, + "ttft_p99_ms": 327246.22319525003, + "tpot_p50_ms": 694.1213535215974, + "tpot_p95_ms": 1110.7064755825186, + "tpot_p99_ms": 1121.9122485736023, + "e2e_p50_ms": 320114.9868900029, + "e2e_p95_ms": 442507.49014469614, + "e2e_p99_ms": 540593.0392589162, + "itl_p50_ms": 16.51875649986323, + "itl_p95_ms": 2135.694057152432, + "itl_p99_ms": 2348.4220405294036, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o1024_a1.json new file mode 100644 index 0000000..c2df328 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 84.46414080199611, + "request_tps": 0.9471475023648069, + "input_tps": 3879.516169686249, + "output_tps": 969.8790424215623, + "total_tps": 4849.395212107812, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 755.6758849968901, + "ttft_p95_ms": 1184.9343558955297, + "ttft_p99_ms": 1330.5567739978123, + "tpot_p50_ms": 15.835003849462446, + "tpot_p95_ms": 16.285198607422213, + "tpot_p99_ms": 16.55195564271857, + "e2e_p50_ms": 16962.706995996996, + "e2e_p95_ms": 17168.894911993266, + "e2e_p99_ms": 17178.149301250814, + "itl_p50_ms": 15.242625500832219, + "itl_p95_ms": 16.113438254251378, + "itl_p99_ms": 16.882939507922856, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o128_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o128_a1.json new file mode 100644 index 0000000..d764008 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o128_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 44.308171736003715, + "request_tps": 1.8055360188782963, + "input_tps": 7395.475533325502, + "output_tps": 231.10861041642193, + "total_tps": 7626.584143741924, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 128.0, + "ttft_p50_ms": 1465.1815819961485, + "ttft_p95_ms": 23821.693100642005, + "ttft_p99_ms": 24259.730825575534, + "tpot_p50_ms": 22.75063312988532, + "tpot_p95_ms": 194.92663006346027, + "tpot_p99_ms": 197.04311093216822, + "e2e_p50_ms": 4635.849632992176, + "e2e_p95_ms": 26182.2721034383, + "e2e_p99_ms": 26191.62205229819, + "itl_p50_ms": 15.226024501316715, + "itl_p95_ms": 16.216387940949062, + "itl_p99_ms": 414.4044824814774, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o2048_a1.json new file mode 100644 index 0000000..b696ba5 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 161.3202662160038, + "request_tps": 0.4959079344245688, + "input_tps": 2031.2388994030339, + "output_tps": 1015.6194497015169, + "total_tps": 3046.858349104551, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 766.6918885006453, + "ttft_p95_ms": 1151.3519862077374, + "ttft_p99_ms": 1174.3669241876341, + "tpot_p50_ms": 15.32661797704204, + "tpot_p95_ms": 15.672662826941613, + "tpot_p99_ms": 15.77215861052261, + "e2e_p50_ms": 32366.54036600521, + "e2e_p95_ms": 32729.988757940737, + "e2e_p99_ms": 32742.016700837994, + "itl_p50_ms": 15.24984300340293, + "itl_p95_ms": 15.832478402444394, + "itl_p99_ms": 16.448479092650818, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o256_a1.json new file mode 100644 index 0000000..428c75c --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 32.527519440001925, + "request_tps": 2.4594559123256423, + "input_tps": 10073.931416885831, + "output_tps": 629.6207135553644, + "total_tps": 10703.552130441196, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 1533.099703505286, + "ttft_p95_ms": 2719.3758142995644, + "ttft_p99_ms": 2959.5566013005737, + "tpot_p50_ms": 19.392000529407206, + "tpot_p95_ms": 23.99643604938105, + "tpot_p99_ms": 24.379314330662144, + "e2e_p50_ms": 6627.143721503671, + "e2e_p95_ms": 6811.619238646381, + "e2e_p99_ms": 6835.494205219293, + "itl_p50_ms": 15.377331001218408, + "itl_p95_ms": 16.272442803892773, + "itl_p99_ms": 18.94523348833901, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o4096_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o4096_a1.json new file mode 100644 index 0000000..94b37a8 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o4096_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 316.8896808080026, + "request_tps": 0.252453787059322, + "input_tps": 1034.0507117949828, + "output_tps": 1034.0507117949828, + "total_tps": 2068.1014235899656, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 4096.0, + "ttft_p50_ms": 658.9246715011541, + "ttft_p95_ms": 1005.5294943493209, + "ttft_p99_ms": 1026.091546237294, + "tpot_p50_ms": 15.320106219048359, + "tpot_p95_ms": 15.378604879511569, + "tpot_p99_ms": 15.45278130631852, + "e2e_p50_ms": 63453.26198550174, + "e2e_p95_ms": 63513.736396850436, + "e2e_p99_ms": 63521.74632662762, + "itl_p50_ms": 15.256563012371771, + "itl_p95_ms": 15.777264998177998, + "itl_p99_ms": 16.359419524669644, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o512_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o512_a1.json new file mode 100644 index 0000000..66f3453 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i4096_o512_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 48.17216287400515, + "request_tps": 1.6607101534809827, + "input_tps": 6802.268788658105, + "output_tps": 850.2835985822632, + "total_tps": 7652.552387240369, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 512.0, + "ttft_p50_ms": 1226.4635695100878, + "ttft_p95_ms": 1866.224592289654, + "ttft_p99_ms": 2426.2945817218856, + "tpot_p50_ms": 16.140000863013167, + "tpot_p95_ms": 17.640392132493695, + "tpot_p99_ms": 18.638964293663246, + "e2e_p50_ms": 9504.232979503286, + "e2e_p95_ms": 10157.524074796675, + "e2e_p99_ms": 10178.896629959345, + "itl_p50_ms": 15.288406488252804, + "itl_p95_ms": 15.857895543740598, + "itl_p99_ms": 17.156079869600944, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o1024_a1.json new file mode 100644 index 0000000..d3b9611 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 376.9979162859963, + "request_tps": 0.21220276437631763, + "input_tps": 13906.920366166352, + "output_tps": 217.29563072134926, + "total_tps": 14124.215996887702, + "mean_input_tokens": 65536.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 25524.604717495095, + "ttft_p95_ms": 56727.187289898575, + "ttft_p99_ms": 65385.280855649864, + "tpot_p50_ms": 45.977114398337235, + "tpot_p95_ms": 68.72763672496647, + "tpot_p99_ms": 71.65886099602294, + "e2e_p50_ms": 72648.60455600865, + "e2e_p95_ms": 102850.95250014929, + "e2e_p99_ms": 128267.6254233617, + "itl_p50_ms": 15.504026996495668, + "itl_p95_ms": 16.275252601190004, + "itl_p99_ms": 1222.4654011012171, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o128_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o128_a1.json new file mode 100644 index 0000000..610bdb1 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o128_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 263.11688939499436, + "request_tps": 0.30404737675316235, + "input_tps": 19926.048882895248, + "output_tps": 38.91806422440478, + "total_tps": 19964.966947119654, + "mean_input_tokens": 65536.0, + "mean_output_tokens": 128.0, + "ttft_p50_ms": 21783.06187000271, + "ttft_p95_ms": 48708.6885400975, + "ttft_p99_ms": 53530.08918488397, + "tpot_p50_ms": 208.85715699601343, + "tpot_p95_ms": 353.9625207657282, + "tpot_p99_ms": 368.66045906663794, + "e2e_p50_ms": 53675.07621049299, + "e2e_p95_ms": 64500.535845247214, + "e2e_p99_ms": 78019.4687606334, + "itl_p50_ms": 15.624218496668618, + "itl_p95_ms": 1377.797994745198, + "itl_p99_ms": 1593.279839480674, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o256_a1.json new file mode 100644 index 0000000..e0c732e --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 318.6807930810028, + "request_tps": 0.2510348967898592, + "input_tps": 16451.82299602021, + "output_tps": 64.26493357820395, + "total_tps": 16516.087929598416, + "mean_input_tokens": 65536.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 25208.387847495032, + "ttft_p95_ms": 57570.60906205442, + "ttft_p99_ms": 64559.6027293129, + "tpot_p50_ms": 139.20286317058046, + "tpot_p95_ms": 218.68997558943673, + "tpot_p99_ms": 237.28532163909728, + "e2e_p50_ms": 62259.9117714999, + "e2e_p95_ms": 93757.1761358522, + "e2e_p99_ms": 109198.36791741903, + "itl_p50_ms": 15.581318992190063, + "itl_p95_ms": 1060.9013718029018, + "itl_p99_ms": 1579.705025086296, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o512_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o512_a1.json new file mode 100644 index 0000000..ecf98cb --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i65536_o512_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 337.8231221129972, + "request_tps": 0.23681031511289238, + "input_tps": 15519.600811238515, + "output_tps": 121.2468813378009, + "total_tps": 15640.847692576317, + "mean_input_tokens": 65536.0, + "mean_output_tokens": 512.0, + "ttft_p50_ms": 24695.115143498697, + "ttft_p95_ms": 57546.68756374739, + "ttft_p99_ms": 64468.70614748111, + "tpot_p50_ms": 76.04405581409218, + "tpot_p95_ms": 120.77805700313024, + "tpot_p99_ms": 135.3599259530104, + "e2e_p50_ms": 66170.59861750022, + "e2e_p95_ms": 92553.88260725957, + "e2e_p99_ms": 109859.61571042174, + "itl_p50_ms": 15.599313002894633, + "itl_p95_ms": 824.2479222499242, + "itl_p99_ms": 1410.173195308016, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i131072_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i131072_o256_a1.json new file mode 100644 index 0000000..e56a2be --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i131072_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 5, + "failed": 0, + "duration_s": 97.25075286399806, + "request_tps": 0.051413483728936615, + "input_tps": 6738.86813931918, + "output_tps": 13.161851834607774, + "total_tps": 6752.029991153788, + "mean_input_tokens": 131072.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 16969.752281991532, + "ttft_p95_ms": 17121.84501980082, + "ttft_p99_ms": 17125.887859960785, + "tpot_p50_ms": 9.60843350975683, + "tpot_p95_ms": 9.634889066653034, + "tpot_p99_ms": 9.637108200772976, + "e2e_p50_ms": 19427.356342988787, + "e2e_p95_ms": 19571.998772607185, + "e2e_p99_ms": 19575.258900129702, + "itl_p50_ms": 10.00273649697192, + "itl_p95_ms": 10.19823560709483, + "itl_p99_ms": 11.044782063545426, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i131072_o512_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i131072_o512_a1.json new file mode 100644 index 0000000..6039a8f --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i131072_o512_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 5, + "failed": 0, + "duration_s": 110.12223424699914, + "request_tps": 0.04540409150058859, + "input_tps": 5951.205081165148, + "output_tps": 23.24689484830136, + "total_tps": 5974.45197601345, + "mean_input_tokens": 131072.0, + "mean_output_tokens": 512.0, + "ttft_p50_ms": 16962.474314001156, + "ttft_p95_ms": 17155.541892390465, + "ttft_p99_ms": 17171.51858087047, + "tpot_p50_ms": 9.821654945213728, + "tpot_p95_ms": 9.835613834436566, + "tpot_p99_ms": 9.837956189974383, + "e2e_p50_ms": 21989.969162998023, + "e2e_p95_ms": 22173.37846419541, + "e2e_p99_ms": 22190.17843683483, + "itl_p50_ms": 9.989651007344946, + "itl_p95_ms": 10.151538599166088, + "itl_p99_ms": 10.469720991677605, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i262144_o128_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i262144_o128_a1.json new file mode 100644 index 0000000..ffc3fcb --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i262144_o128_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 5, + "failed": 0, + "duration_s": 215.86107532100868, + "request_tps": 0.02316304591999489, + "input_tps": 6072.05350965114, + "output_tps": 2.964869877759346, + "total_tps": 6075.0183795289, + "mean_input_tokens": 262144.0, + "mean_output_tokens": 128.0, + "ttft_p50_ms": 42189.58094799018, + "ttft_p95_ms": 42206.01177979843, + "ttft_p99_ms": 42207.67736715847, + "tpot_p50_ms": 9.026367795253979, + "tpot_p95_ms": 9.051597466134083, + "tpot_p99_ms": 9.053308627064942, + "e2e_p50_ms": 43335.3122970002, + "e2e_p95_ms": 43353.944078995846, + "e2e_p99_ms": 43357.12344699539, + "itl_p50_ms": 10.197708994382992, + "itl_p95_ms": 10.398621491185622, + "itl_p99_ms": 11.979914264811663, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i262144_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i262144_o256_a1.json new file mode 100644 index 0000000..631b127 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c1_i262144_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 5, + "failed": 0, + "duration_s": 222.6894240649999, + "request_tps": 0.022452795057481358, + "input_tps": 5885.865507548393, + "output_tps": 5.7479155347152275, + "total_tps": 5891.6134230831085, + "mean_input_tokens": 262144.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 41992.54209799983, + "ttft_p95_ms": 42385.26861999417, + "ttft_p99_ms": 42405.3075527912, + "tpot_p50_ms": 9.564196976419884, + "tpot_p95_ms": 9.778543265853717, + "tpot_p99_ms": 9.797764013144736, + "e2e_p50_ms": 44431.4123269869, + "e2e_p95_ms": 44819.5239008026, + "e2e_p99_ms": 44832.609473761986, + "itl_p50_ms": 10.17798750399379, + "itl_p95_ms": 10.34693365872954, + "itl_p99_ms": 10.991512061009418, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o4096_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o4096_a1.json new file mode 100644 index 0000000..910132b --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o4096_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 160, + "failed": 0, + "duration_s": 390.9258493729867, + "request_tps": 0.40928477934275004, + "input_tps": 419.10761404697604, + "output_tps": 1676.4304561879042, + "total_tps": 2095.5380702348803, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 4096.0, + "ttft_p50_ms": 1059.6694034975371, + "ttft_p95_ms": 1531.6532846380142, + "ttft_p99_ms": 1558.6441262916196, + "tpot_p50_ms": 18.821365679975997, + "tpot_p95_ms": 19.016937038081775, + "tpot_p99_ms": 19.073814857857453, + "e2e_p50_ms": 77972.91494200181, + "e2e_p95_ms": 78736.66218614817, + "e2e_p99_ms": 78754.35213367702, + "itl_p50_ms": 18.73732249805471, + "itl_p95_ms": 19.672631401772378, + "itl_p99_ms": 20.632040495693218, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o1024_a1.json new file mode 100644 index 0000000..df26fff --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 160, + "failed": 0, + "duration_s": 107.34333604600397, + "request_tps": 1.490544321553683, + "input_tps": 6105.269541083886, + "output_tps": 1526.3173852709715, + "total_tps": 7631.5869263548575, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 1697.2974899981637, + "ttft_p95_ms": 2827.3090714494147, + "ttft_p99_ms": 3041.8824867265357, + "tpot_p50_ms": 19.041262989738744, + "tpot_p95_ms": 20.483581416088622, + "tpot_p99_ms": 20.796884204547236, + "e2e_p50_ms": 21322.126428000047, + "e2e_p95_ms": 21977.338839244476, + "e2e_p99_ms": 21997.641324478464, + "itl_p50_ms": 18.680188004509546, + "itl_p95_ms": 19.772065294091586, + "itl_p99_ms": 21.03478136268678, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o2048_a1.json new file mode 100644 index 0000000..e953c0c --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 160, + "failed": 0, + "duration_s": 203.61964830600482, + "request_tps": 0.7857787857463927, + "input_tps": 3218.5499064172245, + "output_tps": 1609.2749532086123, + "total_tps": 4827.824859625837, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 1823.4369839992723, + "ttft_p95_ms": 2880.811745747633, + "ttft_p99_ms": 2898.1182565601193, + "tpot_p50_ms": 18.92700290888939, + "tpot_p95_ms": 19.648448953663472, + "tpot_p99_ms": 19.892009308511824, + "e2e_p50_ms": 40591.624035492714, + "e2e_p95_ms": 41278.74980429624, + "e2e_p99_ms": 41301.732580115786, + "itl_p50_ms": 18.753395997919142, + "itl_p95_ms": 19.575604447891237, + "itl_p99_ms": 20.460386879567523, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o256_a1.json new file mode 100644 index 0000000..7bc139f --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 160, + "failed": 0, + "duration_s": 66.26290708300075, + "request_tps": 2.414623913188481, + "input_tps": 9890.299548420018, + "output_tps": 618.1437217762511, + "total_tps": 10508.443270196269, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 3032.6789590035332, + "ttft_p95_ms": 5014.513322354456, + "ttft_p99_ms": 14411.20142712994, + "tpot_p50_ms": 25.094734256865696, + "tpot_p95_ms": 108.72940146214187, + "tpot_p99_ms": 110.79764089783636, + "e2e_p50_ms": 9434.836903004907, + "e2e_p95_ms": 29872.09643335227, + "e2e_p99_ms": 29884.739060596476, + "itl_p50_ms": 18.683340997085907, + "itl_p95_ms": 19.744293195253704, + "itl_p99_ms": 26.305865632602966, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o4096_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o4096_a1.json new file mode 100644 index 0000000..73299cc --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o4096_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 160, + "failed": 0, + "duration_s": 389.95699894400605, + "request_tps": 0.4103016497543987, + "input_tps": 1680.595557394017, + "output_tps": 1680.595557394017, + "total_tps": 3361.191114788034, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 4096.0, + "ttft_p50_ms": 1123.4332419990096, + "ttft_p95_ms": 1676.0268465513946, + "ttft_p99_ms": 1717.2243972464637, + "tpot_p50_ms": 18.85827924127029, + "tpot_p95_ms": 19.032755866286788, + "tpot_p99_ms": 19.08965374972383, + "e2e_p50_ms": 77915.13783499977, + "e2e_p95_ms": 78629.53818804745, + "e2e_p99_ms": 78646.88334376202, + "itl_p50_ms": 18.702409994148184, + "itl_p95_ms": 19.55858354413067, + "itl_p99_ms": 20.443303814536193, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o512_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o512_a1.json new file mode 100644 index 0000000..1696527 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i4096_o512_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 160, + "failed": 0, + "duration_s": 63.387327321004705, + "request_tps": 2.52416384728972, + "input_tps": 10338.975118498693, + "output_tps": 1292.3718898123366, + "total_tps": 11631.347008311028, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 512.0, + "ttft_p50_ms": 2139.6533224979066, + "ttft_p95_ms": 3791.57651143978, + "ttft_p99_ms": 4309.74766367377, + "tpot_p50_ms": 20.01632839431788, + "tpot_p95_ms": 24.519710790119547, + "tpot_p99_ms": 25.129288290261652, + "e2e_p50_ms": 12805.874345496704, + "e2e_p95_ms": 13800.694600059069, + "e2e_p99_ms": 13819.40531841843, + "itl_p50_ms": 18.705550508457236, + "itl_p95_ms": 20.301984495017674, + "itl_p99_ms": 21.65679321915376, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o4096_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o4096_a1.json new file mode 100644 index 0000000..d0fe36e --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o4096_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 240, + "failed": 0, + "duration_s": 395.37849693700264, + "request_tps": 0.6070132843826361, + "input_tps": 621.5816032078194, + "output_tps": 2486.3264128312776, + "total_tps": 3107.9080160390968, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 4096.0, + "ttft_p50_ms": 1144.204484502552, + "ttft_p95_ms": 1999.596584997198, + "ttft_p99_ms": 2035.2585943651502, + "tpot_p50_ms": 18.99256894627527, + "tpot_p95_ms": 19.224950862944212, + "tpot_p99_ms": 19.230104005436754, + "e2e_p50_ms": 78907.89274750568, + "e2e_p95_ms": 79839.60575450546, + "e2e_p99_ms": 79862.91444651186, + "itl_p50_ms": 18.858423005440272, + "itl_p95_ms": 20.01599159266334, + "itl_p99_ms": 21.22574228560552, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o1024_a1.json new file mode 100644 index 0000000..93b7d02 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 240, + "failed": 0, + "duration_s": 114.72205163900799, + "request_tps": 2.092012795893852, + "input_tps": 8568.884411981218, + "output_tps": 2142.2211029953046, + "total_tps": 10711.105514976523, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 2002.436676499201, + "ttft_p95_ms": 5319.875290898926, + "ttft_p99_ms": 6160.708812107041, + "tpot_p50_ms": 19.824865686698722, + "tpot_p95_ms": 23.0216869557243, + "tpot_p99_ms": 23.20206612341515, + "e2e_p50_ms": 21923.11628299649, + "e2e_p95_ms": 25281.340099497902, + "e2e_p99_ms": 25297.796992440562, + "itl_p50_ms": 18.851815999369137, + "itl_p95_ms": 20.00413840287365, + "itl_p99_ms": 21.528728755656623, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o2048_a1.json new file mode 100644 index 0000000..bc24536 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 240, + "failed": 0, + "duration_s": 209.64498886499496, + "request_tps": 1.1447924479346976, + "input_tps": 4689.069866740521, + "output_tps": 2344.5349333702607, + "total_tps": 7033.604800110782, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 1850.1262010031496, + "ttft_p95_ms": 5557.007933600835, + "ttft_p99_ms": 5728.514949967212, + "tpot_p50_ms": 19.032042742548224, + "tpot_p95_ms": 20.8632619918908, + "tpot_p99_ms": 21.040059630580483, + "e2e_p50_ms": 40867.48082550184, + "e2e_p95_ms": 44286.82900869899, + "e2e_p99_ms": 44307.41918329746, + "itl_p50_ms": 18.80523749423446, + "itl_p95_ms": 20.01578960043844, + "itl_p99_ms": 21.193549290037485, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o4096_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o4096_a1.json new file mode 100644 index 0000000..8584a0d --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o4096_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 240, + "failed": 0, + "duration_s": 402.4469851330068, + "request_tps": 0.596351839785012, + "input_tps": 2442.6571357594094, + "output_tps": 2442.6571357594094, + "total_tps": 4885.314271518819, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 4096.0, + "ttft_p50_ms": 2047.3789034949732, + "ttft_p95_ms": 4081.0472775985545, + "ttft_p99_ms": 4647.203718657256, + "tpot_p50_ms": 19.02283846043863, + "tpot_p95_ms": 19.43320571131982, + "tpot_p99_ms": 19.899421790027564, + "e2e_p50_ms": 80315.7795865045, + "e2e_p95_ms": 81844.74676285972, + "e2e_p99_ms": 81865.64722886324, + "itl_p50_ms": 18.825628008926287, + "itl_p95_ms": 19.975638002506457, + "itl_p99_ms": 21.221037310315303, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o512_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o512_a1.json new file mode 100644 index 0000000..657cf85 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i4096_o512_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 240, + "failed": 0, + "duration_s": 77.56397197001206, + "request_tps": 3.0942200857479203, + "input_tps": 12673.925471223482, + "output_tps": 1584.2406839029352, + "total_tps": 14258.166155126417, + "mean_input_tokens": 4096.0, + "mean_output_tokens": 512.0, + "ttft_p50_ms": 2946.3751199946273, + "ttft_p95_ms": 6820.920404194476, + "ttft_p99_ms": 8050.301623321715, + "tpot_p50_ms": 23.094741272006033, + "tpot_p95_ms": 31.340273439141725, + "tpot_p99_ms": 33.989053702566764, + "e2e_p50_ms": 15109.904168501089, + "e2e_p95_ms": 19630.664032053755, + "e2e_p99_ms": 19649.577008009364, + "itl_p50_ms": 19.291372009320185, + "itl_p95_ms": 22.182088205590844, + "itl_p99_ms": 24.491042752633813, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o4096_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o4096_a1.json new file mode 100644 index 0000000..a7e2b4a --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o4096_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 320, + "failed": 0, + "duration_s": 443.57241661399894, + "request_tps": 0.7214154623110101, + "input_tps": 738.7294334064743, + "output_tps": 2954.917733625897, + "total_tps": 3693.6471670323713, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 4096.0, + "ttft_p50_ms": 1471.407580494997, + "ttft_p95_ms": 2597.0414660041592, + "ttft_p99_ms": 2976.737847158656, + "tpot_p50_ms": 21.723876868741236, + "tpot_p95_ms": 22.00444482439625, + "tpot_p99_ms": 22.07639913107326, + "e2e_p50_ms": 90742.18811100582, + "e2e_p95_ms": 91541.6523386979, + "e2e_p99_ms": 91559.50254334108, + "itl_p50_ms": 21.4493349994882, + "itl_p95_ms": 22.813913105346728, + "itl_p99_ms": 24.170049769309127, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o4096_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o4096_a1.json new file mode 100644 index 0000000..5ebd9ff --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o4096_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 400, + "failed": 0, + "duration_s": 463.4503973050014, + "request_tps": 0.8630912872791345, + "input_tps": 883.8054781738338, + "output_tps": 3535.221912695335, + "total_tps": 4419.027390869169, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 4096.0, + "ttft_p50_ms": 2439.6097005082993, + "ttft_p95_ms": 3686.9959791976726, + "ttft_p99_ms": 3709.631283667256, + "tpot_p50_ms": 22.095354364345052, + "tpot_p95_ms": 22.518257637104984, + "tpot_p99_ms": 22.625587380977006, + "e2e_p50_ms": 92209.49949500209, + "e2e_p95_ms": 93844.80823813865, + "e2e_p99_ms": 93874.67774137796, + "itl_p50_ms": 21.87070349464193, + "itl_p95_ms": 23.618548751983326, + "itl_p99_ms": 25.773182249395177, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i131072_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i131072_o256_a1.json new file mode 100644 index 0000000..992af65 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i131072_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 40, + "failed": 0, + "duration_s": 341.0657499819936, + "request_tps": 0.11727943952775025, + "input_tps": 15372.05069778128, + "output_tps": 30.023536519104063, + "total_tps": 15402.074234300384, + "mean_input_tokens": 131072.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 38767.44465551019, + "ttft_p95_ms": 67753.44985890988, + "ttft_p99_ms": 68613.70976785096, + "tpot_p50_ms": 99.49740432157162, + "tpot_p95_ms": 179.5039385709941, + "tpot_p99_ms": 227.61194875988735, + "e2e_p50_ms": 70759.12754750607, + "e2e_p95_ms": 71843.55354870277, + "e2e_p99_ms": 71887.88047623617, + "itl_p50_ms": 12.980383995454758, + "itl_p95_ms": 1125.97120024584, + "itl_p99_ms": 1602.8499752024072, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i131072_o512_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i131072_o512_a1.json new file mode 100644 index 0000000..58cd809 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i131072_o512_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 40, + "failed": 0, + "duration_s": 368.12840555600997, + "request_tps": 0.10865773843120097, + "input_tps": 14241.987091654373, + "output_tps": 55.632762076774895, + "total_tps": 14297.619853731148, + "mean_input_tokens": 131072.0, + "mean_output_tokens": 512.0, + "ttft_p50_ms": 36897.75435599586, + "ttft_p95_ms": 68026.47409820274, + "ttft_p99_ms": 74667.961793141, + "tpot_p50_ms": 63.40141743053057, + "tpot_p95_ms": 116.81624726055898, + "tpot_p99_ms": 143.86918745584376, + "e2e_p50_ms": 74333.1425399956, + "e2e_p95_ms": 83933.02225654986, + "e2e_p99_ms": 104491.18695786949, + "itl_p50_ms": 12.98504300211789, + "itl_p95_ms": 18.927634652209232, + "itl_p99_ms": 1447.1019217558205, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i262144_o128_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i262144_o128_a1.json new file mode 100644 index 0000000..c3d1cd4 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i262144_o128_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 40, + "failed": 0, + "duration_s": 1143.3382095730049, + "request_tps": 0.03498527353068918, + "input_tps": 9171.179544428984, + "output_tps": 4.478115011928215, + "total_tps": 9175.65765944091, + "mean_input_tokens": 262144.0, + "mean_output_tokens": 128.0, + "ttft_p50_ms": 60434.31567850348, + "ttft_p95_ms": 171402.15240599928, + "ttft_p99_ms": 221100.73346630885, + "tpot_p50_ms": 1432.7522763937384, + "tpot_p95_ms": 1566.4298780566398, + "tpot_p99_ms": 1577.578224665822, + "e2e_p50_ms": 243043.1625314959, + "e2e_p95_ms": 344139.9401556495, + "e2e_p99_ms": 419677.4824362584, + "itl_p50_ms": 1394.7447990067303, + "itl_p95_ms": 1954.1919693932869, + "itl_p99_ms": 2118.3612401573937, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i262144_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i262144_o256_a1.json new file mode 100644 index 0000000..91ead40 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c8_i262144_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 40, + "failed": 0, + "duration_s": 818.7336148600007, + "request_tps": 0.04885593955591991, + "input_tps": 12807.291418947068, + "output_tps": 12.507120526315497, + "total_tps": 12819.798539473382, + "mean_input_tokens": 262144.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 105032.74306450476, + "ttft_p95_ms": 162785.75602490891, + "ttft_p99_ms": 165083.89961342298, + "tpot_p50_ms": 214.14210347058685, + "tpot_p95_ms": 422.65357399257476, + "tpot_p99_ms": 440.83721461195177, + "e2e_p50_ms": 164185.14075000712, + "e2e_p95_ms": 168861.28058029353, + "e2e_p99_ms": 169246.668313704, + "itl_p50_ms": 13.376075992709957, + "itl_p95_ms": 1720.7928034986253, + "itl_p99_ms": 2045.3944211971263, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c96_i1024_o4096_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c96_i1024_o4096_a1.json new file mode 100644 index 0000000..4e0c207 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c96_i1024_o4096_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 480, + "failed": 0, + "duration_s": 524.4931549339963, + "request_tps": 0.9151692362132059, + "input_tps": 937.1332978823228, + "output_tps": 3748.5331915292913, + "total_tps": 4685.666489411614, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 4096.0, + "ttft_p50_ms": 2751.5591720002703, + "ttft_p95_ms": 4227.864060657878, + "ttft_p99_ms": 4536.041375702771, + "tpot_p50_ms": 24.893937553724363, + "tpot_p95_ms": 25.43370572488336, + "tpot_p99_ms": 25.74416057706036, + "e2e_p50_ms": 104575.3224170112, + "e2e_p95_ms": 106446.31684675842, + "e2e_p99_ms": 106478.32935052022, + "itl_p50_ms": 24.56568699562922, + "itl_p95_ms": 27.559315803227946, + "itl_p99_ms": 30.286136486683972, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp4_dp2/server_cmd.txt b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp4_dp2/server_cmd.txt new file mode 100644 index 0000000..236c49e --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp4_dp2/server_cmd.txt @@ -0,0 +1 @@ +vllm serve /data1/hf_models/DeepSeek-V4-Flash --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 4 --gpu-memory-utilization 0.9 --max-model-len 1048576 --max-num-seqs 128 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 --data-parallel-size 2 diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh index 5e4ce15..fb18812 100755 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh @@ -166,5 +166,8 @@ export SEARCH_ADDEND=16 # If the initial concurrency violates the TTFT SLO, search downward. Stop at # the first acceptable value (16 -> 8; only try 1 when 8 still violates it). export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1" +# When concurrency 1 still has a severely excessive TTFT, stop the remaining +# shapes in this TP/DP group. Zero disables this rule. +export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}" adaptive_main "$@" diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh.bak.20260716-084832 b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh.bak.20260716-084832 new file mode 100755 index 0000000..1a4b3bc --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/run_adaptive_concurrency_add16.sh.bak.20260716-084832 @@ -0,0 +1,167 @@ +#!/usr/bin/env bash +# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape. +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")" + +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/lib.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/platform.sh" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/adaptive_config.env" +# shellcheck source=/dev/null +source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh" + +ENGINE="vllm" +ENGINE_PORT="$VLLM_PORT" +RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" +ACTIVE_ENGINE_SERVER_LOG="" + +if [[ -x "${VENV_CLIENT}/bin/python" ]]; then + PYTHON="${VENV_CLIENT}/bin/python" +else + PYTHON="$(command -v python3)" +fi + +DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}" +DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}" + +engine_is_healthy() { + curl --fail --silent --show-error --max-time 5 \ + "http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1 +} + +engine_stop_server() { + local tp="$1" + local dp="$2" + local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid" + + if [[ -f "$pid_file" ]]; then + local pid + pid="$(cat "$pid_file")" + if kill -0 "$pid" 2>/dev/null; then + log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}" + kill "$pid" 2>/dev/null || true + sleep 5 + kill -9 "$pid" 2>/dev/null || true + fi + rm -f "$pid_file" + fi + docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true + ACTIVE_ENGINE_SERVER_LOG="" + sleep 2 +} + +engine_build_server_args() { + local tp="$1" + local dp="$2" + local -a args=( + vllm serve "$MODEL_PATH" + --trust-remote-code + --kv-cache-dtype "$KV_CACHE_DTYPE" + --block-size "$BLOCK_SIZE" + --tensor-parallel-size "$tp" + --gpu-memory-utilization "$GPU_MEMORY_UTILIZATION" + --max-model-len "$MAX_MODEL_LEN" + --max-num-seqs "$MAX_NUM_SEQS" + --no-enable-flashinfer-autotune + --host 0.0.0.0 + --port "$ENGINE_PORT" + ) + if (( dp > 1 )); then + args+=(--data-parallel-size "$dp") + fi + printf '%q ' "${args[@]}" +} + +engine_start_server() { + local tp="$1" + local dp="$2" + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + log "starting vllm server tp=${tp} dp=${dp}" + bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1 + if ! engine_is_healthy; then + log "ERROR: vllm health check failed tp=${tp} dp=${dp}" + return 1 + fi + ACTIVE_ENGINE_SERVER_LOG="$( + find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \ + -name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \ + -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2- + )" + log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}" +} + +engine_detect_oom() { + local detail_log="$1" + local tp="$2" + local dp="$3" + local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory' + local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log" + local -a logs=("$detail_log" "$outer_log") + if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then + logs+=("$ACTIVE_ENGINE_SERVER_LOG") + fi + grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null +} + +engine_run_bench() { + local isl="$1" + local osl="$2" + local concurrency="$3" + local num_prompts="$4" + local output_file="$5" + local warmup_requests + warmup_requests="$(adaptive_warmup_request_count "$concurrency")" + local -a bench_args=( + --backend vllm + --host 127.0.0.1 + --port "$ENGINE_PORT" + --dataset-name "$BENCH_DATASET_NAME" + --random-input-len "$isl" + --random-output-len "$osl" + --random-range-ratio "$RANDOM_RANGE_RATIO" + --num-prompts "$num_prompts" + --max-concurrency "$concurrency" + --request-rate 10000 + --warmup-requests "$warmup_requests" + --output-file "$output_file" + --output-details + --disable-tqdm + ) + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + bench_args+=(--dataset-path "$DATASET_PATH") + else + bench_args+=(--tokenize-prompt) + fi + + if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then + local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}") + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro") + fi + docker run --rm \ + --network host \ + "${volume_args[@]}" \ + -e PYTHONUNBUFFERED=1 \ + -e HF_HUB_OFFLINE=1 \ + -e TRANSFORMERS_OFFLINE=1 \ + "$DOCKER_CLIENT_IMAGE" \ + python -m sglang.bench_serving "${bench_args[@]}" + else + "$PYTHON" -m sglang.bench_serving "${bench_args[@]}" + fi +} + +export -f engine_run_bench +export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT +export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON + +export SEARCH_START_CONCURRENCY=16 +export SEARCH_ADDEND=16 + +adaptive_main "$@" diff --git a/scripts/common/adaptive_bench_lib.sh b/scripts/common/adaptive_bench_lib.sh index fd5444e..12fd1b6 100755 --- a/scripts/common/adaptive_bench_lib.sh +++ b/scripts/common/adaptive_bench_lib.sh @@ -77,6 +77,10 @@ adaptive_validate_config() { log "ERROR: ENABLE_TTFT_SLO_STOP must be 0 or 1, got ${ENABLE_TTFT_SLO_STOP}" return 1 fi + if ! [[ "${TTFT_GROUP_SKIP_MS:-0}" =~ ^[0-9]+$ ]]; then + log "ERROR: TTFT_GROUP_SKIP_MS must be a non-negative integer, got ${TTFT_GROUP_SKIP_MS}" + return 1 + fi local previous_backoff="$SEARCH_START_CONCURRENCY" local backoff for backoff in ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}; do @@ -391,6 +395,7 @@ adaptive_run_shape() { ADAPTIVE_RESTART_NEEDED=0 ADAPTIVE_FATAL_WORKLOAD=0 + ADAPTIVE_SKIP_CONFIG=0 while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do local num_prompts=$((concurrency * NUM_PROMPTS_MULTIPLIER)) @@ -456,6 +461,15 @@ adaptive_run_shape() { plateau_start_concurrency="" continue fi + if (( concurrency == 1 && ${TTFT_GROUP_SKIP_MS:-0} > 0 )) && \ + awk -v ttft="$ttft_p95_ms" -v threshold="${TTFT_GROUP_SKIP_MS}" \ + 'BEGIN { exit !(ttft >= threshold) }'; then + stop_reason="TTFT_GROUP_SKIP_THRESHOLD" + shape_status="TTFT_BOUNDARY" + ADAPTIVE_SKIP_CONFIG=1 + log "probe result c=1 total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms >= group_skip_threshold=${TTFT_GROUP_SKIP_MS}ms; skipping remaining shapes for tp=${tp} dp=${dp}" + break + fi stop_reason="TTFT_SLO_EXCEEDED" shape_status="TTFT_BOUNDARY" log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape" @@ -608,6 +622,39 @@ PY return 1 } +adaptive_find_ttft_group_boundary() { + local tp="$1" + local dp="$2" + local points_jsonl="$3" + local threshold_ms="${TTFT_GROUP_SKIP_MS:-0}" + if (( threshold_ms == 0 )) || [[ ! -f "$points_jsonl" ]]; then + return 1 + fi + "$PYTHON" - "$tp" "$dp" "$threshold_ms" "$points_jsonl" <<'PY' +import json, sys + +tp, dp = int(sys.argv[1]), int(sys.argv[2]) +threshold = float(sys.argv[3]) +best = None +with open(sys.argv[4], "r", encoding="utf-8") as f: + for line in f: + try: + point = json.loads(line) + if point.get("tp") != tp or point.get("dp") != dp: + continue + if int(point.get("concurrency", -1)) != 1: + continue + ttft = float(point["ttft_p95_ms"]) + except (ValueError, TypeError, KeyError, json.JSONDecodeError): + continue + if ttft >= threshold and (best is None or ttft > best[0]): + best = (ttft, point.get("isl"), point.get("osl")) +if best is None: + raise SystemExit(1) +print(f"isl={best[1]} osl={best[2]} c=1 ttft_p95={best[0]:.3f}ms") +PY +} + adaptive_main() { local resume_mode=0 local resume_shapes_jsonl="" @@ -683,7 +730,7 @@ adaptive_main() { fi log "experiment=${EXPERIMENT_NAME} engine=${ENGINE} run_id=${RUN_ID}" - log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} initial_backoff=[${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-none}] min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}" + log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} initial_backoff=[${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-none}] min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS} ttft_group_skip_ms=${TTFT_GROUP_SKIP_MS:-0}" log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}" "$PYTHON" - \ @@ -706,6 +753,7 @@ adaptive_main() { "$BENCH_WARMUP_MAX_REQUESTS" \ "${TTFT_SLO_MS:-4000}" \ "${ENABLE_TTFT_SLO_STOP:-1}" \ + "${TTFT_GROUP_SKIP_MS:-0}" \ > "${ADAPTIVE_RUN_ROOT}/run_manifest.json" <<'PY' import json, sys @@ -713,7 +761,7 @@ import json, sys tokenize_prompt, random_range_ratio, search_start, search_cap, search_multiplier, initial_backoff_concurrencies, prompts_multiplier, min_gain_pct, plateau_patience, warmup_max_requests, ttft_slo_ms, - enable_ttft_slo_stop) = sys.argv[1:20] + enable_ttft_slo_stop, ttft_group_skip_ms) = sys.argv[1:21] print(json.dumps({ "experiment": experiment, @@ -738,6 +786,7 @@ print(json.dumps({ "warmup_max_requests": int(warmup_max_requests), "ttft_slo_ms": float(ttft_slo_ms), "enable_ttft_slo_stop": int(enable_ttft_slo_stop), + "ttft_group_skip_ms": float(ttft_group_skip_ms), }, }, indent=2, ensure_ascii=False)) PY @@ -755,6 +804,12 @@ PY if ! adaptive_matches_filter "$tp" "$TP_LIST"; then continue fi + local prior_ttft_boundary="" + if (( resume_mode == 1 )) && \ + prior_ttft_boundary="$(adaptive_find_ttft_group_boundary "$tp" "$dp" "$ADAPTIVE_POINTS_JSONL")"; then + log "RESUME skip remaining config tp=${tp} dp=${dp}: prior ${prior_ttft_boundary} >= group_skip_threshold=${TTFT_GROUP_SKIP_MS}ms" + continue + fi config_root="${ADAPTIVE_RUN_ROOT}/tp${tp}_dp${dp}" mkdir -p "${config_root}/raw_outputs" "${config_root}/metrics" \ "${config_root}/logs" "${config_root}/gpu_logs" @@ -816,6 +871,10 @@ PY abort_all=1 break fi + if (( ADAPTIVE_SKIP_CONFIG == 1 )); then + log "TTFT group-skip threshold reached; skipping remaining shapes for tp=${tp} dp=${dp}" + break + fi if (( ADAPTIVE_RESTART_NEEDED == 1 )); then if ! adaptive_restart_server "$tp" "$dp"; then log "ERROR: server restart failed; skipping remaining shapes for tp=${tp} dp=${dp}" diff --git a/scripts/common/adaptive_bench_lib.sh.bak.20260716-084832 b/scripts/common/adaptive_bench_lib.sh.bak.20260716-084832 new file mode 100755 index 0000000..fcd06f4 --- /dev/null +++ b/scripts/common/adaptive_bench_lib.sh.bak.20260716-084832 @@ -0,0 +1,778 @@ +#!/usr/bin/env bash +# Shared adaptive-concurrency search loop for serving benchmarks. + +ADAPTIVE_COMMON_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ADAPTIVE_HELPER="${ADAPTIVE_COMMON_DIR}/adaptive_concurrency.py" + +adaptive_timestamp() { + date --iso-8601=seconds +} + +adaptive_json_number_or_null() { + local value="${1:-}" + if [[ -n "$value" ]]; then + printf '%s' "$value" + else + printf 'null' + fi +} + +adaptive_warmup_request_count() { + local concurrency="$1" + if (( BENCH_WARMUP_MAX_REQUESTS > 0 && concurrency > BENCH_WARMUP_MAX_REQUESTS )); then + printf '%s' "$BENCH_WARMUP_MAX_REQUESTS" + else + printf '%s' "$concurrency" + fi +} +export -f adaptive_warmup_request_count + +adaptive_validate_config() { + local name + for name in SEARCH_START_CONCURRENCY SEARCH_MAX_CONCURRENCY \ + NUM_PROMPTS_MULTIPLIER PLATEAU_PATIENCE MAX_POINT_RETRIES \ + BENCH_WARMUP_MAX_REQUESTS; do + if ! [[ "${!name}" =~ ^[0-9]+$ ]]; then + log "ERROR: ${name} must be a non-negative integer, got ${!name}" + return 1 + fi + done + if (( SEARCH_START_CONCURRENCY < 1 )); then + log "ERROR: SEARCH_START_CONCURRENCY must be at least 1" + return 1 + fi + if (( SEARCH_MAX_CONCURRENCY < SEARCH_START_CONCURRENCY )); then + log "ERROR: SEARCH_MAX_CONCURRENCY must be >= SEARCH_START_CONCURRENCY" + return 1 + fi + if [[ -n "${SEARCH_ADDEND:-}" ]]; then + if ! [[ "${SEARCH_ADDEND}" =~ ^[0-9]+$ ]] || (( SEARCH_ADDEND < 1 )); then + log "ERROR: SEARCH_ADDEND must be at least 1" + return 1 + fi + else + if ! [[ "${SEARCH_MULTIPLIER:-}" =~ ^[0-9]+$ ]] || (( SEARCH_MULTIPLIER < 2 )); then + log "ERROR: SEARCH_MULTIPLIER must be at least 2" + return 1 + fi + fi + if (( NUM_PROMPTS_MULTIPLIER < 1 || PLATEAU_PATIENCE < 1 )); then + log "ERROR: NUM_PROMPTS_MULTIPLIER and PLATEAU_PATIENCE must be at least 1" + return 1 + fi + if ! [[ "$TPS_MIN_GAIN_PCT" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + log "ERROR: TPS_MIN_GAIN_PCT must be a non-negative number" + return 1 + fi + if ! [[ "$RANDOM_RANGE_RATIO" =~ ^[0-9]+([.][0-9]+)?$ ]] || \ + ! awk -v ratio="$RANDOM_RANGE_RATIO" 'BEGIN { exit !(ratio >= 0 && ratio <= 1) }'; then + log "ERROR: RANDOM_RANGE_RATIO must be between 0.0 and 1.0" + return 1 + fi + if ! [[ "${TTFT_SLO_MS:-4000}" =~ ^[0-9]+$ ]] || (( TTFT_SLO_MS < 1 )); then + log "ERROR: TTFT_SLO_MS must be a positive integer, got ${TTFT_SLO_MS}" + return 1 + fi + if [[ "${ENABLE_TTFT_SLO_STOP:-1}" != "0" && "${ENABLE_TTFT_SLO_STOP:-1}" != "1" ]]; then + log "ERROR: ENABLE_TTFT_SLO_STOP must be 0 or 1, got ${ENABLE_TTFT_SLO_STOP}" + return 1 + fi +} + +adaptive_start_gpu_monitor() { + local csv_path="$1" + mkdir -p "$(dirname "$csv_path")" + nvidia-smi \ + --query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \ + --format=csv \ + -l "$GPU_MEM_SAMPLE_INTERVAL_S" \ + > "$csv_path" 2>/dev/null & + echo $! +} + +adaptive_stop_gpu_monitor() { + local pid="$1" + if kill -0 "$pid" 2>/dev/null; then + kill "$pid" 2>/dev/null || true + sleep 1 + kill -9 "$pid" 2>/dev/null || true + fi +} + +adaptive_warmup() { + local log_path="$1" + log "running synthetic warmup" + timeout "$SCENARIO_TIMEOUT_S" bash -c \ + 'engine_run_bench 1024 128 1 1 /dev/null' \ + >> "$log_path" 2>&1 +} + +adaptive_restart_server() { + local tp="$1" + local dp="$2" + log "restarting ${ENGINE} server tp=${tp} dp=${dp}" + engine_stop_server "$tp" "$dp" + sleep "$SERVER_RESTART_COOLDOWN_S" + engine_start_server "$tp" "$dp" + adaptive_warmup "${ADAPTIVE_LOG_DIR}/${ENGINE}_tp${tp}_dp${dp}.warmup.log" +} + +adaptive_append_point_from_metrics() { + local metrics_file="$1" + local tp="$2" + local dp="$3" + local mark="$4" + local isl="$5" + local osl="$6" + local concurrency="$7" + local num_prompts="$8" + local attempt="$9" + local gain_pct="${10:-}" + local plateau_streak="${11:-0}" + local raw_file="${12}" + local detail_log="${13}" + local gain_json + gain_json="$(adaptive_json_number_or_null "$gain_pct")" + + jq -c \ + --arg timestamp "$(adaptive_timestamp)" \ + --arg engine "$ENGINE" \ + --argjson tp "$tp" \ + --argjson dp "$dp" \ + --arg mark "$mark" \ + --argjson isl "$isl" \ + --argjson osl "$osl" \ + --argjson concurrency "$concurrency" \ + --argjson num_prompts "$num_prompts" \ + --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ + --argjson attempt "$attempt" \ + --argjson gain_pct "$gain_json" \ + --argjson plateau_streak "$plateau_streak" \ + --arg raw_file "$raw_file" \ + --arg detail_log "$detail_log" \ + '. + { + timestamp: $timestamp, + engine: $engine, + tp: $tp, + dp: $dp, + mark: $mark, + isl: $isl, + osl: $osl, + concurrency: $concurrency, + num_prompts: $num_prompts, + warmup_requests: $warmup_requests, + attempt: $attempt, + gain_pct: $gain_pct, + plateau_streak: $plateau_streak, + error_type: "", + raw_file: $raw_file, + detail_log: $detail_log + }' "$metrics_file" >> "$ADAPTIVE_POINTS_JSONL" +} + +adaptive_append_failed_point() { + local tp="$1" + local dp="$2" + local mark="$3" + local isl="$4" + local osl="$5" + local concurrency="$6" + local num_prompts="$7" + local attempt="$8" + local status="$9" + local error_type="${10}" + local raw_file="${11}" + local detail_log="${12}" + + jq -nc \ + --arg timestamp "$(adaptive_timestamp)" \ + --arg engine "$ENGINE" \ + --argjson tp "$tp" \ + --argjson dp "$dp" \ + --arg mark "$mark" \ + --argjson isl "$isl" \ + --argjson osl "$osl" \ + --argjson concurrency "$concurrency" \ + --argjson num_prompts "$num_prompts" \ + --argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \ + --argjson attempt "$attempt" \ + --arg status "$status" \ + --arg error_type "$error_type" \ + --arg raw_file "$raw_file" \ + --arg detail_log "$detail_log" \ + '{ + timestamp: $timestamp, + engine: $engine, + tp: $tp, + dp: $dp, + mark: $mark, + isl: $isl, + osl: $osl, + concurrency: $concurrency, + num_prompts: $num_prompts, + warmup_requests: $warmup_requests, + attempt: $attempt, + status: $status, + completed: 0, + failed: $num_prompts, + error_type: $error_type, + validation_errors: [], + gain_pct: null, + plateau_streak: 0, + raw_file: $raw_file, + detail_log: $detail_log + }' >> "$ADAPTIVE_POINTS_JSONL" +} + +adaptive_append_shape_summary() { + local tp="$1" + local dp="$2" + local mark="$3" + local isl="$4" + local osl="$5" + local status="$6" + local stop_reason="$7" + local tested_points="$8" + local max_successful_concurrency="$9" + local saturation_concurrency="${10:-}" + local stop_probe_concurrency="${11:-}" + local best_tps_concurrency="${12:-}" + local best_total_tps="${13:-}" + local last_total_tps="${14:-}" + local saturation_json stop_probe_json best_c_json best_tps_json last_tps_json + saturation_json="$(adaptive_json_number_or_null "$saturation_concurrency")" + stop_probe_json="$(adaptive_json_number_or_null "$stop_probe_concurrency")" + best_c_json="$(adaptive_json_number_or_null "$best_tps_concurrency")" + best_tps_json="$(adaptive_json_number_or_null "$best_total_tps")" + last_tps_json="$(adaptive_json_number_or_null "$last_total_tps")" + + jq -nc \ + --arg timestamp "$(adaptive_timestamp)" \ + --arg engine "$ENGINE" \ + --argjson tp "$tp" \ + --argjson dp "$dp" \ + --arg mark "$mark" \ + --argjson isl "$isl" \ + --argjson osl "$osl" \ + --arg status "$status" \ + --arg stop_reason "$stop_reason" \ + --argjson tested_points "$tested_points" \ + --argjson search_cap "$SEARCH_MAX_CONCURRENCY" \ + --argjson max_successful_concurrency "$max_successful_concurrency" \ + --argjson saturation_concurrency "$saturation_json" \ + --argjson stop_probe_concurrency "$stop_probe_json" \ + --argjson best_tps_concurrency "$best_c_json" \ + --argjson best_total_tps "$best_tps_json" \ + --argjson last_total_tps "$last_tps_json" \ + '{ + timestamp: $timestamp, + engine: $engine, + tp: $tp, + dp: $dp, + mark: $mark, + isl: $isl, + osl: $osl, + status: $status, + stop_reason: $stop_reason, + tested_points: $tested_points, + search_cap: $search_cap, + max_successful_concurrency: $max_successful_concurrency, + saturation_concurrency: $saturation_concurrency, + stop_probe_concurrency: $stop_probe_concurrency, + best_tps_concurrency: $best_tps_concurrency, + best_total_tps: $best_total_tps, + last_total_tps: $last_total_tps + }' >> "$ADAPTIVE_SHAPES_JSONL" +} + +adaptive_run_point() { + local tp="$1" + local dp="$2" + local isl="$3" + local osl="$4" + local concurrency="$5" + local num_prompts="$6" + local config_root="$7" + local attempt bench_rc parse_rc gpu_pid + + POINT_RAW_FILE="" + POINT_DETAIL_LOG="" + POINT_METRICS_FILE="" + POINT_ATTEMPT=0 + POINT_ERROR_TYPE="" + + for (( attempt = 1; attempt <= MAX_POINT_RETRIES + 1; attempt++ )); do + POINT_ATTEMPT="$attempt" + POINT_RAW_FILE="${config_root}/raw_outputs/${ENGINE}_adaptive_c${concurrency}_i${isl}_o${osl}_a${attempt}.jsonl" + POINT_DETAIL_LOG="${config_root}/logs/${ENGINE}_c${concurrency}_i${isl}_o${osl}_a${attempt}.log" + POINT_METRICS_FILE="${config_root}/metrics/${ENGINE}_c${concurrency}_i${isl}_o${osl}_a${attempt}.json" + local gpu_csv="${config_root}/gpu_logs/gpu_c${concurrency}_i${isl}_o${osl}_a${attempt}.csv" + + log "probe ${ENGINE} tp=${tp} dp=${dp} isl=${isl} osl=${osl} c=${concurrency} warmup=$(adaptive_warmup_request_count "$concurrency") n=${num_prompts} attempt=${attempt}" + gpu_pid="$(adaptive_start_gpu_monitor "$gpu_csv")" + bench_rc=0 + timeout "$SCENARIO_TIMEOUT_S" bash -c \ + 'engine_run_bench "$@"' _ \ + "$isl" "$osl" "$concurrency" "$num_prompts" "$POINT_RAW_FILE" \ + > "$POINT_DETAIL_LOG" 2>&1 || bench_rc=$? + adaptive_stop_gpu_monitor "$gpu_pid" + + if (( bench_rc == 0 )); then + parse_rc=0 + "$PYTHON" "$ADAPTIVE_HELPER" parse-result \ + --input "$POINT_RAW_FILE" \ + --output "$POINT_METRICS_FILE" \ + --expected-prompts "$num_prompts" \ + --isl "$isl" \ + --osl "$osl" \ + --input-tolerance-pct "$INPUT_LENGTH_TOLERANCE_PCT" \ + --output-tolerance-pct "$OUTPUT_LENGTH_TOLERANCE_PCT" \ + >> "$POINT_DETAIL_LOG" 2>&1 || parse_rc=$? + if (( parse_rc == 0 )); then + return 0 + fi + POINT_ERROR_TYPE="workload validation failed" + return 21 + fi + + if engine_detect_oom "$POINT_DETAIL_LOG" "$tp" "$dp"; then + POINT_ERROR_TYPE="detected out of memory" + return 20 + fi + + POINT_ERROR_TYPE="benchmark rc=${bench_rc}" + if (( attempt <= MAX_POINT_RETRIES )); then + log "probe failed without OOM; restarting server before retry" + if ! adaptive_restart_server "$tp" "$dp"; then + POINT_ERROR_TYPE="benchmark failed and server restart failed" + return 22 + fi + continue + fi + return 22 + done +} + +adaptive_run_shape() { + local tp="$1" + local dp="$2" + local mark="$3" + local isl="$4" + local osl="$5" + local config_root="$6" + local concurrency="$SEARCH_START_CONCURRENCY" + local previous_tps="" + local current_tps="" + local best_tps="" + local best_concurrency="" + local max_successful_concurrency=0 + local last_total_tps="" + local plateau_streak=0 + local plateau_start_concurrency="" + local saturation_concurrency="" + local stop_probe_concurrency="" + local stop_reason="SEARCH_CAP_REACHED" + local shape_status="COMPLETED" + local tested_points=0 + local gain_pct="" + local meaningful_gain=1 + local point_rc=0 + + ADAPTIVE_RESTART_NEEDED=0 + ADAPTIVE_FATAL_WORKLOAD=0 + + while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do + local num_prompts=$((concurrency * NUM_PROMPTS_MULTIPLIER)) + point_rc=0 + adaptive_run_point "$tp" "$dp" "$isl" "$osl" "$concurrency" "$num_prompts" "$config_root" || point_rc=$? + tested_points=$((tested_points + 1)) + stop_probe_concurrency="$concurrency" + + if (( point_rc == 0 )); then + current_tps="$(jq -r '.total_tps' "$POINT_METRICS_FILE")" + gain_pct="" + meaningful_gain=1 + if [[ -n "$previous_tps" ]]; then + IFS=$'\t' read -r gain_pct meaningful_gain < <( + "$PYTHON" "$ADAPTIVE_HELPER" gain \ + --previous "$previous_tps" \ + --current "$current_tps" \ + --threshold-pct "$TPS_MIN_GAIN_PCT" + ) + if (( meaningful_gain == 1 )); then + plateau_streak=0 + plateau_start_concurrency="" + else + plateau_streak=$((plateau_streak + 1)) + if [[ -z "$plateau_start_concurrency" ]]; then + plateau_start_concurrency="$concurrency" + fi + fi + fi + + if [[ -z "$best_tps" ]] || awk -v current="$current_tps" -v best="$best_tps" 'BEGIN { exit !(current > best) }'; then + best_tps="$current_tps" + best_concurrency="$concurrency" + fi + max_successful_concurrency="$concurrency" + last_total_tps="$current_tps" + + # TTFT SLO check: stop searching if TTFT P95 exceeds the SLO threshold + local ttft_p95_ms="" + if [[ "${ENABLE_TTFT_SLO_STOP:-1}" == "1" ]]; then + ttft_p95_ms="$(jq -r '.ttft_p95_ms' "$POINT_METRICS_FILE")" + if awk -v ttft="$ttft_p95_ms" -v slo="${TTFT_SLO_MS:-4000}" 'BEGIN { exit !(ttft > slo) }'; then + adaptive_append_point_from_metrics \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ + "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ + "$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + stop_reason="TTFT_SLO_EXCEEDED" + shape_status="TTFT_BOUNDARY" + log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape" + break + fi + fi + + adaptive_append_point_from_metrics \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ + "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \ + "$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + + log "probe result c=${concurrency} total_tps=${current_tps} gain_pct=${gain_pct:-first} plateau_streak=${plateau_streak}/${PLATEAU_PATIENCE}" + if (( plateau_streak >= PLATEAU_PATIENCE )); then + saturation_concurrency="$plateau_start_concurrency" + stop_reason="TPS_PLATEAU" + break + fi + previous_tps="$current_tps" + elif (( point_rc == 20 )); then + adaptive_append_failed_point \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \ + "$POINT_ATTEMPT" "OOM" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + shape_status="OOM_BOUNDARY" + stop_reason="OOM" + ADAPTIVE_RESTART_NEEDED=1 + break + elif (( point_rc == 21 )); then + adaptive_append_point_from_metrics \ + "$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \ + "$concurrency" "$num_prompts" "$POINT_ATTEMPT" "" 0 \ + "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + shape_status="INVALID_WORKLOAD" + stop_reason="INVALID_WORKLOAD" + ADAPTIVE_FATAL_WORKLOAD=1 + break + else + adaptive_append_failed_point \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \ + "$POINT_ATTEMPT" "FAILED" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG" + shape_status="FAILED" + stop_reason="BENCHMARK_FAILED" + ADAPTIVE_RESTART_NEEDED=1 + break + fi + + if (( concurrency == SEARCH_MAX_CONCURRENCY )); then + stop_reason="SEARCH_CAP_REACHED" + break + fi + local next_concurrency + if [[ -n "${SEARCH_ADDEND:-}" ]]; then + next_concurrency=$((concurrency + SEARCH_ADDEND)) + else + next_concurrency=$((concurrency * SEARCH_MULTIPLIER)) + fi + if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then + next_concurrency="$SEARCH_MAX_CONCURRENCY" + fi + concurrency="$next_concurrency" + done + + adaptive_append_shape_summary \ + "$tp" "$dp" "$mark" "$isl" "$osl" "$shape_status" "$stop_reason" \ + "$tested_points" "$max_successful_concurrency" "$saturation_concurrency" \ + "$stop_probe_concurrency" "$best_concurrency" "$best_tps" "$last_total_tps" + + log "shape done tp=${tp} dp=${dp} isl=${isl} osl=${osl} stop=${stop_reason} saturation_c=${saturation_concurrency:-none} best_c=${best_concurrency:-none} best_total_tps=${best_tps:-none}" +} + +adaptive_matches_filter() { + local value="$1" + local filter="${2:-}" + [[ -z "$filter" || " $filter " == *" $value "* ]] +} + +adaptive_print_sequence() { + local concurrency="$SEARCH_START_CONCURRENCY" + local -a sequence=() + while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do + sequence+=("$concurrency") + if (( concurrency == SEARCH_MAX_CONCURRENCY )); then + break + fi + local next_concurrency + if [[ -n "${SEARCH_ADDEND:-}" ]]; then + next_concurrency=$((concurrency + SEARCH_ADDEND)) + else + next_concurrency=$((concurrency * SEARCH_MULTIPLIER)) + fi + if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then + next_concurrency="$SEARCH_MAX_CONCURRENCY" + fi + concurrency="$next_concurrency" + done + printf '%s ' "${sequence[@]}" +} + +adaptive_cleanup_active_server() { + if [[ -n "${ADAPTIVE_ACTIVE_TP:-}" && -n "${ADAPTIVE_ACTIVE_DP:-}" ]]; then + engine_stop_server "$ADAPTIVE_ACTIVE_TP" "$ADAPTIVE_ACTIVE_DP" || true + ADAPTIVE_ACTIVE_TP="" + ADAPTIVE_ACTIVE_DP="" + fi +} + +adaptive_shape_already_tested() { + local tp="$1" + local dp="$2" + local mark="$3" + local isl="$4" + local osl="$5" + local resume_shapes_jsonl="$6" + if [[ ! -f "$resume_shapes_jsonl" ]]; then + return 1 + fi + local count + count="$(jq -s -r --argjson tp "$tp" --argjson dp "$dp" --arg mark "$mark" --argjson isl "$isl" --argjson osl "$osl" ' + [ .[] | select(.tp == $tp and .dp == $dp and .mark == $mark and .isl == $isl and .osl == $osl) ] | length + ' "$resume_shapes_jsonl" 2>/dev/null)" + if [[ -z "$count" ]]; then + return 1 + fi + if (( count > 0 )); then + return 0 + fi + return 1 +} + +adaptive_main() { + local resume_mode=0 + local resume_shapes_jsonl="" + if [[ -n "${RESUME_RUN_ID:-}" ]]; then + ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RESUME_RUN_ID}" + if [[ ! -d "$ADAPTIVE_RUN_ROOT" ]]; then + log "ERROR: RESUME_RUN_ID=${RESUME_RUN_ID} does not exist under ${RESULT_BASE}" + return 1 + fi + resume_mode=1 + RUN_ID="$RESUME_RUN_ID" + resume_shapes_jsonl="${ADAPTIVE_RUN_ROOT}/adaptive_shapes.jsonl" + if [[ ! -f "$resume_shapes_jsonl" ]]; then + log "WARNING: no adaptive_shapes.jsonl found in ${ADAPTIVE_RUN_ROOT}; will resume with empty history" + fi + log "RESUME mode: continuing run_id=${RESUME_RUN_ID}" + else + RUN_ID="${RUN_ID:-adaptive_$(date '+%Y%m%d-%H%M%S')}" + ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RUN_ID}" + fi + RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}" + MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}" + MATRIX_MODE="${MATRIX_MODE:-Y}" + ADAPTIVE_LOG_DIR="${ADAPTIVE_RUN_ROOT}/logs" + ADAPTIVE_POINTS_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_points.jsonl" + ADAPTIVE_SHAPES_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_shapes.jsonl" + local shapes_tsv="${ADAPTIVE_RUN_ROOT}/shapes.tsv" + mkdir -p "$ADAPTIVE_LOG_DIR" + if (( resume_mode == 0 )); then + : > "$ADAPTIVE_POINTS_JSONL" + : > "$ADAPTIVE_SHAPES_JSONL" + fi + log_init "${ADAPTIVE_LOG_DIR}/orchestrator.log" + + adaptive_validate_config + "$PYTHON" "$ADAPTIVE_HELPER" shapes \ + --matrix "$MATRIX_FILE" \ + --mode "$MATRIX_MODE" \ + > "$shapes_tsv" + + local tokenize_prompt_json=true + if [[ "$BENCH_DATASET_NAME" == "random" ]]; then + local required_dataset_rows=$((SEARCH_MAX_CONCURRENCY * NUM_PROMPTS_MULTIPLIER)) + local dataset_capacity=0 + local dataset_error="" + if [[ ! -f "$DATASET_PATH" ]]; then + dataset_error="DATASET_PATH does not exist: ${DATASET_PATH}" + elif ! dataset_capacity="$( + "$PYTHON" "$ADAPTIVE_HELPER" dataset-capacity --path "$DATASET_PATH" 2>/dev/null + )"; then + dataset_error="DATASET_PATH is not a valid ShareGPT JSON file: ${DATASET_PATH}" + fi + if [[ -n "$dataset_error" ]]; then + if [[ "$DRY_RUN" == "1" ]]; then + log "WARNING: ${dataset_error}" + else + log "ERROR: ${dataset_error}" + return 1 + fi + fi + if [[ -z "$dataset_error" ]] && (( dataset_capacity < required_dataset_rows )); then + if [[ "$DRY_RUN" == "1" ]]; then + log "WARNING: DATASET_PATH has ${dataset_capacity} usable conversations, but a real run may request ${required_dataset_rows}." + else + log "ERROR: DATASET_PATH has ${dataset_capacity} usable conversations, but adaptive search may request ${required_dataset_rows}. Set DATASET_PATH to a larger ShareGPT file or explicitly use BENCH_DATASET_NAME=random-ids." + return 1 + fi + fi + tokenize_prompt_json=false + elif [[ "$BENCH_DATASET_NAME" != random* ]]; then + log "ERROR: BENCH_DATASET_NAME must be random or start with random" + return 1 + fi + + log "experiment=${EXPERIMENT_NAME} engine=${ENGINE} run_id=${RUN_ID}" + log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}" + log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}" + + jq -n \ + --arg experiment "$EXPERIMENT_NAME" \ + --arg engine "$ENGINE" \ + --arg run_id "$RUN_ID" \ + --arg model "$MODEL_PATH" \ + --arg hardware "$HARDWARE" \ + --arg matrix "$MATRIX_FILE" \ + --arg dataset "$BENCH_DATASET_NAME" \ + --argjson tokenize_prompt "$tokenize_prompt_json" \ + --argjson random_range_ratio "$RANDOM_RANGE_RATIO" \ + --argjson search_start "$SEARCH_START_CONCURRENCY" \ + --argjson search_cap "$SEARCH_MAX_CONCURRENCY" \ + --argjson search_multiplier "$SEARCH_MULTIPLIER" \ + --argjson prompts_multiplier "$NUM_PROMPTS_MULTIPLIER" \ + --argjson min_gain_pct "$TPS_MIN_GAIN_PCT" \ + --argjson plateau_patience "$PLATEAU_PATIENCE" \ + --argjson warmup_max_requests "$BENCH_WARMUP_MAX_REQUESTS" \ + --argjson ttft_slo_ms "${TTFT_SLO_MS:-4000}" \ + --argjson enable_ttft_slo_stop "${ENABLE_TTFT_SLO_STOP:-1}" \ + '{ + experiment: $experiment, + engine: $engine, + run_id: $run_id, + model: $model, + hardware: $hardware, + matrix: $matrix, + dataset: $dataset, + tokenize_prompt: $tokenize_prompt, + random_range_ratio: $random_range_ratio, + search: { + start_concurrency: $search_start, + max_concurrency: $search_cap, + multiplier: $search_multiplier, + num_prompts_multiplier: $prompts_multiplier, + min_tps_gain_pct: $min_gain_pct, + plateau_patience: $plateau_patience, + warmup_max_requests: $warmup_max_requests, + ttft_slo_ms: $ttft_slo_ms, + enable_ttft_slo_stop: $enable_ttft_slo_stop + } + }' > "${ADAPTIVE_RUN_ROOT}/run_manifest.json" + + ADAPTIVE_ACTIVE_TP="" + ADAPTIVE_ACTIVE_DP="" + trap adaptive_cleanup_active_server EXIT + trap 'adaptive_cleanup_active_server; exit 130' INT + trap 'adaptive_cleanup_active_server; exit 143' TERM + + local abort_all=0 + local cfg tp dp config_root server_args shape_count mark isl osl + for cfg in "${PARALLEL_CONFIGS[@]}"; do + read -r tp dp <<< "$cfg" + if ! adaptive_matches_filter "$tp" "$TP_LIST"; then + continue + fi + config_root="${ADAPTIVE_RUN_ROOT}/tp${tp}_dp${dp}" + mkdir -p "${config_root}/raw_outputs" "${config_root}/metrics" \ + "${config_root}/logs" "${config_root}/gpu_logs" + server_args="$(engine_build_server_args "$tp" "$dp")" + printf '%s\n' "$server_args" > "${config_root}/server_cmd.txt" + + if [[ "$DRY_RUN" == "1" ]]; then + log "DRY_RUN server tp=${tp} dp=${dp}: ${server_args}" + shape_count=0 + while IFS=$'\t' read -r mark isl osl; do + [[ "$mark" == "mark" ]] && continue + adaptive_matches_filter "$isl" "$ISL_LIST" || continue + adaptive_matches_filter "$osl" "$OSL_LIST" || continue + if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then + break + fi + shape_count=$((shape_count + 1)) + if (( resume_mode == 1 )) && adaptive_shape_already_tested "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl"; then + log "DRY_RUN RESUME skip already tested shape tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark}" + continue + fi + log "DRY_RUN tp=${tp} dp=${dp} isl=${isl} osl=${osl} C=[$(adaptive_print_sequence)]" + done < "$shapes_tsv" + continue + fi + + ADAPTIVE_ACTIVE_TP="$tp" + ADAPTIVE_ACTIVE_DP="$dp" + if ! engine_start_server "$tp" "$dp"; then + log "ERROR: failed to start ${ENGINE} tp=${tp} dp=${dp}; skipping config" + adaptive_cleanup_active_server + continue + fi + adaptive_warmup "${ADAPTIVE_LOG_DIR}/${ENGINE}_tp${tp}_dp${dp}.warmup.log" || { + log "ERROR: warmup failed for tp=${tp} dp=${dp}; skipping config" + adaptive_cleanup_active_server + continue + } + + shape_count=0 + while IFS=$'\t' read -r mark isl osl; do + [[ "$mark" == "mark" ]] && continue + adaptive_matches_filter "$isl" "$ISL_LIST" || continue + adaptive_matches_filter "$osl" "$OSL_LIST" || continue + if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then + log "GRID_LIMIT=${GRID_LIMIT} reached for tp=${tp} dp=${dp}" + break + fi + shape_count=$((shape_count + 1)) + if (( resume_mode == 1 )) && adaptive_shape_already_tested "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl"; then + log "RESUME skip already tested shape tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark}" + continue + fi + log "===== shape START tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark} =====" + adaptive_run_shape "$tp" "$dp" "$mark" "$isl" "$osl" "$config_root" + + if (( ADAPTIVE_FATAL_WORKLOAD == 1 )); then + log "ERROR: invalid synthetic workload; aborting remaining searches" + abort_all=1 + break + fi + if (( ADAPTIVE_RESTART_NEEDED == 1 )); then + if ! adaptive_restart_server "$tp" "$dp"; then + log "ERROR: server restart failed; skipping remaining shapes for tp=${tp} dp=${dp}" + break + fi + fi + done < "$shapes_tsv" + + adaptive_cleanup_active_server + if (( abort_all == 1 )); then + break + fi + done + + "$PYTHON" "$ADAPTIVE_HELPER" summarize \ + --points "$ADAPTIVE_POINTS_JSONL" \ + --shapes "$ADAPTIVE_SHAPES_JSONL" \ + --output-dir "$ADAPTIVE_RUN_ROOT" \ + >> "${ADAPTIVE_LOG_DIR}/summarize.log" 2>&1 + log "adaptive results saved to ${ADAPTIVE_RUN_ROOT}" + + trap - EXIT INT TERM + adaptive_cleanup_active_server + if (( abort_all == 1 )); then + return 2 + fi +}