update stop method
This commit is contained in:
SSKJ Dev
parent
589d3b079d
commit
0591c401ab
@ -14,3 +14,47 @@
|
|||||||
{"status":"COMPLETED","completed":400,"failed":0,"duration_s":235.45233723000274,"request_tps":1.6988576316796469,"input_tps":1739.6302148399584,"output_tps":3479.260429679917,"total_tps":5218.890644519875,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":1586.3785705005284,"ttft_p95_ms":3133.5033947099873,"ttft_p99_ms":3142.967526005523,"tpot_p50_ms":22.12632127210705,"tpot_p95_ms":22.91146984701974,"tpot_p99_ms":22.976990950927,"e2e_p50_ms":47054.46670300444,"e2e_p95_ms":48105.06032910416,"e2e_p99_ms":48135.96432496226,"itl_p50_ms":21.907794500293676,"itl_p95_ms":23.54792225087294,"itl_p99_ms":25.59503832002519,"validation_errors":[],"timestamp":"2026-07-16T06:11:01+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":22.791758,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c80_i1024_o2048_a1.log"}
|
{"status":"COMPLETED","completed":400,"failed":0,"duration_s":235.45233723000274,"request_tps":1.6988576316796469,"input_tps":1739.6302148399584,"output_tps":3479.260429679917,"total_tps":5218.890644519875,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":1586.3785705005284,"ttft_p95_ms":3133.5033947099873,"ttft_p99_ms":3142.967526005523,"tpot_p50_ms":22.12632127210705,"tpot_p95_ms":22.91146984701974,"tpot_p99_ms":22.976990950927,"e2e_p50_ms":47054.46670300444,"e2e_p95_ms":48105.06032910416,"e2e_p99_ms":48135.96432496226,"itl_p50_ms":21.907794500293676,"itl_p95_ms":23.54792225087294,"itl_p99_ms":25.59503832002519,"validation_errors":[],"timestamp":"2026-07-16T06:11:01+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":22.791758,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c80_i1024_o2048_a1.log"}
|
||||||
{"status":"COMPLETED","completed":480,"failed":0,"duration_s":261.48785026300175,"request_tps":1.8356493409434549,"input_tps":1879.7049251260978,"output_tps":3759.4098502521956,"total_tps":5639.114775378293,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":2183.7680815006024,"ttft_p95_ms":3805.7011090502783,"ttft_p99_ms":4558.676817088854,"tpot_p50_ms":24.876903371278917,"tpot_p95_ms":25.891116515634604,"tpot_p99_ms":26.181815688893472,"e2e_p50_ms":53079.94003850763,"e2e_p95_ms":54830.756290954014,"e2e_p99_ms":54843.78045572128,"itl_p50_ms":24.47853601188399,"itl_p95_ms":26.07234639581293,"itl_p99_ms":27.980867308797315,"validation_errors":[],"timestamp":"2026-07-16T06:16:08+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":8.051982,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c96_i1024_o2048_a1.log"}
|
{"status":"COMPLETED","completed":480,"failed":0,"duration_s":261.48785026300175,"request_tps":1.8356493409434549,"input_tps":1879.7049251260978,"output_tps":3759.4098502521956,"total_tps":5639.114775378293,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":2183.7680815006024,"ttft_p95_ms":3805.7011090502783,"ttft_p99_ms":4558.676817088854,"tpot_p50_ms":24.876903371278917,"tpot_p95_ms":25.891116515634604,"tpot_p99_ms":26.181815688893472,"e2e_p50_ms":53079.94003850763,"e2e_p95_ms":54830.756290954014,"e2e_p99_ms":54843.78045572128,"itl_p50_ms":24.47853601188399,"itl_p95_ms":26.07234639581293,"itl_p99_ms":27.980867308797315,"validation_errors":[],"timestamp":"2026-07-16T06:16:08+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":8.051982,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c96_i1024_o2048_a1.log"}
|
||||||
{"status":"COMPLETED","completed":560,"failed":0,"duration_s":271.93914603099984,"request_tps":2.059284248602305,"input_tps":2108.70707056876,"output_tps":4217.41414113752,"total_tps":6326.121211706281,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":2360.8657174918335,"ttft_p95_ms":4785.738604894141,"ttft_p99_ms":5083.200499021332,"tpot_p50_ms":25.078506466781775,"tpot_p95_ms":26.261017000906556,"tpot_p99_ms":26.82069243732855,"e2e_p50_ms":54802.16980399564,"e2e_p95_ms":55957.59202675035,"e2e_p99_ms":55983.62555622443,"itl_p50_ms":24.729957003728487,"itl_p95_ms":26.741590403253216,"itl_p99_ms":29.055896277423017,"validation_errors":[],"timestamp":"2026-07-16T06:21:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":12.182877,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c112_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c112_i1024_o2048_a1.log"}
|
{"status":"COMPLETED","completed":560,"failed":0,"duration_s":271.93914603099984,"request_tps":2.059284248602305,"input_tps":2108.70707056876,"output_tps":4217.41414113752,"total_tps":6326.121211706281,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":2360.8657174918335,"ttft_p95_ms":4785.738604894141,"ttft_p99_ms":5083.200499021332,"tpot_p50_ms":25.078506466781775,"tpot_p95_ms":26.261017000906556,"tpot_p99_ms":26.82069243732855,"e2e_p50_ms":54802.16980399564,"e2e_p95_ms":55957.59202675035,"e2e_p99_ms":55983.62555622443,"itl_p50_ms":24.729957003728487,"itl_p95_ms":26.741590403253216,"itl_p99_ms":29.055896277423017,"validation_errors":[],"timestamp":"2026-07-16T06:21:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":12.182877,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c112_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c112_i1024_o2048_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":316.64033691798977,"request_tps":0.2526525861445129,"input_tps":258.7162482119812,"output_tps":1034.864992847925,"total_tps":1293.581241059906,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":575.7432910031639,"ttft_p95_ms":887.6251749068615,"ttft_p99_ms":907.8684282244649,"tpot_p50_ms":15.333096270205825,"tpot_p95_ms":15.443372411818606,"tpot_p99_ms":15.483845902724966,"e2e_p50_ms":63438.04139450367,"e2e_p95_ms":63812.25281149527,"e2e_p99_ms":63831.44774455199,"itl_p50_ms":15.270748001057655,"itl_p95_ms":15.853451890870927,"itl_p99_ms":16.49769142619334,"validation_errors":[],"timestamp":"2026-07-16T06:27:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i1024_o4096_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":160,"failed":0,"duration_s":390.9258493729867,"request_tps":0.40928477934275004,"input_tps":419.10761404697604,"output_tps":1676.4304561879042,"total_tps":2095.5380702348803,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":1059.6694034975371,"ttft_p95_ms":1531.6532846380142,"ttft_p99_ms":1558.6441262916196,"tpot_p50_ms":18.821365679975997,"tpot_p95_ms":19.016937038081775,"tpot_p99_ms":19.073814857857453,"e2e_p50_ms":77972.91494200181,"e2e_p95_ms":78736.66218614817,"e2e_p99_ms":78754.35213367702,"itl_p50_ms":18.73732249805471,"itl_p95_ms":19.672631401772378,"itl_p99_ms":20.632040495693218,"validation_errors":[],"timestamp":"2026-07-16T06:34:34+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":61.995088,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i1024_o4096_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":240,"failed":0,"duration_s":395.37849693700264,"request_tps":0.6070132843826361,"input_tps":621.5816032078194,"output_tps":2486.3264128312776,"total_tps":3107.9080160390968,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":1144.204484502552,"ttft_p95_ms":1999.596584997198,"ttft_p99_ms":2035.2585943651502,"tpot_p50_ms":18.99256894627527,"tpot_p95_ms":19.224950862944212,"tpot_p99_ms":19.230104005436754,"e2e_p50_ms":78907.89274750568,"e2e_p95_ms":79839.60575450546,"e2e_p99_ms":79862.91444651186,"itl_p50_ms":18.858423005440272,"itl_p95_ms":20.01599159266334,"itl_p99_ms":21.22574228560552,"validation_errors":[],"timestamp":"2026-07-16T06:41:52+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":48.31074,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i1024_o4096_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":320,"failed":0,"duration_s":443.57241661399894,"request_tps":0.7214154623110101,"input_tps":738.7294334064743,"output_tps":2954.917733625897,"total_tps":3693.6471670323713,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":1471.407580494997,"ttft_p95_ms":2597.0414660041592,"ttft_p99_ms":2976.737847158656,"tpot_p50_ms":21.723876868741236,"tpot_p95_ms":22.00444482439625,"tpot_p99_ms":22.07639913107326,"e2e_p50_ms":90742.18811100582,"e2e_p95_ms":91541.6523386979,"e2e_p99_ms":91559.50254334108,"itl_p50_ms":21.4493349994882,"itl_p95_ms":22.813913105346728,"itl_p99_ms":24.170049769309127,"validation_errors":[],"timestamp":"2026-07-16T06:50:08+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":18.846734,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c64_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c64_i1024_o4096_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":400,"failed":0,"duration_s":463.4503973050014,"request_tps":0.8630912872791345,"input_tps":883.8054781738338,"output_tps":3535.221912695335,"total_tps":4419.027390869169,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":2439.6097005082993,"ttft_p95_ms":3686.9959791976726,"ttft_p99_ms":3709.631283667256,"tpot_p50_ms":22.095354364345052,"tpot_p95_ms":22.518257637104984,"tpot_p99_ms":22.625587380977006,"e2e_p50_ms":92209.49949500209,"e2e_p95_ms":93844.80823813865,"e2e_p99_ms":93874.67774137796,"itl_p50_ms":21.87070349464193,"itl_p95_ms":23.618548751983326,"itl_p99_ms":25.773182249395177,"validation_errors":[],"timestamp":"2026-07-16T06:58:45+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":19.63859,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c80_i1024_o4096_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":480,"failed":0,"duration_s":524.4931549339963,"request_tps":0.9151692362132059,"input_tps":937.1332978823228,"output_tps":3748.5331915292913,"total_tps":4685.666489411614,"mean_input_tokens":1024,"mean_output_tokens":4096,"ttft_p50_ms":2751.5591720002703,"ttft_p95_ms":4227.864060657878,"ttft_p99_ms":4536.041375702771,"tpot_p50_ms":24.893937553724363,"tpot_p95_ms":25.43370572488336,"tpot_p99_ms":25.74416057706036,"e2e_p50_ms":104575.3224170112,"e2e_p95_ms":106446.31684675842,"e2e_p99_ms":106478.32935052022,"itl_p50_ms":24.56568699562922,"itl_p95_ms":27.559315803227946,"itl_p99_ms":30.286136486683972,"validation_errors":[],"timestamp":"2026-07-16T07:08:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":6.033887,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c96_i1024_o4096_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":44.308171736003715,"request_tps":1.8055360188782963,"input_tps":7395.475533325502,"output_tps":231.10861041642193,"total_tps":7626.584143741924,"mean_input_tokens":4096,"mean_output_tokens":128,"ttft_p50_ms":1465.1815819961485,"ttft_p95_ms":23821.693100642005,"ttft_p99_ms":24259.730825575534,"tpot_p50_ms":22.75063312988532,"tpot_p95_ms":194.92663006346027,"tpot_p99_ms":197.04311093216822,"e2e_p50_ms":4635.849632992176,"e2e_p95_ms":26182.2721034383,"e2e_p99_ms":26191.62205229819,"itl_p50_ms":15.226024501316715,"itl_p95_ms":16.216387940949062,"itl_p99_ms":414.4044824814774,"validation_errors":[],"timestamp":"2026-07-16T07:09:51+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o128_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":32.527519440001925,"request_tps":2.4594559123256423,"input_tps":10073.931416885831,"output_tps":629.6207135553644,"total_tps":10703.552130441196,"mean_input_tokens":4096,"mean_output_tokens":256,"ttft_p50_ms":1533.099703505286,"ttft_p95_ms":2719.3758142995644,"ttft_p99_ms":2959.5566013005737,"tpot_p50_ms":19.392000529407206,"tpot_p95_ms":23.99643604938105,"tpot_p99_ms":24.379314330662144,"e2e_p50_ms":6627.143721503671,"e2e_p95_ms":6811.619238646381,"e2e_p99_ms":6835.494205219293,"itl_p50_ms":15.377331001218408,"itl_p95_ms":16.272442803892773,"itl_p99_ms":18.94523348833901,"validation_errors":[],"timestamp":"2026-07-16T07:11:08+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":160,"failed":0,"duration_s":66.26290708300075,"request_tps":2.414623913188481,"input_tps":9890.299548420018,"output_tps":618.1437217762511,"total_tps":10508.443270196269,"mean_input_tokens":4096,"mean_output_tokens":256,"ttft_p50_ms":3032.6789590035332,"ttft_p95_ms":5014.513322354456,"ttft_p99_ms":14411.20142712994,"tpot_p50_ms":25.094734256865696,"tpot_p95_ms":108.72940146214187,"tpot_p99_ms":110.79764089783636,"e2e_p50_ms":9434.836903004907,"e2e_p95_ms":29872.09643335227,"e2e_p99_ms":29884.739060596476,"itl_p50_ms":18.683340997085907,"itl_p95_ms":19.744293195253704,"itl_p99_ms":26.305865632602966,"validation_errors":[],"timestamp":"2026-07-16T07:12:56+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":256,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":-1.822842,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":48.17216287400515,"request_tps":1.6607101534809827,"input_tps":6802.268788658105,"output_tps":850.2835985822632,"total_tps":7652.552387240369,"mean_input_tokens":4096,"mean_output_tokens":512,"ttft_p50_ms":1226.4635695100878,"ttft_p95_ms":1866.224592289654,"ttft_p99_ms":2426.2945817218856,"tpot_p50_ms":16.140000863013167,"tpot_p95_ms":17.640392132493695,"tpot_p99_ms":18.638964293663246,"e2e_p50_ms":9504.232979503286,"e2e_p95_ms":10157.524074796675,"e2e_p99_ms":10178.896629959345,"itl_p50_ms":15.288406488252804,"itl_p95_ms":15.857895543740598,"itl_p99_ms":17.156079869600944,"validation_errors":[],"timestamp":"2026-07-16T07:14:23+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":160,"failed":0,"duration_s":63.387327321004705,"request_tps":2.52416384728972,"input_tps":10338.975118498693,"output_tps":1292.3718898123366,"total_tps":11631.347008311028,"mean_input_tokens":4096,"mean_output_tokens":512,"ttft_p50_ms":2139.6533224979066,"ttft_p95_ms":3791.57651143978,"ttft_p99_ms":4309.74766367377,"tpot_p50_ms":20.01632839431788,"tpot_p95_ms":24.519710790119547,"tpot_p99_ms":25.129288290261652,"e2e_p50_ms":12805.874345496704,"e2e_p95_ms":13800.694600059069,"e2e_p99_ms":13819.40531841843,"itl_p50_ms":18.705550508457236,"itl_p95_ms":20.301984495017674,"itl_p99_ms":21.65679321915376,"validation_errors":[],"timestamp":"2026-07-16T07:16:15+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":51.99304,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":240,"failed":0,"duration_s":77.56397197001206,"request_tps":3.0942200857479203,"input_tps":12673.925471223482,"output_tps":1584.2406839029352,"total_tps":14258.166155126417,"mean_input_tokens":4096,"mean_output_tokens":512,"ttft_p50_ms":2946.3751199946273,"ttft_p95_ms":6820.920404194476,"ttft_p99_ms":8050.301623321715,"tpot_p50_ms":23.094741272006033,"tpot_p95_ms":31.340273439141725,"tpot_p99_ms":33.989053702566764,"e2e_p50_ms":15109.904168501089,"e2e_p95_ms":19630.664032053755,"e2e_p99_ms":19649.577008009364,"itl_p50_ms":19.291372009320185,"itl_p95_ms":22.182088205590844,"itl_p99_ms":24.491042752633813,"validation_errors":[],"timestamp":"2026-07-16T07:18:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":22.583963,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i4096_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i4096_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":84.46414080199611,"request_tps":0.9471475023648069,"input_tps":3879.516169686249,"output_tps":969.8790424215623,"total_tps":4849.395212107812,"mean_input_tokens":4096,"mean_output_tokens":1024,"ttft_p50_ms":755.6758849968901,"ttft_p95_ms":1184.9343558955297,"ttft_p99_ms":1330.5567739978123,"tpot_p50_ms":15.835003849462446,"tpot_p95_ms":16.285198607422213,"tpot_p99_ms":16.55195564271857,"e2e_p50_ms":16962.706995996996,"e2e_p95_ms":17168.894911993266,"e2e_p99_ms":17178.149301250814,"itl_p50_ms":15.242625500832219,"itl_p95_ms":16.113438254251378,"itl_p99_ms":16.882939507922856,"validation_errors":[],"timestamp":"2026-07-16T07:20:28+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o1024_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":160,"failed":0,"duration_s":107.34333604600397,"request_tps":1.490544321553683,"input_tps":6105.269541083886,"output_tps":1526.3173852709715,"total_tps":7631.5869263548575,"mean_input_tokens":4096,"mean_output_tokens":1024,"ttft_p50_ms":1697.2974899981637,"ttft_p95_ms":2827.3090714494147,"ttft_p99_ms":3041.8824867265357,"tpot_p50_ms":19.041262989738744,"tpot_p95_ms":20.483581416088622,"tpot_p99_ms":20.796884204547236,"e2e_p50_ms":21322.126428000047,"e2e_p95_ms":21977.338839244476,"e2e_p99_ms":21997.641324478464,"itl_p50_ms":18.680188004509546,"itl_p95_ms":19.772065294091586,"itl_p99_ms":21.03478136268678,"validation_errors":[],"timestamp":"2026-07-16T07:23:01+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":57.371932,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o1024_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":240,"failed":0,"duration_s":114.72205163900799,"request_tps":2.092012795893852,"input_tps":8568.884411981218,"output_tps":2142.2211029953046,"total_tps":10711.105514976523,"mean_input_tokens":4096,"mean_output_tokens":1024,"ttft_p50_ms":2002.436676499201,"ttft_p95_ms":5319.875290898926,"ttft_p99_ms":6160.708812107041,"tpot_p50_ms":19.824865686698722,"tpot_p95_ms":23.0216869557243,"tpot_p99_ms":23.20206612341515,"e2e_p50_ms":21923.11628299649,"e2e_p95_ms":25281.340099497902,"e2e_p99_ms":25297.796992440562,"itl_p50_ms":18.851815999369137,"itl_p95_ms":20.00413840287365,"itl_p99_ms":21.528728755656623,"validation_errors":[],"timestamp":"2026-07-16T07:25:36+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":40.35227,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i4096_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i4096_o1024_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":161.3202662160038,"request_tps":0.4959079344245688,"input_tps":2031.2388994030339,"output_tps":1015.6194497015169,"total_tps":3046.858349104551,"mean_input_tokens":4096,"mean_output_tokens":2048,"ttft_p50_ms":766.6918885006453,"ttft_p95_ms":1151.3519862077374,"ttft_p99_ms":1174.3669241876341,"tpot_p50_ms":15.32661797704204,"tpot_p95_ms":15.672662826941613,"tpot_p99_ms":15.77215861052261,"e2e_p50_ms":32366.54036600521,"e2e_p95_ms":32729.988757940737,"e2e_p99_ms":32742.016700837994,"itl_p50_ms":15.24984300340293,"itl_p95_ms":15.832478402444394,"itl_p99_ms":16.448479092650818,"validation_errors":[],"timestamp":"2026-07-16T07:28:59+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o2048_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":160,"failed":0,"duration_s":203.61964830600482,"request_tps":0.7857787857463927,"input_tps":3218.5499064172245,"output_tps":1609.2749532086123,"total_tps":4827.824859625837,"mean_input_tokens":4096,"mean_output_tokens":2048,"ttft_p50_ms":1823.4369839992723,"ttft_p95_ms":2880.811745747633,"ttft_p99_ms":2898.1182565601193,"tpot_p50_ms":18.92700290888939,"tpot_p95_ms":19.648448953663472,"tpot_p99_ms":19.892009308511824,"e2e_p50_ms":40591.624035492714,"e2e_p95_ms":41278.74980429624,"e2e_p99_ms":41301.732580115786,"itl_p50_ms":18.753395997919142,"itl_p95_ms":19.575604447891237,"itl_p99_ms":20.460386879567523,"validation_errors":[],"timestamp":"2026-07-16T07:33:01+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":58.452554,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o2048_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":240,"failed":0,"duration_s":209.64498886499496,"request_tps":1.1447924479346976,"input_tps":4689.069866740521,"output_tps":2344.5349333702607,"total_tps":7033.604800110782,"mean_input_tokens":4096,"mean_output_tokens":2048,"ttft_p50_ms":1850.1262010031496,"ttft_p95_ms":5557.007933600835,"ttft_p99_ms":5728.514949967212,"tpot_p50_ms":19.032042742548224,"tpot_p95_ms":20.8632619918908,"tpot_p99_ms":21.040059630580483,"e2e_p50_ms":40867.48082550184,"e2e_p95_ms":44286.82900869899,"e2e_p99_ms":44307.41918329746,"itl_p50_ms":18.80523749423446,"itl_p95_ms":20.01578960043844,"itl_p99_ms":21.193549290037485,"validation_errors":[],"timestamp":"2026-07-16T07:37:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":45.688897,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i4096_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i4096_o2048_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":316.8896808080026,"request_tps":0.252453787059322,"input_tps":1034.0507117949828,"output_tps":1034.0507117949828,"total_tps":2068.1014235899656,"mean_input_tokens":4096,"mean_output_tokens":4096,"ttft_p50_ms":658.9246715011541,"ttft_p95_ms":1005.5294943493209,"ttft_p99_ms":1026.091546237294,"tpot_p50_ms":15.320106219048359,"tpot_p95_ms":15.378604879511569,"tpot_p99_ms":15.45278130631852,"e2e_p50_ms":63453.26198550174,"e2e_p95_ms":63513.736396850436,"e2e_p99_ms":63521.74632662762,"itl_p50_ms":15.256563012371771,"itl_p95_ms":15.777264998177998,"itl_p99_ms":16.359419524669644,"validation_errors":[],"timestamp":"2026-07-16T07:43:10+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i4096_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i4096_o4096_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":160,"failed":0,"duration_s":389.95699894400605,"request_tps":0.4103016497543987,"input_tps":1680.595557394017,"output_tps":1680.595557394017,"total_tps":3361.191114788034,"mean_input_tokens":4096,"mean_output_tokens":4096,"ttft_p50_ms":1123.4332419990096,"ttft_p95_ms":1676.0268465513946,"ttft_p99_ms":1717.2243972464637,"tpot_p50_ms":18.85827924127029,"tpot_p95_ms":19.032755866286788,"tpot_p99_ms":19.08965374972383,"e2e_p50_ms":77915.13783499977,"e2e_p95_ms":78629.53818804745,"e2e_p99_ms":78646.88334376202,"itl_p50_ms":18.702409994148184,"itl_p95_ms":19.55858354413067,"itl_p99_ms":20.443303814536193,"validation_errors":[],"timestamp":"2026-07-16T07:50:22+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":62.525449,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i4096_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i4096_o4096_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":240,"failed":0,"duration_s":402.4469851330068,"request_tps":0.596351839785012,"input_tps":2442.6571357594094,"output_tps":2442.6571357594094,"total_tps":4885.314271518819,"mean_input_tokens":4096,"mean_output_tokens":4096,"ttft_p50_ms":2047.3789034949732,"ttft_p95_ms":4081.0472775985545,"ttft_p99_ms":4647.203718657256,"tpot_p50_ms":19.02283846043863,"tpot_p95_ms":19.43320571131982,"tpot_p99_ms":19.899421790027564,"e2e_p50_ms":80315.7795865045,"e2e_p95_ms":81844.74676285972,"e2e_p99_ms":81865.64722886324,"itl_p50_ms":18.825628008926287,"itl_p95_ms":19.975638002506457,"itl_p99_ms":21.221037310315303,"validation_errors":[],"timestamp":"2026-07-16T07:57:45+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":45.344734,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i4096_o4096_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i4096_o4096_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":56.001648912992096,"request_tps":1.4285293657030234,"input_tps":23405.025127678335,"output_tps":182.851758809987,"total_tps":23587.87688648832,"mean_input_tokens":16384,"mean_output_tokens":128,"ttft_p50_ms":5949.235703003069,"ttft_p95_ms":9227.944630107957,"ttft_p99_ms":11097.429403630083,"tpot_p50_ms":39.72050276768026,"tpot_p95_ms":72.95558840476615,"tpot_p99_ms":80.88457267013912,"e2e_p50_ms":11065.538848997676,"e2e_p95_ms":13142.54083404594,"e2e_p99_ms":14229.989901815272,"itl_p50_ms":15.273823504685424,"itl_p95_ms":17.06440234847832,"itl_p99_ms":1069.5194568477748,"validation_errors":[],"timestamp":"2026-07-16T07:59:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o128_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":64.32757946698985,"request_tps":1.2436345446676191,"input_tps":20375.708379834272,"output_tps":318.3704434349105,"total_tps":20694.07882326918,"mean_input_tokens":16384,"mean_output_tokens":256,"ttft_p50_ms":4847.543320000113,"ttft_p95_ms":8945.170914454502,"ttft_p99_ms":10200.10803171811,"tpot_p50_ms":30.697892317645408,"tpot_p95_ms":45.12670790233909,"tpot_p99_ms":46.21028031682417,"e2e_p50_ms":12845.963567502622,"e2e_p95_ms":15142.341455946007,"e2e_p99_ms":15201.770440481196,"itl_p50_ms":15.241901011904702,"itl_p95_ms":16.105174794211056,"itl_p99_ms":806.8968500866322,"validation_errors":[],"timestamp":"2026-07-16T08:01:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":72.41434827599733,"request_tps":1.1047534349835064,"input_tps":18100.28027876977,"output_tps":565.6337587115553,"total_tps":18665.914037481325,"mean_input_tokens":16384,"mean_output_tokens":512,"ttft_p50_ms":3977.608113505994,"ttft_p95_ms":6827.726796599017,"ttft_p99_ms":7064.793197970429,"tpot_p50_ms":20.6349563414749,"tpot_p95_ms":26.606408149222858,"tpot_p99_ms":27.689232515706706,"e2e_p50_ms":14308.850026493019,"e2e_p95_ms":14974.010983452172,"e2e_p99_ms":14998.68528622057,"itl_p50_ms":15.342150996730197,"itl_p95_ms":16.593942245526705,"itl_p99_ms":19.848780611646326,"validation_errors":[],"timestamp":"2026-07-16T08:03:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":111.50969841099868,"request_tps":0.7174263865833329,"input_tps":11754.313917781326,"output_tps":734.6446198613329,"total_tps":12488.958537642658,"mean_input_tokens":16384,"mean_output_tokens":1024,"ttft_p50_ms":4311.803800497728,"ttft_p95_ms":7437.313835108215,"ttft_p99_ms":9130.135508555395,"tpot_p50_ms":17.93213432404895,"tpot_p95_ms":22.04782108866086,"tpot_p99_ms":22.43329354472889,"e2e_p50_ms":21698.23511350114,"e2e_p95_ms":24644.28777230205,"e2e_p99_ms":26412.721827901405,"itl_p50_ms":15.224864997435361,"itl_p95_ms":15.731755192973651,"itl_p99_ms":16.860831878148,"validation_errors":[],"timestamp":"2026-07-16T08:05:39+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o1024_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":188.87208026400185,"request_tps":0.4235671036617879,"input_tps":6939.723426394733,"output_tps":867.4654282993416,"total_tps":7807.1888546940745,"mean_input_tokens":16384,"mean_output_tokens":2048,"ttft_p50_ms":2707.821463998698,"ttft_p95_ms":6159.3108133580245,"ttft_p99_ms":7476.327504011569,"tpot_p50_ms":16.789645155595252,"tpot_p95_ms":19.235649654987466,"tpot_p99_ms":19.809279338319755,"e2e_p50_ms":37331.52540450101,"e2e_p95_ms":41633.905589940696,"e2e_p99_ms":41644.779238444025,"itl_p50_ms":15.280855994205922,"itl_p95_ms":18.66166050749598,"itl_p99_ms":19.038321707921565,"validation_errors":[],"timestamp":"2026-07-16T08:09:28+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i16384_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i16384_o2048_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":263.11688939499436,"request_tps":0.30404737675316235,"input_tps":19926.048882895248,"output_tps":38.91806422440478,"total_tps":19964.966947119654,"mean_input_tokens":65536,"mean_output_tokens":128,"ttft_p50_ms":21783.06187000271,"ttft_p95_ms":48708.6885400975,"ttft_p99_ms":53530.08918488397,"tpot_p50_ms":208.85715699601343,"tpot_p95_ms":353.9625207657282,"tpot_p99_ms":368.66045906663794,"e2e_p50_ms":53675.07621049299,"e2e_p95_ms":64500.535845247214,"e2e_p99_ms":78019.4687606334,"itl_p50_ms":15.624218496668618,"itl_p95_ms":1377.797994745198,"itl_p99_ms":1593.279839480674,"validation_errors":[],"timestamp":"2026-07-16T08:14:46+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i65536_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i65536_o128_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":318.6807930810028,"request_tps":0.2510348967898592,"input_tps":16451.82299602021,"output_tps":64.26493357820395,"total_tps":16516.087929598416,"mean_input_tokens":65536,"mean_output_tokens":256,"ttft_p50_ms":25208.387847495032,"ttft_p95_ms":57570.60906205442,"ttft_p99_ms":64559.6027293129,"tpot_p50_ms":139.20286317058046,"tpot_p95_ms":218.68997558943673,"tpot_p99_ms":237.28532163909728,"e2e_p50_ms":62259.9117714999,"e2e_p95_ms":93757.1761358522,"e2e_p99_ms":109198.36791741903,"itl_p50_ms":15.581318992190063,"itl_p95_ms":1060.9013718029018,"itl_p99_ms":1579.705025086296,"validation_errors":[],"timestamp":"2026-07-16T08:21:04+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i65536_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i65536_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":337.8231221129972,"request_tps":0.23681031511289238,"input_tps":15519.600811238515,"output_tps":121.2468813378009,"total_tps":15640.847692576317,"mean_input_tokens":65536,"mean_output_tokens":512,"ttft_p50_ms":24695.115143498697,"ttft_p95_ms":57546.68756374739,"ttft_p99_ms":64468.70614748111,"tpot_p50_ms":76.04405581409218,"tpot_p95_ms":120.77805700313024,"tpot_p99_ms":135.3599259530104,"e2e_p50_ms":66170.59861750022,"e2e_p95_ms":92553.88260725957,"e2e_p99_ms":109859.61571042174,"itl_p50_ms":15.599313002894633,"itl_p95_ms":824.2479222499242,"itl_p99_ms":1410.173195308016,"validation_errors":[],"timestamp":"2026-07-16T08:27:37+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i65536_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i65536_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":376.9979162859963,"request_tps":0.21220276437631763,"input_tps":13906.920366166352,"output_tps":217.29563072134926,"total_tps":14124.215996887702,"mean_input_tokens":65536,"mean_output_tokens":1024,"ttft_p50_ms":25524.604717495095,"ttft_p95_ms":56727.187289898575,"ttft_p99_ms":65385.280855649864,"tpot_p50_ms":45.977114398337235,"tpot_p95_ms":68.72763672496647,"tpot_p99_ms":71.65886099602294,"e2e_p50_ms":72648.60455600865,"e2e_p95_ms":102850.95250014929,"e2e_p99_ms":128267.6254233617,"itl_p50_ms":15.504026996495668,"itl_p95_ms":16.275252601190004,"itl_p99_ms":1222.4654011012171,"validation_errors":[],"timestamp":"2026-07-16T08:34:46+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i65536_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i65536_o1024_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":653.2824726870022,"request_tps":0.12245851273332609,"input_tps":16050.882180982517,"output_tps":15.674689629865739,"total_tps":16066.556870612381,"mean_input_tokens":131072,"mean_output_tokens":128,"ttft_p50_ms":55664.04897299799,"ttft_p95_ms":118594.41714475105,"ttft_p99_ms":139872.23895933607,"tpot_p50_ms":544.0846560747494,"tpot_p95_ms":902.7456552543442,"tpot_p99_ms":958.5754168834561,"e2e_p50_ms":129924.55754899856,"e2e_p95_ms":185606.83365028963,"e2e_p99_ms":243322.26077544419,"itl_p50_ms":16.123155495733954,"itl_p95_ms":1741.9904875576322,"itl_p99_ms":1853.2892793747305,"validation_errors":[],"timestamp":"2026-07-16T08:46:49+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i131072_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i131072_o128_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":702.479465484008,"request_tps":0.11388233241078448,"input_tps":14926.785073746343,"output_tps":29.153877097160827,"total_tps":14955.938950843505,"mean_input_tokens":131072,"mean_output_tokens":256,"ttft_p50_ms":57824.69000250421,"ttft_p95_ms":127985.28046924763,"ttft_p99_ms":151725.3745295175,"tpot_p50_ms":253.09320623725628,"tpot_p95_ms":525.102048641203,"tpot_p99_ms":619.6555398026641,"e2e_p50_ms":138529.73486200062,"e2e_p95_ms":184438.2270416012,"e2e_p99_ms":185996.91346610835,"itl_p50_ms":15.797007494256832,"itl_p95_ms":1590.6224329526603,"itl_p99_ms":1839.2413294677688,"validation_errors":[],"timestamp":"2026-07-16T09:15:16+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i131072_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i131072_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":40,"failed":0,"duration_s":341.0657499819936,"request_tps":0.11727943952775025,"input_tps":15372.05069778128,"output_tps":30.023536519104063,"total_tps":15402.074234300384,"mean_input_tokens":131072,"mean_output_tokens":256,"ttft_p50_ms":38767.44465551019,"ttft_p95_ms":67753.44985890988,"ttft_p99_ms":68613.70976785096,"tpot_p50_ms":99.49740432157162,"tpot_p95_ms":179.5039385709941,"tpot_p99_ms":227.61194875988735,"e2e_p50_ms":70759.12754750607,"e2e_p95_ms":71843.55354870277,"e2e_p99_ms":71887.88047623617,"itl_p50_ms":12.980383995454758,"itl_p95_ms":1125.97120024584,"itl_p99_ms":1602.8499752024072,"validation_errors":[],"timestamp":"2026-07-16T09:22:39+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c8_i131072_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c8_i131072_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":5,"failed":0,"duration_s":97.25075286399806,"request_tps":0.051413483728936615,"input_tps":6738.86813931918,"output_tps":13.161851834607774,"total_tps":6752.029991153788,"mean_input_tokens":131072,"mean_output_tokens":256,"ttft_p50_ms":16969.752281991532,"ttft_p95_ms":17121.84501980082,"ttft_p99_ms":17125.887859960785,"tpot_p50_ms":9.60843350975683,"tpot_p95_ms":9.634889066653034,"tpot_p99_ms":9.637108200772976,"e2e_p50_ms":19427.356342988787,"e2e_p95_ms":19571.998772607185,"e2e_p99_ms":19575.258900129702,"itl_p50_ms":10.00273649697192,"itl_p95_ms":10.19823560709483,"itl_p99_ms":11.044782063545426,"validation_errors":[],"timestamp":"2026-07-16T09:25:11+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":256,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c1_i131072_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c1_i131072_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":667.9211692050012,"request_tps":0.1197746136646944,"input_tps":15699.098162258824,"output_tps":61.32460219632353,"total_tps":15760.422764455148,"mean_input_tokens":131072,"mean_output_tokens":512,"ttft_p50_ms":55981.32588399312,"ttft_p95_ms":121882.91273160065,"ttft_p99_ms":140760.88047753967,"tpot_p50_ms":141.8939732465815,"tpot_p95_ms":235.749314115342,"tpot_p99_ms":254.5075704539367,"e2e_p50_ms":132681.28659549984,"e2e_p95_ms":175247.86795944526,"e2e_p99_ms":234920.12947427476,"itl_p50_ms":15.690204003476538,"itl_p95_ms":1249.2242270018323,"itl_p99_ms":1797.7115144065467,"validation_errors":[],"timestamp":"2026-07-16T09:37:52+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i131072_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i131072_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":40,"failed":0,"duration_s":368.12840555600997,"request_tps":0.10865773843120097,"input_tps":14241.987091654373,"output_tps":55.632762076774895,"total_tps":14297.619853731148,"mean_input_tokens":131072,"mean_output_tokens":512,"ttft_p50_ms":36897.75435599586,"ttft_p95_ms":68026.47409820274,"ttft_p99_ms":74667.961793141,"tpot_p50_ms":63.40141743053057,"tpot_p95_ms":116.81624726055898,"tpot_p99_ms":143.86918745584376,"e2e_p50_ms":74333.1425399956,"e2e_p95_ms":83933.02225654986,"e2e_p99_ms":104491.18695786949,"itl_p50_ms":12.98504300211789,"itl_p95_ms":18.927634652209232,"itl_p99_ms":1447.1019217558205,"validation_errors":[],"timestamp":"2026-07-16T09:45:12+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":512,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c8_i131072_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c8_i131072_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":5,"failed":0,"duration_s":110.12223424699914,"request_tps":0.04540409150058859,"input_tps":5951.205081165148,"output_tps":23.24689484830136,"total_tps":5974.45197601345,"mean_input_tokens":131072,"mean_output_tokens":512,"ttft_p50_ms":16962.474314001156,"ttft_p95_ms":17155.541892390465,"ttft_p99_ms":17171.51858087047,"tpot_p50_ms":9.821654945213728,"tpot_p95_ms":9.835613834436566,"tpot_p99_ms":9.837956189974383,"e2e_p50_ms":21989.969162998023,"e2e_p95_ms":22173.37846419541,"e2e_p99_ms":22190.17843683483,"itl_p50_ms":9.989651007344946,"itl_p95_ms":10.151538599166088,"itl_p99_ms":10.469720991677605,"validation_errors":[],"timestamp":"2026-07-16T09:47:57+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":512,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c1_i131072_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c1_i131072_o512_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1511.6757434350002,"request_tps":0.05292140219053523,"input_tps":13873.028055835668,"output_tps":6.77393948038851,"total_tps":13879.801995316056,"mean_input_tokens":262144,"mean_output_tokens":128,"ttft_p50_ms":70474.41060249548,"ttft_p95_ms":206680.64425829216,"ttft_p99_ms":263259.9471526309,"tpot_p50_ms":1770.9045697204808,"tpot_p95_ms":1859.1631874913205,"tpot_p99_ms":1883.4251564144938,"e2e_p50_ms":293508.3537885075,"e2e_p95_ms":418159.4987228506,"e2e_p99_ms":491266.18717678235,"itl_p50_ms":1741.0211320020608,"itl_p95_ms":2297.1252548493794,"itl_p99_ms":2395.285054906126,"validation_errors":[],"timestamp":"2026-07-16T10:15:00+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i262144_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i262144_o128_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":40,"failed":0,"duration_s":1143.3382095730049,"request_tps":0.03498527353068918,"input_tps":9171.179544428984,"output_tps":4.478115011928215,"total_tps":9175.65765944091,"mean_input_tokens":262144,"mean_output_tokens":128,"ttft_p50_ms":60434.31567850348,"ttft_p95_ms":171402.15240599928,"ttft_p99_ms":221100.73346630885,"tpot_p50_ms":1432.7522763937384,"tpot_p95_ms":1566.4298780566398,"tpot_p99_ms":1577.578224665822,"e2e_p50_ms":243043.1625314959,"e2e_p95_ms":344139.9401556495,"e2e_p99_ms":419677.4824362584,"itl_p50_ms":1394.7447990067303,"itl_p95_ms":1954.1919693932869,"itl_p99_ms":2118.3612401573937,"validation_errors":[],"timestamp":"2026-07-16T10:35:53+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":128,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c8_i262144_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c8_i262144_o128_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":5,"failed":0,"duration_s":215.86107532100868,"request_tps":0.02316304591999489,"input_tps":6072.05350965114,"output_tps":2.964869877759346,"total_tps":6075.0183795289,"mean_input_tokens":262144,"mean_output_tokens":128,"ttft_p50_ms":42189.58094799018,"ttft_p95_ms":42206.01177979843,"ttft_p99_ms":42207.67736715847,"tpot_p50_ms":9.026367795253979,"tpot_p95_ms":9.051597466134083,"tpot_p99_ms":9.053308627064942,"e2e_p50_ms":43335.3122970002,"e2e_p95_ms":43353.944078995846,"e2e_p99_ms":43357.12344699539,"itl_p50_ms":10.197708994382992,"itl_p95_ms":10.398621491185622,"itl_p99_ms":11.979914264811663,"validation_errors":[],"timestamp":"2026-07-16T10:40:50+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":128,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c1_i262144_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c1_i262144_o128_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":80,"failed":0,"duration_s":1557.1239152909984,"request_tps":0.0513767717613209,"input_tps":13468.112456599705,"output_tps":13.15245357089815,"total_tps":13481.264910170603,"mean_input_tokens":262144,"mean_output_tokens":256,"ttft_p50_ms":128086.70317550423,"ttft_p95_ms":293147.14115650055,"ttft_p99_ms":327246.22319525003,"tpot_p50_ms":694.1213535215974,"tpot_p95_ms":1110.7064755825186,"tpot_p99_ms":1121.9122485736023,"e2e_p50_ms":320114.9868900029,"e2e_p95_ms":442507.49014469614,"e2e_p99_ms":540593.0392589162,"itl_p50_ms":16.51875649986323,"itl_p95_ms":2135.694057152432,"itl_p99_ms":2348.4220405294036,"validation_errors":[],"timestamp":"2026-07-16T11:08:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i262144_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i262144_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":40,"failed":0,"duration_s":818.7336148600007,"request_tps":0.04885593955591991,"input_tps":12807.291418947068,"output_tps":12.507120526315497,"total_tps":12819.798539473382,"mean_input_tokens":262144,"mean_output_tokens":256,"ttft_p50_ms":105032.74306450476,"ttft_p95_ms":162785.75602490891,"ttft_p99_ms":165083.89961342298,"tpot_p50_ms":214.14210347058685,"tpot_p95_ms":422.65357399257476,"tpot_p99_ms":440.83721461195177,"e2e_p50_ms":164185.14075000712,"e2e_p95_ms":168861.28058029353,"e2e_p99_ms":169246.668313704,"itl_p50_ms":13.376075992709957,"itl_p95_ms":1720.7928034986253,"itl_p99_ms":2045.3944211971263,"validation_errors":[],"timestamp":"2026-07-16T11:23:47+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":256,"concurrency":8,"num_prompts":40,"warmup_requests":8,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c8_i262144_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c8_i262144_o256_a1.log"}
|
||||||
|
{"status":"COMPLETED","completed":5,"failed":0,"duration_s":222.6894240649999,"request_tps":0.022452795057481358,"input_tps":5885.865507548393,"output_tps":5.7479155347152275,"total_tps":5891.6134230831085,"mean_input_tokens":262144,"mean_output_tokens":256,"ttft_p50_ms":41992.54209799983,"ttft_p95_ms":42385.26861999417,"ttft_p99_ms":42405.3075527912,"tpot_p50_ms":9.564196976419884,"tpot_p95_ms":9.778543265853717,"tpot_p99_ms":9.797764013144736,"e2e_p50_ms":44431.4123269869,"e2e_p95_ms":44819.5239008026,"e2e_p99_ms":44832.609473761986,"itl_p50_ms":10.17798750399379,"itl_p95_ms":10.34693365872954,"itl_p99_ms":10.991512061009418,"validation_errors":[],"timestamp":"2026-07-16T11:28:54+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":256,"concurrency":1,"num_prompts":5,"warmup_requests":1,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c1_i262144_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c1_i262144_o256_a1.log"}
|
||||||
|
{"timestamp":"2026-07-16T12:39:42+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":524288,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":2,"status":"FAILED","completed":0,"failed":80,"error_type":"benchmark rc=124","validation_errors":[],"gain_pct":null,"plateau_streak":0,"raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i524288_o128_a2.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i524288_o128_a2.log"}
|
||||||
|
|||||||
@ -3,3 +3,25 @@
|
|||||||
{"timestamp":"2026-07-16T05:37:07+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":1933.4323295182799,"last_total_tps":1933.4323295182799}
|
{"timestamp":"2026-07-16T05:37:07+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":1933.4323295182799,"last_total_tps":1933.4323295182799}
|
||||||
{"timestamp":"2026-07-16T05:50:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":5,"search_cap":512,"max_successful_concurrency":80,"saturation_concurrency":null,"stop_probe_concurrency":80,"best_tps_concurrency":80,"best_total_tps":5711.313811148789,"last_total_tps":5711.313811148789}
|
{"timestamp":"2026-07-16T05:50:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":5,"search_cap":512,"max_successful_concurrency":80,"saturation_concurrency":null,"stop_probe_concurrency":80,"best_tps_concurrency":80,"best_total_tps":5711.313811148789,"last_total_tps":5711.313811148789}
|
||||||
{"timestamp":"2026-07-16T06:21:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":7,"search_cap":512,"max_successful_concurrency":112,"saturation_concurrency":null,"stop_probe_concurrency":112,"best_tps_concurrency":112,"best_total_tps":6326.121211706281,"last_total_tps":6326.121211706281}
|
{"timestamp":"2026-07-16T06:21:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":7,"search_cap":512,"max_successful_concurrency":112,"saturation_concurrency":null,"stop_probe_concurrency":112,"best_tps_concurrency":112,"best_total_tps":6326.121211706281,"last_total_tps":6326.121211706281}
|
||||||
|
{"timestamp":"2026-07-16T07:08:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":4096,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":6,"search_cap":512,"max_successful_concurrency":96,"saturation_concurrency":null,"stop_probe_concurrency":96,"best_tps_concurrency":96,"best_total_tps":4685.666489411614,"last_total_tps":4685.666489411614}
|
||||||
|
{"timestamp":"2026-07-16T07:09:51+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":7626.584143741924,"last_total_tps":7626.584143741924}
|
||||||
|
{"timestamp":"2026-07-16T07:12:56+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":512,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":10703.552130441196,"last_total_tps":10508.443270196269}
|
||||||
|
{"timestamp":"2026-07-16T07:18:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":14258.166155126417,"last_total_tps":14258.166155126417}
|
||||||
|
{"timestamp":"2026-07-16T07:25:36+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":10711.105514976523,"last_total_tps":10711.105514976523}
|
||||||
|
{"timestamp":"2026-07-16T07:37:17+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":7033.604800110782,"last_total_tps":7033.604800110782}
|
||||||
|
{"timestamp":"2026-07-16T07:57:45+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":4096,"osl":4096,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":48,"saturation_concurrency":null,"stop_probe_concurrency":48,"best_tps_concurrency":48,"best_total_tps":4885.314271518819,"last_total_tps":4885.314271518819}
|
||||||
|
{"timestamp":"2026-07-16T07:59:21+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":23587.87688648832,"last_total_tps":23587.87688648832}
|
||||||
|
{"timestamp":"2026-07-16T08:01:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":20694.07882326918,"last_total_tps":20694.07882326918}
|
||||||
|
{"timestamp":"2026-07-16T08:03:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":18665.914037481325,"last_total_tps":18665.914037481325}
|
||||||
|
{"timestamp":"2026-07-16T08:05:39+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":12488.958537642658,"last_total_tps":12488.958537642658}
|
||||||
|
{"timestamp":"2026-07-16T08:09:28+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":16384,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":7807.1888546940745,"last_total_tps":7807.1888546940745}
|
||||||
|
{"timestamp":"2026-07-16T08:14:46+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":19964.966947119654,"last_total_tps":19964.966947119654}
|
||||||
|
{"timestamp":"2026-07-16T08:21:04+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":16516.087929598416,"last_total_tps":16516.087929598416}
|
||||||
|
{"timestamp":"2026-07-16T08:27:37+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":15640.847692576317,"last_total_tps":15640.847692576317}
|
||||||
|
{"timestamp":"2026-07-16T08:34:46+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":65536,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":14124.215996887702,"last_total_tps":14124.215996887702}
|
||||||
|
{"timestamp":"2026-07-16T08:46:49+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":16066.556870612381,"last_total_tps":16066.556870612381}
|
||||||
|
{"timestamp":"2026-07-16T09:25:11+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":8,"best_total_tps":15402.074234300384,"last_total_tps":6752.029991153788}
|
||||||
|
{"timestamp":"2026-07-16T09:47:57+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":131072,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":15760.422764455148,"last_total_tps":5974.45197601345}
|
||||||
|
{"timestamp":"2026-07-16T10:40:50+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":13879.801995316056,"last_total_tps":6075.0183795289}
|
||||||
|
{"timestamp":"2026-07-16T11:28:54+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":262144,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":3,"search_cap":512,"max_successful_concurrency":1,"saturation_concurrency":null,"stop_probe_concurrency":1,"best_tps_concurrency":16,"best_total_tps":13481.264910170603,"last_total_tps":5891.6134230831085}
|
||||||
|
{"timestamp":"2026-07-16T12:39:42+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":524288,"osl":128,"status":"FAILED","stop_reason":"BENCHMARK_FAILED","tested_points":1,"search_cap":512,"max_successful_concurrency":0,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":null,"best_total_tps":null,"last_total_tps":null}
|
||||||
|
|||||||
@ -7,16 +7,21 @@
|
|||||||
"matrix": "/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/matrix.json",
|
"matrix": "/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/matrix.json",
|
||||||
"dataset": "random",
|
"dataset": "random",
|
||||||
"tokenize_prompt": false,
|
"tokenize_prompt": false,
|
||||||
"random_range_ratio": 1,
|
"random_range_ratio": 1.0,
|
||||||
"search": {
|
"search": {
|
||||||
"start_concurrency": 16,
|
"start_concurrency": 16,
|
||||||
"max_concurrency": 512,
|
"max_concurrency": 512,
|
||||||
"multiplier": 2,
|
"multiplier": 2,
|
||||||
|
"initial_backoff_concurrencies": [
|
||||||
|
8,
|
||||||
|
1
|
||||||
|
],
|
||||||
"num_prompts_multiplier": 5,
|
"num_prompts_multiplier": 5,
|
||||||
"min_tps_gain_pct": 2,
|
"min_tps_gain_pct": 2.0,
|
||||||
"plateau_patience": 2,
|
"plateau_patience": 2,
|
||||||
"warmup_max_requests": 0,
|
"warmup_max_requests": 0,
|
||||||
"ttft_slo_ms": 4000,
|
"ttft_slo_ms": 4000.0,
|
||||||
"enable_ttft_slo_stop": 1
|
"enable_ttft_slo_stop": 1,
|
||||||
|
"ttft_group_skip_ms": 8000.0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 316.64033691798977,
|
||||||
|
"request_tps": 0.2526525861445129,
|
||||||
|
"input_tps": 258.7162482119812,
|
||||||
|
"output_tps": 1034.864992847925,
|
||||||
|
"total_tps": 1293.581241059906,
|
||||||
|
"mean_input_tokens": 1024.0,
|
||||||
|
"mean_output_tokens": 4096.0,
|
||||||
|
"ttft_p50_ms": 575.7432910031639,
|
||||||
|
"ttft_p95_ms": 887.6251749068615,
|
||||||
|
"ttft_p99_ms": 907.8684282244649,
|
||||||
|
"tpot_p50_ms": 15.333096270205825,
|
||||||
|
"tpot_p95_ms": 15.443372411818606,
|
||||||
|
"tpot_p99_ms": 15.483845902724966,
|
||||||
|
"e2e_p50_ms": 63438.04139450367,
|
||||||
|
"e2e_p95_ms": 63812.25281149527,
|
||||||
|
"e2e_p99_ms": 63831.44774455199,
|
||||||
|
"itl_p50_ms": 15.270748001057655,
|
||||||
|
"itl_p95_ms": 15.853451890870927,
|
||||||
|
"itl_p99_ms": 16.49769142619334,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 653.2824726870022,
|
||||||
|
"request_tps": 0.12245851273332609,
|
||||||
|
"input_tps": 16050.882180982517,
|
||||||
|
"output_tps": 15.674689629865739,
|
||||||
|
"total_tps": 16066.556870612381,
|
||||||
|
"mean_input_tokens": 131072.0,
|
||||||
|
"mean_output_tokens": 128.0,
|
||||||
|
"ttft_p50_ms": 55664.04897299799,
|
||||||
|
"ttft_p95_ms": 118594.41714475105,
|
||||||
|
"ttft_p99_ms": 139872.23895933607,
|
||||||
|
"tpot_p50_ms": 544.0846560747494,
|
||||||
|
"tpot_p95_ms": 902.7456552543442,
|
||||||
|
"tpot_p99_ms": 958.5754168834561,
|
||||||
|
"e2e_p50_ms": 129924.55754899856,
|
||||||
|
"e2e_p95_ms": 185606.83365028963,
|
||||||
|
"e2e_p99_ms": 243322.26077544419,
|
||||||
|
"itl_p50_ms": 16.123155495733954,
|
||||||
|
"itl_p95_ms": 1741.9904875576322,
|
||||||
|
"itl_p99_ms": 1853.2892793747305,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 702.479465484008,
|
||||||
|
"request_tps": 0.11388233241078448,
|
||||||
|
"input_tps": 14926.785073746343,
|
||||||
|
"output_tps": 29.153877097160827,
|
||||||
|
"total_tps": 14955.938950843505,
|
||||||
|
"mean_input_tokens": 131072.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 57824.69000250421,
|
||||||
|
"ttft_p95_ms": 127985.28046924763,
|
||||||
|
"ttft_p99_ms": 151725.3745295175,
|
||||||
|
"tpot_p50_ms": 253.09320623725628,
|
||||||
|
"tpot_p95_ms": 525.102048641203,
|
||||||
|
"tpot_p99_ms": 619.6555398026641,
|
||||||
|
"e2e_p50_ms": 138529.73486200062,
|
||||||
|
"e2e_p95_ms": 184438.2270416012,
|
||||||
|
"e2e_p99_ms": 185996.91346610835,
|
||||||
|
"itl_p50_ms": 15.797007494256832,
|
||||||
|
"itl_p95_ms": 1590.6224329526603,
|
||||||
|
"itl_p99_ms": 1839.2413294677688,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 667.9211692050012,
|
||||||
|
"request_tps": 0.1197746136646944,
|
||||||
|
"input_tps": 15699.098162258824,
|
||||||
|
"output_tps": 61.32460219632353,
|
||||||
|
"total_tps": 15760.422764455148,
|
||||||
|
"mean_input_tokens": 131072.0,
|
||||||
|
"mean_output_tokens": 512.0,
|
||||||
|
"ttft_p50_ms": 55981.32588399312,
|
||||||
|
"ttft_p95_ms": 121882.91273160065,
|
||||||
|
"ttft_p99_ms": 140760.88047753967,
|
||||||
|
"tpot_p50_ms": 141.8939732465815,
|
||||||
|
"tpot_p95_ms": 235.749314115342,
|
||||||
|
"tpot_p99_ms": 254.5075704539367,
|
||||||
|
"e2e_p50_ms": 132681.28659549984,
|
||||||
|
"e2e_p95_ms": 175247.86795944526,
|
||||||
|
"e2e_p99_ms": 234920.12947427476,
|
||||||
|
"itl_p50_ms": 15.690204003476538,
|
||||||
|
"itl_p95_ms": 1249.2242270018323,
|
||||||
|
"itl_p99_ms": 1797.7115144065467,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 111.50969841099868,
|
||||||
|
"request_tps": 0.7174263865833329,
|
||||||
|
"input_tps": 11754.313917781326,
|
||||||
|
"output_tps": 734.6446198613329,
|
||||||
|
"total_tps": 12488.958537642658,
|
||||||
|
"mean_input_tokens": 16384.0,
|
||||||
|
"mean_output_tokens": 1024.0,
|
||||||
|
"ttft_p50_ms": 4311.803800497728,
|
||||||
|
"ttft_p95_ms": 7437.313835108215,
|
||||||
|
"ttft_p99_ms": 9130.135508555395,
|
||||||
|
"tpot_p50_ms": 17.93213432404895,
|
||||||
|
"tpot_p95_ms": 22.04782108866086,
|
||||||
|
"tpot_p99_ms": 22.43329354472889,
|
||||||
|
"e2e_p50_ms": 21698.23511350114,
|
||||||
|
"e2e_p95_ms": 24644.28777230205,
|
||||||
|
"e2e_p99_ms": 26412.721827901405,
|
||||||
|
"itl_p50_ms": 15.224864997435361,
|
||||||
|
"itl_p95_ms": 15.731755192973651,
|
||||||
|
"itl_p99_ms": 16.860831878148,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 56.001648912992096,
|
||||||
|
"request_tps": 1.4285293657030234,
|
||||||
|
"input_tps": 23405.025127678335,
|
||||||
|
"output_tps": 182.851758809987,
|
||||||
|
"total_tps": 23587.87688648832,
|
||||||
|
"mean_input_tokens": 16384.0,
|
||||||
|
"mean_output_tokens": 128.0,
|
||||||
|
"ttft_p50_ms": 5949.235703003069,
|
||||||
|
"ttft_p95_ms": 9227.944630107957,
|
||||||
|
"ttft_p99_ms": 11097.429403630083,
|
||||||
|
"tpot_p50_ms": 39.72050276768026,
|
||||||
|
"tpot_p95_ms": 72.95558840476615,
|
||||||
|
"tpot_p99_ms": 80.88457267013912,
|
||||||
|
"e2e_p50_ms": 11065.538848997676,
|
||||||
|
"e2e_p95_ms": 13142.54083404594,
|
||||||
|
"e2e_p99_ms": 14229.989901815272,
|
||||||
|
"itl_p50_ms": 15.273823504685424,
|
||||||
|
"itl_p95_ms": 17.06440234847832,
|
||||||
|
"itl_p99_ms": 1069.5194568477748,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 188.87208026400185,
|
||||||
|
"request_tps": 0.4235671036617879,
|
||||||
|
"input_tps": 6939.723426394733,
|
||||||
|
"output_tps": 867.4654282993416,
|
||||||
|
"total_tps": 7807.1888546940745,
|
||||||
|
"mean_input_tokens": 16384.0,
|
||||||
|
"mean_output_tokens": 2048.0,
|
||||||
|
"ttft_p50_ms": 2707.821463998698,
|
||||||
|
"ttft_p95_ms": 6159.3108133580245,
|
||||||
|
"ttft_p99_ms": 7476.327504011569,
|
||||||
|
"tpot_p50_ms": 16.789645155595252,
|
||||||
|
"tpot_p95_ms": 19.235649654987466,
|
||||||
|
"tpot_p99_ms": 19.809279338319755,
|
||||||
|
"e2e_p50_ms": 37331.52540450101,
|
||||||
|
"e2e_p95_ms": 41633.905589940696,
|
||||||
|
"e2e_p99_ms": 41644.779238444025,
|
||||||
|
"itl_p50_ms": 15.280855994205922,
|
||||||
|
"itl_p95_ms": 18.66166050749598,
|
||||||
|
"itl_p99_ms": 19.038321707921565,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 64.32757946698985,
|
||||||
|
"request_tps": 1.2436345446676191,
|
||||||
|
"input_tps": 20375.708379834272,
|
||||||
|
"output_tps": 318.3704434349105,
|
||||||
|
"total_tps": 20694.07882326918,
|
||||||
|
"mean_input_tokens": 16384.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 4847.543320000113,
|
||||||
|
"ttft_p95_ms": 8945.170914454502,
|
||||||
|
"ttft_p99_ms": 10200.10803171811,
|
||||||
|
"tpot_p50_ms": 30.697892317645408,
|
||||||
|
"tpot_p95_ms": 45.12670790233909,
|
||||||
|
"tpot_p99_ms": 46.21028031682417,
|
||||||
|
"e2e_p50_ms": 12845.963567502622,
|
||||||
|
"e2e_p95_ms": 15142.341455946007,
|
||||||
|
"e2e_p99_ms": 15201.770440481196,
|
||||||
|
"itl_p50_ms": 15.241901011904702,
|
||||||
|
"itl_p95_ms": 16.105174794211056,
|
||||||
|
"itl_p99_ms": 806.8968500866322,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 72.41434827599733,
|
||||||
|
"request_tps": 1.1047534349835064,
|
||||||
|
"input_tps": 18100.28027876977,
|
||||||
|
"output_tps": 565.6337587115553,
|
||||||
|
"total_tps": 18665.914037481325,
|
||||||
|
"mean_input_tokens": 16384.0,
|
||||||
|
"mean_output_tokens": 512.0,
|
||||||
|
"ttft_p50_ms": 3977.608113505994,
|
||||||
|
"ttft_p95_ms": 6827.726796599017,
|
||||||
|
"ttft_p99_ms": 7064.793197970429,
|
||||||
|
"tpot_p50_ms": 20.6349563414749,
|
||||||
|
"tpot_p95_ms": 26.606408149222858,
|
||||||
|
"tpot_p99_ms": 27.689232515706706,
|
||||||
|
"e2e_p50_ms": 14308.850026493019,
|
||||||
|
"e2e_p95_ms": 14974.010983452172,
|
||||||
|
"e2e_p99_ms": 14998.68528622057,
|
||||||
|
"itl_p50_ms": 15.342150996730197,
|
||||||
|
"itl_p95_ms": 16.593942245526705,
|
||||||
|
"itl_p99_ms": 19.848780611646326,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 1511.6757434350002,
|
||||||
|
"request_tps": 0.05292140219053523,
|
||||||
|
"input_tps": 13873.028055835668,
|
||||||
|
"output_tps": 6.77393948038851,
|
||||||
|
"total_tps": 13879.801995316056,
|
||||||
|
"mean_input_tokens": 262144.0,
|
||||||
|
"mean_output_tokens": 128.0,
|
||||||
|
"ttft_p50_ms": 70474.41060249548,
|
||||||
|
"ttft_p95_ms": 206680.64425829216,
|
||||||
|
"ttft_p99_ms": 263259.9471526309,
|
||||||
|
"tpot_p50_ms": 1770.9045697204808,
|
||||||
|
"tpot_p95_ms": 1859.1631874913205,
|
||||||
|
"tpot_p99_ms": 1883.4251564144938,
|
||||||
|
"e2e_p50_ms": 293508.3537885075,
|
||||||
|
"e2e_p95_ms": 418159.4987228506,
|
||||||
|
"e2e_p99_ms": 491266.18717678235,
|
||||||
|
"itl_p50_ms": 1741.0211320020608,
|
||||||
|
"itl_p95_ms": 2297.1252548493794,
|
||||||
|
"itl_p99_ms": 2395.285054906126,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 1557.1239152909984,
|
||||||
|
"request_tps": 0.0513767717613209,
|
||||||
|
"input_tps": 13468.112456599705,
|
||||||
|
"output_tps": 13.15245357089815,
|
||||||
|
"total_tps": 13481.264910170603,
|
||||||
|
"mean_input_tokens": 262144.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 128086.70317550423,
|
||||||
|
"ttft_p95_ms": 293147.14115650055,
|
||||||
|
"ttft_p99_ms": 327246.22319525003,
|
||||||
|
"tpot_p50_ms": 694.1213535215974,
|
||||||
|
"tpot_p95_ms": 1110.7064755825186,
|
||||||
|
"tpot_p99_ms": 1121.9122485736023,
|
||||||
|
"e2e_p50_ms": 320114.9868900029,
|
||||||
|
"e2e_p95_ms": 442507.49014469614,
|
||||||
|
"e2e_p99_ms": 540593.0392589162,
|
||||||
|
"itl_p50_ms": 16.51875649986323,
|
||||||
|
"itl_p95_ms": 2135.694057152432,
|
||||||
|
"itl_p99_ms": 2348.4220405294036,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 84.46414080199611,
|
||||||
|
"request_tps": 0.9471475023648069,
|
||||||
|
"input_tps": 3879.516169686249,
|
||||||
|
"output_tps": 969.8790424215623,
|
||||||
|
"total_tps": 4849.395212107812,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 1024.0,
|
||||||
|
"ttft_p50_ms": 755.6758849968901,
|
||||||
|
"ttft_p95_ms": 1184.9343558955297,
|
||||||
|
"ttft_p99_ms": 1330.5567739978123,
|
||||||
|
"tpot_p50_ms": 15.835003849462446,
|
||||||
|
"tpot_p95_ms": 16.285198607422213,
|
||||||
|
"tpot_p99_ms": 16.55195564271857,
|
||||||
|
"e2e_p50_ms": 16962.706995996996,
|
||||||
|
"e2e_p95_ms": 17168.894911993266,
|
||||||
|
"e2e_p99_ms": 17178.149301250814,
|
||||||
|
"itl_p50_ms": 15.242625500832219,
|
||||||
|
"itl_p95_ms": 16.113438254251378,
|
||||||
|
"itl_p99_ms": 16.882939507922856,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 44.308171736003715,
|
||||||
|
"request_tps": 1.8055360188782963,
|
||||||
|
"input_tps": 7395.475533325502,
|
||||||
|
"output_tps": 231.10861041642193,
|
||||||
|
"total_tps": 7626.584143741924,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 128.0,
|
||||||
|
"ttft_p50_ms": 1465.1815819961485,
|
||||||
|
"ttft_p95_ms": 23821.693100642005,
|
||||||
|
"ttft_p99_ms": 24259.730825575534,
|
||||||
|
"tpot_p50_ms": 22.75063312988532,
|
||||||
|
"tpot_p95_ms": 194.92663006346027,
|
||||||
|
"tpot_p99_ms": 197.04311093216822,
|
||||||
|
"e2e_p50_ms": 4635.849632992176,
|
||||||
|
"e2e_p95_ms": 26182.2721034383,
|
||||||
|
"e2e_p99_ms": 26191.62205229819,
|
||||||
|
"itl_p50_ms": 15.226024501316715,
|
||||||
|
"itl_p95_ms": 16.216387940949062,
|
||||||
|
"itl_p99_ms": 414.4044824814774,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 161.3202662160038,
|
||||||
|
"request_tps": 0.4959079344245688,
|
||||||
|
"input_tps": 2031.2388994030339,
|
||||||
|
"output_tps": 1015.6194497015169,
|
||||||
|
"total_tps": 3046.858349104551,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 2048.0,
|
||||||
|
"ttft_p50_ms": 766.6918885006453,
|
||||||
|
"ttft_p95_ms": 1151.3519862077374,
|
||||||
|
"ttft_p99_ms": 1174.3669241876341,
|
||||||
|
"tpot_p50_ms": 15.32661797704204,
|
||||||
|
"tpot_p95_ms": 15.672662826941613,
|
||||||
|
"tpot_p99_ms": 15.77215861052261,
|
||||||
|
"e2e_p50_ms": 32366.54036600521,
|
||||||
|
"e2e_p95_ms": 32729.988757940737,
|
||||||
|
"e2e_p99_ms": 32742.016700837994,
|
||||||
|
"itl_p50_ms": 15.24984300340293,
|
||||||
|
"itl_p95_ms": 15.832478402444394,
|
||||||
|
"itl_p99_ms": 16.448479092650818,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 32.527519440001925,
|
||||||
|
"request_tps": 2.4594559123256423,
|
||||||
|
"input_tps": 10073.931416885831,
|
||||||
|
"output_tps": 629.6207135553644,
|
||||||
|
"total_tps": 10703.552130441196,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 1533.099703505286,
|
||||||
|
"ttft_p95_ms": 2719.3758142995644,
|
||||||
|
"ttft_p99_ms": 2959.5566013005737,
|
||||||
|
"tpot_p50_ms": 19.392000529407206,
|
||||||
|
"tpot_p95_ms": 23.99643604938105,
|
||||||
|
"tpot_p99_ms": 24.379314330662144,
|
||||||
|
"e2e_p50_ms": 6627.143721503671,
|
||||||
|
"e2e_p95_ms": 6811.619238646381,
|
||||||
|
"e2e_p99_ms": 6835.494205219293,
|
||||||
|
"itl_p50_ms": 15.377331001218408,
|
||||||
|
"itl_p95_ms": 16.272442803892773,
|
||||||
|
"itl_p99_ms": 18.94523348833901,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 316.8896808080026,
|
||||||
|
"request_tps": 0.252453787059322,
|
||||||
|
"input_tps": 1034.0507117949828,
|
||||||
|
"output_tps": 1034.0507117949828,
|
||||||
|
"total_tps": 2068.1014235899656,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 4096.0,
|
||||||
|
"ttft_p50_ms": 658.9246715011541,
|
||||||
|
"ttft_p95_ms": 1005.5294943493209,
|
||||||
|
"ttft_p99_ms": 1026.091546237294,
|
||||||
|
"tpot_p50_ms": 15.320106219048359,
|
||||||
|
"tpot_p95_ms": 15.378604879511569,
|
||||||
|
"tpot_p99_ms": 15.45278130631852,
|
||||||
|
"e2e_p50_ms": 63453.26198550174,
|
||||||
|
"e2e_p95_ms": 63513.736396850436,
|
||||||
|
"e2e_p99_ms": 63521.74632662762,
|
||||||
|
"itl_p50_ms": 15.256563012371771,
|
||||||
|
"itl_p95_ms": 15.777264998177998,
|
||||||
|
"itl_p99_ms": 16.359419524669644,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 48.17216287400515,
|
||||||
|
"request_tps": 1.6607101534809827,
|
||||||
|
"input_tps": 6802.268788658105,
|
||||||
|
"output_tps": 850.2835985822632,
|
||||||
|
"total_tps": 7652.552387240369,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 512.0,
|
||||||
|
"ttft_p50_ms": 1226.4635695100878,
|
||||||
|
"ttft_p95_ms": 1866.224592289654,
|
||||||
|
"ttft_p99_ms": 2426.2945817218856,
|
||||||
|
"tpot_p50_ms": 16.140000863013167,
|
||||||
|
"tpot_p95_ms": 17.640392132493695,
|
||||||
|
"tpot_p99_ms": 18.638964293663246,
|
||||||
|
"e2e_p50_ms": 9504.232979503286,
|
||||||
|
"e2e_p95_ms": 10157.524074796675,
|
||||||
|
"e2e_p99_ms": 10178.896629959345,
|
||||||
|
"itl_p50_ms": 15.288406488252804,
|
||||||
|
"itl_p95_ms": 15.857895543740598,
|
||||||
|
"itl_p99_ms": 17.156079869600944,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 376.9979162859963,
|
||||||
|
"request_tps": 0.21220276437631763,
|
||||||
|
"input_tps": 13906.920366166352,
|
||||||
|
"output_tps": 217.29563072134926,
|
||||||
|
"total_tps": 14124.215996887702,
|
||||||
|
"mean_input_tokens": 65536.0,
|
||||||
|
"mean_output_tokens": 1024.0,
|
||||||
|
"ttft_p50_ms": 25524.604717495095,
|
||||||
|
"ttft_p95_ms": 56727.187289898575,
|
||||||
|
"ttft_p99_ms": 65385.280855649864,
|
||||||
|
"tpot_p50_ms": 45.977114398337235,
|
||||||
|
"tpot_p95_ms": 68.72763672496647,
|
||||||
|
"tpot_p99_ms": 71.65886099602294,
|
||||||
|
"e2e_p50_ms": 72648.60455600865,
|
||||||
|
"e2e_p95_ms": 102850.95250014929,
|
||||||
|
"e2e_p99_ms": 128267.6254233617,
|
||||||
|
"itl_p50_ms": 15.504026996495668,
|
||||||
|
"itl_p95_ms": 16.275252601190004,
|
||||||
|
"itl_p99_ms": 1222.4654011012171,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 263.11688939499436,
|
||||||
|
"request_tps": 0.30404737675316235,
|
||||||
|
"input_tps": 19926.048882895248,
|
||||||
|
"output_tps": 38.91806422440478,
|
||||||
|
"total_tps": 19964.966947119654,
|
||||||
|
"mean_input_tokens": 65536.0,
|
||||||
|
"mean_output_tokens": 128.0,
|
||||||
|
"ttft_p50_ms": 21783.06187000271,
|
||||||
|
"ttft_p95_ms": 48708.6885400975,
|
||||||
|
"ttft_p99_ms": 53530.08918488397,
|
||||||
|
"tpot_p50_ms": 208.85715699601343,
|
||||||
|
"tpot_p95_ms": 353.9625207657282,
|
||||||
|
"tpot_p99_ms": 368.66045906663794,
|
||||||
|
"e2e_p50_ms": 53675.07621049299,
|
||||||
|
"e2e_p95_ms": 64500.535845247214,
|
||||||
|
"e2e_p99_ms": 78019.4687606334,
|
||||||
|
"itl_p50_ms": 15.624218496668618,
|
||||||
|
"itl_p95_ms": 1377.797994745198,
|
||||||
|
"itl_p99_ms": 1593.279839480674,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 318.6807930810028,
|
||||||
|
"request_tps": 0.2510348967898592,
|
||||||
|
"input_tps": 16451.82299602021,
|
||||||
|
"output_tps": 64.26493357820395,
|
||||||
|
"total_tps": 16516.087929598416,
|
||||||
|
"mean_input_tokens": 65536.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 25208.387847495032,
|
||||||
|
"ttft_p95_ms": 57570.60906205442,
|
||||||
|
"ttft_p99_ms": 64559.6027293129,
|
||||||
|
"tpot_p50_ms": 139.20286317058046,
|
||||||
|
"tpot_p95_ms": 218.68997558943673,
|
||||||
|
"tpot_p99_ms": 237.28532163909728,
|
||||||
|
"e2e_p50_ms": 62259.9117714999,
|
||||||
|
"e2e_p95_ms": 93757.1761358522,
|
||||||
|
"e2e_p99_ms": 109198.36791741903,
|
||||||
|
"itl_p50_ms": 15.581318992190063,
|
||||||
|
"itl_p95_ms": 1060.9013718029018,
|
||||||
|
"itl_p99_ms": 1579.705025086296,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 80,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 337.8231221129972,
|
||||||
|
"request_tps": 0.23681031511289238,
|
||||||
|
"input_tps": 15519.600811238515,
|
||||||
|
"output_tps": 121.2468813378009,
|
||||||
|
"total_tps": 15640.847692576317,
|
||||||
|
"mean_input_tokens": 65536.0,
|
||||||
|
"mean_output_tokens": 512.0,
|
||||||
|
"ttft_p50_ms": 24695.115143498697,
|
||||||
|
"ttft_p95_ms": 57546.68756374739,
|
||||||
|
"ttft_p99_ms": 64468.70614748111,
|
||||||
|
"tpot_p50_ms": 76.04405581409218,
|
||||||
|
"tpot_p95_ms": 120.77805700313024,
|
||||||
|
"tpot_p99_ms": 135.3599259530104,
|
||||||
|
"e2e_p50_ms": 66170.59861750022,
|
||||||
|
"e2e_p95_ms": 92553.88260725957,
|
||||||
|
"e2e_p99_ms": 109859.61571042174,
|
||||||
|
"itl_p50_ms": 15.599313002894633,
|
||||||
|
"itl_p95_ms": 824.2479222499242,
|
||||||
|
"itl_p99_ms": 1410.173195308016,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 5,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 97.25075286399806,
|
||||||
|
"request_tps": 0.051413483728936615,
|
||||||
|
"input_tps": 6738.86813931918,
|
||||||
|
"output_tps": 13.161851834607774,
|
||||||
|
"total_tps": 6752.029991153788,
|
||||||
|
"mean_input_tokens": 131072.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 16969.752281991532,
|
||||||
|
"ttft_p95_ms": 17121.84501980082,
|
||||||
|
"ttft_p99_ms": 17125.887859960785,
|
||||||
|
"tpot_p50_ms": 9.60843350975683,
|
||||||
|
"tpot_p95_ms": 9.634889066653034,
|
||||||
|
"tpot_p99_ms": 9.637108200772976,
|
||||||
|
"e2e_p50_ms": 19427.356342988787,
|
||||||
|
"e2e_p95_ms": 19571.998772607185,
|
||||||
|
"e2e_p99_ms": 19575.258900129702,
|
||||||
|
"itl_p50_ms": 10.00273649697192,
|
||||||
|
"itl_p95_ms": 10.19823560709483,
|
||||||
|
"itl_p99_ms": 11.044782063545426,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 5,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 110.12223424699914,
|
||||||
|
"request_tps": 0.04540409150058859,
|
||||||
|
"input_tps": 5951.205081165148,
|
||||||
|
"output_tps": 23.24689484830136,
|
||||||
|
"total_tps": 5974.45197601345,
|
||||||
|
"mean_input_tokens": 131072.0,
|
||||||
|
"mean_output_tokens": 512.0,
|
||||||
|
"ttft_p50_ms": 16962.474314001156,
|
||||||
|
"ttft_p95_ms": 17155.541892390465,
|
||||||
|
"ttft_p99_ms": 17171.51858087047,
|
||||||
|
"tpot_p50_ms": 9.821654945213728,
|
||||||
|
"tpot_p95_ms": 9.835613834436566,
|
||||||
|
"tpot_p99_ms": 9.837956189974383,
|
||||||
|
"e2e_p50_ms": 21989.969162998023,
|
||||||
|
"e2e_p95_ms": 22173.37846419541,
|
||||||
|
"e2e_p99_ms": 22190.17843683483,
|
||||||
|
"itl_p50_ms": 9.989651007344946,
|
||||||
|
"itl_p95_ms": 10.151538599166088,
|
||||||
|
"itl_p99_ms": 10.469720991677605,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 5,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 215.86107532100868,
|
||||||
|
"request_tps": 0.02316304591999489,
|
||||||
|
"input_tps": 6072.05350965114,
|
||||||
|
"output_tps": 2.964869877759346,
|
||||||
|
"total_tps": 6075.0183795289,
|
||||||
|
"mean_input_tokens": 262144.0,
|
||||||
|
"mean_output_tokens": 128.0,
|
||||||
|
"ttft_p50_ms": 42189.58094799018,
|
||||||
|
"ttft_p95_ms": 42206.01177979843,
|
||||||
|
"ttft_p99_ms": 42207.67736715847,
|
||||||
|
"tpot_p50_ms": 9.026367795253979,
|
||||||
|
"tpot_p95_ms": 9.051597466134083,
|
||||||
|
"tpot_p99_ms": 9.053308627064942,
|
||||||
|
"e2e_p50_ms": 43335.3122970002,
|
||||||
|
"e2e_p95_ms": 43353.944078995846,
|
||||||
|
"e2e_p99_ms": 43357.12344699539,
|
||||||
|
"itl_p50_ms": 10.197708994382992,
|
||||||
|
"itl_p95_ms": 10.398621491185622,
|
||||||
|
"itl_p99_ms": 11.979914264811663,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 5,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 222.6894240649999,
|
||||||
|
"request_tps": 0.022452795057481358,
|
||||||
|
"input_tps": 5885.865507548393,
|
||||||
|
"output_tps": 5.7479155347152275,
|
||||||
|
"total_tps": 5891.6134230831085,
|
||||||
|
"mean_input_tokens": 262144.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 41992.54209799983,
|
||||||
|
"ttft_p95_ms": 42385.26861999417,
|
||||||
|
"ttft_p99_ms": 42405.3075527912,
|
||||||
|
"tpot_p50_ms": 9.564196976419884,
|
||||||
|
"tpot_p95_ms": 9.778543265853717,
|
||||||
|
"tpot_p99_ms": 9.797764013144736,
|
||||||
|
"e2e_p50_ms": 44431.4123269869,
|
||||||
|
"e2e_p95_ms": 44819.5239008026,
|
||||||
|
"e2e_p99_ms": 44832.609473761986,
|
||||||
|
"itl_p50_ms": 10.17798750399379,
|
||||||
|
"itl_p95_ms": 10.34693365872954,
|
||||||
|
"itl_p99_ms": 10.991512061009418,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 160,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 390.9258493729867,
|
||||||
|
"request_tps": 0.40928477934275004,
|
||||||
|
"input_tps": 419.10761404697604,
|
||||||
|
"output_tps": 1676.4304561879042,
|
||||||
|
"total_tps": 2095.5380702348803,
|
||||||
|
"mean_input_tokens": 1024.0,
|
||||||
|
"mean_output_tokens": 4096.0,
|
||||||
|
"ttft_p50_ms": 1059.6694034975371,
|
||||||
|
"ttft_p95_ms": 1531.6532846380142,
|
||||||
|
"ttft_p99_ms": 1558.6441262916196,
|
||||||
|
"tpot_p50_ms": 18.821365679975997,
|
||||||
|
"tpot_p95_ms": 19.016937038081775,
|
||||||
|
"tpot_p99_ms": 19.073814857857453,
|
||||||
|
"e2e_p50_ms": 77972.91494200181,
|
||||||
|
"e2e_p95_ms": 78736.66218614817,
|
||||||
|
"e2e_p99_ms": 78754.35213367702,
|
||||||
|
"itl_p50_ms": 18.73732249805471,
|
||||||
|
"itl_p95_ms": 19.672631401772378,
|
||||||
|
"itl_p99_ms": 20.632040495693218,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 160,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 107.34333604600397,
|
||||||
|
"request_tps": 1.490544321553683,
|
||||||
|
"input_tps": 6105.269541083886,
|
||||||
|
"output_tps": 1526.3173852709715,
|
||||||
|
"total_tps": 7631.5869263548575,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 1024.0,
|
||||||
|
"ttft_p50_ms": 1697.2974899981637,
|
||||||
|
"ttft_p95_ms": 2827.3090714494147,
|
||||||
|
"ttft_p99_ms": 3041.8824867265357,
|
||||||
|
"tpot_p50_ms": 19.041262989738744,
|
||||||
|
"tpot_p95_ms": 20.483581416088622,
|
||||||
|
"tpot_p99_ms": 20.796884204547236,
|
||||||
|
"e2e_p50_ms": 21322.126428000047,
|
||||||
|
"e2e_p95_ms": 21977.338839244476,
|
||||||
|
"e2e_p99_ms": 21997.641324478464,
|
||||||
|
"itl_p50_ms": 18.680188004509546,
|
||||||
|
"itl_p95_ms": 19.772065294091586,
|
||||||
|
"itl_p99_ms": 21.03478136268678,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 160,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 203.61964830600482,
|
||||||
|
"request_tps": 0.7857787857463927,
|
||||||
|
"input_tps": 3218.5499064172245,
|
||||||
|
"output_tps": 1609.2749532086123,
|
||||||
|
"total_tps": 4827.824859625837,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 2048.0,
|
||||||
|
"ttft_p50_ms": 1823.4369839992723,
|
||||||
|
"ttft_p95_ms": 2880.811745747633,
|
||||||
|
"ttft_p99_ms": 2898.1182565601193,
|
||||||
|
"tpot_p50_ms": 18.92700290888939,
|
||||||
|
"tpot_p95_ms": 19.648448953663472,
|
||||||
|
"tpot_p99_ms": 19.892009308511824,
|
||||||
|
"e2e_p50_ms": 40591.624035492714,
|
||||||
|
"e2e_p95_ms": 41278.74980429624,
|
||||||
|
"e2e_p99_ms": 41301.732580115786,
|
||||||
|
"itl_p50_ms": 18.753395997919142,
|
||||||
|
"itl_p95_ms": 19.575604447891237,
|
||||||
|
"itl_p99_ms": 20.460386879567523,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 160,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 66.26290708300075,
|
||||||
|
"request_tps": 2.414623913188481,
|
||||||
|
"input_tps": 9890.299548420018,
|
||||||
|
"output_tps": 618.1437217762511,
|
||||||
|
"total_tps": 10508.443270196269,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 3032.6789590035332,
|
||||||
|
"ttft_p95_ms": 5014.513322354456,
|
||||||
|
"ttft_p99_ms": 14411.20142712994,
|
||||||
|
"tpot_p50_ms": 25.094734256865696,
|
||||||
|
"tpot_p95_ms": 108.72940146214187,
|
||||||
|
"tpot_p99_ms": 110.79764089783636,
|
||||||
|
"e2e_p50_ms": 9434.836903004907,
|
||||||
|
"e2e_p95_ms": 29872.09643335227,
|
||||||
|
"e2e_p99_ms": 29884.739060596476,
|
||||||
|
"itl_p50_ms": 18.683340997085907,
|
||||||
|
"itl_p95_ms": 19.744293195253704,
|
||||||
|
"itl_p99_ms": 26.305865632602966,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 160,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 389.95699894400605,
|
||||||
|
"request_tps": 0.4103016497543987,
|
||||||
|
"input_tps": 1680.595557394017,
|
||||||
|
"output_tps": 1680.595557394017,
|
||||||
|
"total_tps": 3361.191114788034,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 4096.0,
|
||||||
|
"ttft_p50_ms": 1123.4332419990096,
|
||||||
|
"ttft_p95_ms": 1676.0268465513946,
|
||||||
|
"ttft_p99_ms": 1717.2243972464637,
|
||||||
|
"tpot_p50_ms": 18.85827924127029,
|
||||||
|
"tpot_p95_ms": 19.032755866286788,
|
||||||
|
"tpot_p99_ms": 19.08965374972383,
|
||||||
|
"e2e_p50_ms": 77915.13783499977,
|
||||||
|
"e2e_p95_ms": 78629.53818804745,
|
||||||
|
"e2e_p99_ms": 78646.88334376202,
|
||||||
|
"itl_p50_ms": 18.702409994148184,
|
||||||
|
"itl_p95_ms": 19.55858354413067,
|
||||||
|
"itl_p99_ms": 20.443303814536193,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 160,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 63.387327321004705,
|
||||||
|
"request_tps": 2.52416384728972,
|
||||||
|
"input_tps": 10338.975118498693,
|
||||||
|
"output_tps": 1292.3718898123366,
|
||||||
|
"total_tps": 11631.347008311028,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 512.0,
|
||||||
|
"ttft_p50_ms": 2139.6533224979066,
|
||||||
|
"ttft_p95_ms": 3791.57651143978,
|
||||||
|
"ttft_p99_ms": 4309.74766367377,
|
||||||
|
"tpot_p50_ms": 20.01632839431788,
|
||||||
|
"tpot_p95_ms": 24.519710790119547,
|
||||||
|
"tpot_p99_ms": 25.129288290261652,
|
||||||
|
"e2e_p50_ms": 12805.874345496704,
|
||||||
|
"e2e_p95_ms": 13800.694600059069,
|
||||||
|
"e2e_p99_ms": 13819.40531841843,
|
||||||
|
"itl_p50_ms": 18.705550508457236,
|
||||||
|
"itl_p95_ms": 20.301984495017674,
|
||||||
|
"itl_p99_ms": 21.65679321915376,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 240,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 395.37849693700264,
|
||||||
|
"request_tps": 0.6070132843826361,
|
||||||
|
"input_tps": 621.5816032078194,
|
||||||
|
"output_tps": 2486.3264128312776,
|
||||||
|
"total_tps": 3107.9080160390968,
|
||||||
|
"mean_input_tokens": 1024.0,
|
||||||
|
"mean_output_tokens": 4096.0,
|
||||||
|
"ttft_p50_ms": 1144.204484502552,
|
||||||
|
"ttft_p95_ms": 1999.596584997198,
|
||||||
|
"ttft_p99_ms": 2035.2585943651502,
|
||||||
|
"tpot_p50_ms": 18.99256894627527,
|
||||||
|
"tpot_p95_ms": 19.224950862944212,
|
||||||
|
"tpot_p99_ms": 19.230104005436754,
|
||||||
|
"e2e_p50_ms": 78907.89274750568,
|
||||||
|
"e2e_p95_ms": 79839.60575450546,
|
||||||
|
"e2e_p99_ms": 79862.91444651186,
|
||||||
|
"itl_p50_ms": 18.858423005440272,
|
||||||
|
"itl_p95_ms": 20.01599159266334,
|
||||||
|
"itl_p99_ms": 21.22574228560552,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 240,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 114.72205163900799,
|
||||||
|
"request_tps": 2.092012795893852,
|
||||||
|
"input_tps": 8568.884411981218,
|
||||||
|
"output_tps": 2142.2211029953046,
|
||||||
|
"total_tps": 10711.105514976523,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 1024.0,
|
||||||
|
"ttft_p50_ms": 2002.436676499201,
|
||||||
|
"ttft_p95_ms": 5319.875290898926,
|
||||||
|
"ttft_p99_ms": 6160.708812107041,
|
||||||
|
"tpot_p50_ms": 19.824865686698722,
|
||||||
|
"tpot_p95_ms": 23.0216869557243,
|
||||||
|
"tpot_p99_ms": 23.20206612341515,
|
||||||
|
"e2e_p50_ms": 21923.11628299649,
|
||||||
|
"e2e_p95_ms": 25281.340099497902,
|
||||||
|
"e2e_p99_ms": 25297.796992440562,
|
||||||
|
"itl_p50_ms": 18.851815999369137,
|
||||||
|
"itl_p95_ms": 20.00413840287365,
|
||||||
|
"itl_p99_ms": 21.528728755656623,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 240,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 209.64498886499496,
|
||||||
|
"request_tps": 1.1447924479346976,
|
||||||
|
"input_tps": 4689.069866740521,
|
||||||
|
"output_tps": 2344.5349333702607,
|
||||||
|
"total_tps": 7033.604800110782,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 2048.0,
|
||||||
|
"ttft_p50_ms": 1850.1262010031496,
|
||||||
|
"ttft_p95_ms": 5557.007933600835,
|
||||||
|
"ttft_p99_ms": 5728.514949967212,
|
||||||
|
"tpot_p50_ms": 19.032042742548224,
|
||||||
|
"tpot_p95_ms": 20.8632619918908,
|
||||||
|
"tpot_p99_ms": 21.040059630580483,
|
||||||
|
"e2e_p50_ms": 40867.48082550184,
|
||||||
|
"e2e_p95_ms": 44286.82900869899,
|
||||||
|
"e2e_p99_ms": 44307.41918329746,
|
||||||
|
"itl_p50_ms": 18.80523749423446,
|
||||||
|
"itl_p95_ms": 20.01578960043844,
|
||||||
|
"itl_p99_ms": 21.193549290037485,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 240,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 402.4469851330068,
|
||||||
|
"request_tps": 0.596351839785012,
|
||||||
|
"input_tps": 2442.6571357594094,
|
||||||
|
"output_tps": 2442.6571357594094,
|
||||||
|
"total_tps": 4885.314271518819,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 4096.0,
|
||||||
|
"ttft_p50_ms": 2047.3789034949732,
|
||||||
|
"ttft_p95_ms": 4081.0472775985545,
|
||||||
|
"ttft_p99_ms": 4647.203718657256,
|
||||||
|
"tpot_p50_ms": 19.02283846043863,
|
||||||
|
"tpot_p95_ms": 19.43320571131982,
|
||||||
|
"tpot_p99_ms": 19.899421790027564,
|
||||||
|
"e2e_p50_ms": 80315.7795865045,
|
||||||
|
"e2e_p95_ms": 81844.74676285972,
|
||||||
|
"e2e_p99_ms": 81865.64722886324,
|
||||||
|
"itl_p50_ms": 18.825628008926287,
|
||||||
|
"itl_p95_ms": 19.975638002506457,
|
||||||
|
"itl_p99_ms": 21.221037310315303,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 240,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 77.56397197001206,
|
||||||
|
"request_tps": 3.0942200857479203,
|
||||||
|
"input_tps": 12673.925471223482,
|
||||||
|
"output_tps": 1584.2406839029352,
|
||||||
|
"total_tps": 14258.166155126417,
|
||||||
|
"mean_input_tokens": 4096.0,
|
||||||
|
"mean_output_tokens": 512.0,
|
||||||
|
"ttft_p50_ms": 2946.3751199946273,
|
||||||
|
"ttft_p95_ms": 6820.920404194476,
|
||||||
|
"ttft_p99_ms": 8050.301623321715,
|
||||||
|
"tpot_p50_ms": 23.094741272006033,
|
||||||
|
"tpot_p95_ms": 31.340273439141725,
|
||||||
|
"tpot_p99_ms": 33.989053702566764,
|
||||||
|
"e2e_p50_ms": 15109.904168501089,
|
||||||
|
"e2e_p95_ms": 19630.664032053755,
|
||||||
|
"e2e_p99_ms": 19649.577008009364,
|
||||||
|
"itl_p50_ms": 19.291372009320185,
|
||||||
|
"itl_p95_ms": 22.182088205590844,
|
||||||
|
"itl_p99_ms": 24.491042752633813,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 320,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 443.57241661399894,
|
||||||
|
"request_tps": 0.7214154623110101,
|
||||||
|
"input_tps": 738.7294334064743,
|
||||||
|
"output_tps": 2954.917733625897,
|
||||||
|
"total_tps": 3693.6471670323713,
|
||||||
|
"mean_input_tokens": 1024.0,
|
||||||
|
"mean_output_tokens": 4096.0,
|
||||||
|
"ttft_p50_ms": 1471.407580494997,
|
||||||
|
"ttft_p95_ms": 2597.0414660041592,
|
||||||
|
"ttft_p99_ms": 2976.737847158656,
|
||||||
|
"tpot_p50_ms": 21.723876868741236,
|
||||||
|
"tpot_p95_ms": 22.00444482439625,
|
||||||
|
"tpot_p99_ms": 22.07639913107326,
|
||||||
|
"e2e_p50_ms": 90742.18811100582,
|
||||||
|
"e2e_p95_ms": 91541.6523386979,
|
||||||
|
"e2e_p99_ms": 91559.50254334108,
|
||||||
|
"itl_p50_ms": 21.4493349994882,
|
||||||
|
"itl_p95_ms": 22.813913105346728,
|
||||||
|
"itl_p99_ms": 24.170049769309127,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 400,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 463.4503973050014,
|
||||||
|
"request_tps": 0.8630912872791345,
|
||||||
|
"input_tps": 883.8054781738338,
|
||||||
|
"output_tps": 3535.221912695335,
|
||||||
|
"total_tps": 4419.027390869169,
|
||||||
|
"mean_input_tokens": 1024.0,
|
||||||
|
"mean_output_tokens": 4096.0,
|
||||||
|
"ttft_p50_ms": 2439.6097005082993,
|
||||||
|
"ttft_p95_ms": 3686.9959791976726,
|
||||||
|
"ttft_p99_ms": 3709.631283667256,
|
||||||
|
"tpot_p50_ms": 22.095354364345052,
|
||||||
|
"tpot_p95_ms": 22.518257637104984,
|
||||||
|
"tpot_p99_ms": 22.625587380977006,
|
||||||
|
"e2e_p50_ms": 92209.49949500209,
|
||||||
|
"e2e_p95_ms": 93844.80823813865,
|
||||||
|
"e2e_p99_ms": 93874.67774137796,
|
||||||
|
"itl_p50_ms": 21.87070349464193,
|
||||||
|
"itl_p95_ms": 23.618548751983326,
|
||||||
|
"itl_p99_ms": 25.773182249395177,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 40,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 341.0657499819936,
|
||||||
|
"request_tps": 0.11727943952775025,
|
||||||
|
"input_tps": 15372.05069778128,
|
||||||
|
"output_tps": 30.023536519104063,
|
||||||
|
"total_tps": 15402.074234300384,
|
||||||
|
"mean_input_tokens": 131072.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 38767.44465551019,
|
||||||
|
"ttft_p95_ms": 67753.44985890988,
|
||||||
|
"ttft_p99_ms": 68613.70976785096,
|
||||||
|
"tpot_p50_ms": 99.49740432157162,
|
||||||
|
"tpot_p95_ms": 179.5039385709941,
|
||||||
|
"tpot_p99_ms": 227.61194875988735,
|
||||||
|
"e2e_p50_ms": 70759.12754750607,
|
||||||
|
"e2e_p95_ms": 71843.55354870277,
|
||||||
|
"e2e_p99_ms": 71887.88047623617,
|
||||||
|
"itl_p50_ms": 12.980383995454758,
|
||||||
|
"itl_p95_ms": 1125.97120024584,
|
||||||
|
"itl_p99_ms": 1602.8499752024072,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 40,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 368.12840555600997,
|
||||||
|
"request_tps": 0.10865773843120097,
|
||||||
|
"input_tps": 14241.987091654373,
|
||||||
|
"output_tps": 55.632762076774895,
|
||||||
|
"total_tps": 14297.619853731148,
|
||||||
|
"mean_input_tokens": 131072.0,
|
||||||
|
"mean_output_tokens": 512.0,
|
||||||
|
"ttft_p50_ms": 36897.75435599586,
|
||||||
|
"ttft_p95_ms": 68026.47409820274,
|
||||||
|
"ttft_p99_ms": 74667.961793141,
|
||||||
|
"tpot_p50_ms": 63.40141743053057,
|
||||||
|
"tpot_p95_ms": 116.81624726055898,
|
||||||
|
"tpot_p99_ms": 143.86918745584376,
|
||||||
|
"e2e_p50_ms": 74333.1425399956,
|
||||||
|
"e2e_p95_ms": 83933.02225654986,
|
||||||
|
"e2e_p99_ms": 104491.18695786949,
|
||||||
|
"itl_p50_ms": 12.98504300211789,
|
||||||
|
"itl_p95_ms": 18.927634652209232,
|
||||||
|
"itl_p99_ms": 1447.1019217558205,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 40,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 1143.3382095730049,
|
||||||
|
"request_tps": 0.03498527353068918,
|
||||||
|
"input_tps": 9171.179544428984,
|
||||||
|
"output_tps": 4.478115011928215,
|
||||||
|
"total_tps": 9175.65765944091,
|
||||||
|
"mean_input_tokens": 262144.0,
|
||||||
|
"mean_output_tokens": 128.0,
|
||||||
|
"ttft_p50_ms": 60434.31567850348,
|
||||||
|
"ttft_p95_ms": 171402.15240599928,
|
||||||
|
"ttft_p99_ms": 221100.73346630885,
|
||||||
|
"tpot_p50_ms": 1432.7522763937384,
|
||||||
|
"tpot_p95_ms": 1566.4298780566398,
|
||||||
|
"tpot_p99_ms": 1577.578224665822,
|
||||||
|
"e2e_p50_ms": 243043.1625314959,
|
||||||
|
"e2e_p95_ms": 344139.9401556495,
|
||||||
|
"e2e_p99_ms": 419677.4824362584,
|
||||||
|
"itl_p50_ms": 1394.7447990067303,
|
||||||
|
"itl_p95_ms": 1954.1919693932869,
|
||||||
|
"itl_p99_ms": 2118.3612401573937,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 40,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 818.7336148600007,
|
||||||
|
"request_tps": 0.04885593955591991,
|
||||||
|
"input_tps": 12807.291418947068,
|
||||||
|
"output_tps": 12.507120526315497,
|
||||||
|
"total_tps": 12819.798539473382,
|
||||||
|
"mean_input_tokens": 262144.0,
|
||||||
|
"mean_output_tokens": 256.0,
|
||||||
|
"ttft_p50_ms": 105032.74306450476,
|
||||||
|
"ttft_p95_ms": 162785.75602490891,
|
||||||
|
"ttft_p99_ms": 165083.89961342298,
|
||||||
|
"tpot_p50_ms": 214.14210347058685,
|
||||||
|
"tpot_p95_ms": 422.65357399257476,
|
||||||
|
"tpot_p99_ms": 440.83721461195177,
|
||||||
|
"e2e_p50_ms": 164185.14075000712,
|
||||||
|
"e2e_p95_ms": 168861.28058029353,
|
||||||
|
"e2e_p99_ms": 169246.668313704,
|
||||||
|
"itl_p50_ms": 13.376075992709957,
|
||||||
|
"itl_p95_ms": 1720.7928034986253,
|
||||||
|
"itl_p99_ms": 2045.3944211971263,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1,25 @@
|
|||||||
|
{
|
||||||
|
"status": "COMPLETED",
|
||||||
|
"completed": 480,
|
||||||
|
"failed": 0,
|
||||||
|
"duration_s": 524.4931549339963,
|
||||||
|
"request_tps": 0.9151692362132059,
|
||||||
|
"input_tps": 937.1332978823228,
|
||||||
|
"output_tps": 3748.5331915292913,
|
||||||
|
"total_tps": 4685.666489411614,
|
||||||
|
"mean_input_tokens": 1024.0,
|
||||||
|
"mean_output_tokens": 4096.0,
|
||||||
|
"ttft_p50_ms": 2751.5591720002703,
|
||||||
|
"ttft_p95_ms": 4227.864060657878,
|
||||||
|
"ttft_p99_ms": 4536.041375702771,
|
||||||
|
"tpot_p50_ms": 24.893937553724363,
|
||||||
|
"tpot_p95_ms": 25.43370572488336,
|
||||||
|
"tpot_p99_ms": 25.74416057706036,
|
||||||
|
"e2e_p50_ms": 104575.3224170112,
|
||||||
|
"e2e_p95_ms": 106446.31684675842,
|
||||||
|
"e2e_p99_ms": 106478.32935052022,
|
||||||
|
"itl_p50_ms": 24.56568699562922,
|
||||||
|
"itl_p95_ms": 27.559315803227946,
|
||||||
|
"itl_p99_ms": 30.286136486683972,
|
||||||
|
"validation_errors": []
|
||||||
|
}
|
||||||
@ -0,0 +1 @@
|
|||||||
|
vllm serve /data1/hf_models/DeepSeek-V4-Flash --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 4 --gpu-memory-utilization 0.9 --max-model-len 1048576 --max-num-seqs 128 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 --data-parallel-size 2
|
||||||
@ -166,5 +166,8 @@ export SEARCH_ADDEND=16
|
|||||||
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
# If the initial concurrency violates the TTFT SLO, search downward. Stop at
|
||||||
# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
|
# the first acceptable value (16 -> 8; only try 1 when 8 still violates it).
|
||||||
export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
|
export SEARCH_INITIAL_BACKOFF_CONCURRENCIES="8 1"
|
||||||
|
# When concurrency 1 still has a severely excessive TTFT, stop the remaining
|
||||||
|
# shapes in this TP/DP group. Zero disables this rule.
|
||||||
|
export TTFT_GROUP_SKIP_MS="${TTFT_GROUP_SKIP_MS:-8000}"
|
||||||
|
|
||||||
adaptive_main "$@"
|
adaptive_main "$@"
|
||||||
|
|||||||
@ -0,0 +1,167 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Find the Total-TPS saturation concurrency for each vLLM TP/DP/ISL/OSL shape.
|
||||||
|
set -Eeuo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
EXPERIMENT_NAME="$(basename "$SCRIPT_DIR")"
|
||||||
|
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/lib.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/platform.sh"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/adaptive_config.env"
|
||||||
|
# shellcheck source=/dev/null
|
||||||
|
source "${SCRIPT_DIR}/../../../scripts/common/adaptive_bench_lib.sh"
|
||||||
|
|
||||||
|
ENGINE="vllm"
|
||||||
|
ENGINE_PORT="$VLLM_PORT"
|
||||||
|
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
|
||||||
|
if [[ -x "${VENV_CLIENT}/bin/python" ]]; then
|
||||||
|
PYTHON="${VENV_CLIENT}/bin/python"
|
||||||
|
else
|
||||||
|
PYTHON="$(command -v python3)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
DOCKER_IMAGE="${DOCKER_IMAGE:-vllm/vllm-openai:latest}"
|
||||||
|
DOCKER_CLIENT_IMAGE="${DOCKER_CLIENT_IMAGE:-lmsysorg/sglang:latest}"
|
||||||
|
|
||||||
|
engine_is_healthy() {
|
||||||
|
curl --fail --silent --show-error --max-time 5 \
|
||||||
|
"http://127.0.0.1:${ENGINE_PORT}/health" >/dev/null 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_stop_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local pid_file="${RUNTIME_BASE}/${EXPERIMENT}_vllm_tp${tp}_dp${dp}.pid"
|
||||||
|
|
||||||
|
if [[ -f "$pid_file" ]]; then
|
||||||
|
local pid
|
||||||
|
pid="$(cat "$pid_file")"
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
log "stopping vllm server pid=${pid} tp=${tp} dp=${dp}"
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 5
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
rm -f "$pid_file"
|
||||||
|
fi
|
||||||
|
docker rm -f "${EXPERIMENT}_vllm_tp${tp}_dp${dp}" >/dev/null 2>&1 || true
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG=""
|
||||||
|
sleep 2
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_build_server_args() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local -a args=(
|
||||||
|
vllm serve "$MODEL_PATH"
|
||||||
|
--trust-remote-code
|
||||||
|
--kv-cache-dtype "$KV_CACHE_DTYPE"
|
||||||
|
--block-size "$BLOCK_SIZE"
|
||||||
|
--tensor-parallel-size "$tp"
|
||||||
|
--gpu-memory-utilization "$GPU_MEMORY_UTILIZATION"
|
||||||
|
--max-model-len "$MAX_MODEL_LEN"
|
||||||
|
--max-num-seqs "$MAX_NUM_SEQS"
|
||||||
|
--no-enable-flashinfer-autotune
|
||||||
|
--host 0.0.0.0
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
)
|
||||||
|
if (( dp > 1 )); then
|
||||||
|
args+=(--data-parallel-size "$dp")
|
||||||
|
fi
|
||||||
|
printf '%q ' "${args[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_start_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
log "starting vllm server tp=${tp} dp=${dp}"
|
||||||
|
bash "${SCRIPT_DIR}/start_vllm_dp.sh" "$tp" "$dp" >> "$outer_log" 2>&1
|
||||||
|
if ! engine_is_healthy; then
|
||||||
|
log "ERROR: vllm health check failed tp=${tp} dp=${dp}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
ACTIVE_ENGINE_SERVER_LOG="$(
|
||||||
|
find "${RUNTIME_BASE}/logs" -maxdepth 1 -type f \
|
||||||
|
-name "${EXPERIMENT}_vllm*tp${tp}_dp${dp}_*.log" \
|
||||||
|
-printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-
|
||||||
|
)"
|
||||||
|
log "vllm server healthy tp=${tp} dp=${dp} log=${ACTIVE_ENGINE_SERVER_LOG:-unknown}"
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_detect_oom() {
|
||||||
|
local detail_log="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local pattern='CUDA out of memory|torch\.OutOfMemoryError|OutOfMemory|out of memory|OOM|RESOURCE_EXHAUSTED|Failed to allocate memory'
|
||||||
|
local outer_log="${ADAPTIVE_LOG_DIR}/vllm_tp${tp}_dp${dp}.server.outer.log"
|
||||||
|
local -a logs=("$detail_log" "$outer_log")
|
||||||
|
if [[ -n "$ACTIVE_ENGINE_SERVER_LOG" ]]; then
|
||||||
|
logs+=("$ACTIVE_ENGINE_SERVER_LOG")
|
||||||
|
fi
|
||||||
|
grep -Eiq "$pattern" "${logs[@]}" 2>/dev/null
|
||||||
|
}
|
||||||
|
|
||||||
|
engine_run_bench() {
|
||||||
|
local isl="$1"
|
||||||
|
local osl="$2"
|
||||||
|
local concurrency="$3"
|
||||||
|
local num_prompts="$4"
|
||||||
|
local output_file="$5"
|
||||||
|
local warmup_requests
|
||||||
|
warmup_requests="$(adaptive_warmup_request_count "$concurrency")"
|
||||||
|
local -a bench_args=(
|
||||||
|
--backend vllm
|
||||||
|
--host 127.0.0.1
|
||||||
|
--port "$ENGINE_PORT"
|
||||||
|
--dataset-name "$BENCH_DATASET_NAME"
|
||||||
|
--random-input-len "$isl"
|
||||||
|
--random-output-len "$osl"
|
||||||
|
--random-range-ratio "$RANDOM_RANGE_RATIO"
|
||||||
|
--num-prompts "$num_prompts"
|
||||||
|
--max-concurrency "$concurrency"
|
||||||
|
--request-rate 10000
|
||||||
|
--warmup-requests "$warmup_requests"
|
||||||
|
--output-file "$output_file"
|
||||||
|
--output-details
|
||||||
|
--disable-tqdm
|
||||||
|
)
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
bench_args+=(--dataset-path "$DATASET_PATH")
|
||||||
|
else
|
||||||
|
bench_args+=(--tokenize-prompt)
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ "$USE_DOCKER_CLIENT" == "1" ]]; then
|
||||||
|
local -a volume_args=(-v "${MODEL_PATH}:${MODEL_PATH}:ro" -v "${RESULT_BASE}:${RESULT_BASE}")
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
volume_args+=(-v "${DATASET_PATH}:${DATASET_PATH}:ro")
|
||||||
|
fi
|
||||||
|
docker run --rm \
|
||||||
|
--network host \
|
||||||
|
"${volume_args[@]}" \
|
||||||
|
-e PYTHONUNBUFFERED=1 \
|
||||||
|
-e HF_HUB_OFFLINE=1 \
|
||||||
|
-e TRANSFORMERS_OFFLINE=1 \
|
||||||
|
"$DOCKER_CLIENT_IMAGE" \
|
||||||
|
python -m sglang.bench_serving "${bench_args[@]}"
|
||||||
|
else
|
||||||
|
"$PYTHON" -m sglang.bench_serving "${bench_args[@]}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
export -f engine_run_bench
|
||||||
|
export ENGINE_PORT MODEL_PATH RESULT_BASE DOCKER_CLIENT_IMAGE USE_DOCKER_CLIENT
|
||||||
|
export BENCH_DATASET_NAME DATASET_PATH RANDOM_RANGE_RATIO BENCH_WARMUP_MAX_REQUESTS PYTHON
|
||||||
|
|
||||||
|
export SEARCH_START_CONCURRENCY=16
|
||||||
|
export SEARCH_ADDEND=16
|
||||||
|
|
||||||
|
adaptive_main "$@"
|
||||||
@ -77,6 +77,10 @@ adaptive_validate_config() {
|
|||||||
log "ERROR: ENABLE_TTFT_SLO_STOP must be 0 or 1, got ${ENABLE_TTFT_SLO_STOP}"
|
log "ERROR: ENABLE_TTFT_SLO_STOP must be 0 or 1, got ${ENABLE_TTFT_SLO_STOP}"
|
||||||
return 1
|
return 1
|
||||||
fi
|
fi
|
||||||
|
if ! [[ "${TTFT_GROUP_SKIP_MS:-0}" =~ ^[0-9]+$ ]]; then
|
||||||
|
log "ERROR: TTFT_GROUP_SKIP_MS must be a non-negative integer, got ${TTFT_GROUP_SKIP_MS}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
local previous_backoff="$SEARCH_START_CONCURRENCY"
|
local previous_backoff="$SEARCH_START_CONCURRENCY"
|
||||||
local backoff
|
local backoff
|
||||||
for backoff in ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}; do
|
for backoff in ${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-}; do
|
||||||
@ -391,6 +395,7 @@ adaptive_run_shape() {
|
|||||||
|
|
||||||
ADAPTIVE_RESTART_NEEDED=0
|
ADAPTIVE_RESTART_NEEDED=0
|
||||||
ADAPTIVE_FATAL_WORKLOAD=0
|
ADAPTIVE_FATAL_WORKLOAD=0
|
||||||
|
ADAPTIVE_SKIP_CONFIG=0
|
||||||
|
|
||||||
while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do
|
while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do
|
||||||
local num_prompts=$((concurrency * NUM_PROMPTS_MULTIPLIER))
|
local num_prompts=$((concurrency * NUM_PROMPTS_MULTIPLIER))
|
||||||
@ -456,6 +461,15 @@ adaptive_run_shape() {
|
|||||||
plateau_start_concurrency=""
|
plateau_start_concurrency=""
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
if (( concurrency == 1 && ${TTFT_GROUP_SKIP_MS:-0} > 0 )) && \
|
||||||
|
awk -v ttft="$ttft_p95_ms" -v threshold="${TTFT_GROUP_SKIP_MS}" \
|
||||||
|
'BEGIN { exit !(ttft >= threshold) }'; then
|
||||||
|
stop_reason="TTFT_GROUP_SKIP_THRESHOLD"
|
||||||
|
shape_status="TTFT_BOUNDARY"
|
||||||
|
ADAPTIVE_SKIP_CONFIG=1
|
||||||
|
log "probe result c=1 total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms >= group_skip_threshold=${TTFT_GROUP_SKIP_MS}ms; skipping remaining shapes for tp=${tp} dp=${dp}"
|
||||||
|
break
|
||||||
|
fi
|
||||||
stop_reason="TTFT_SLO_EXCEEDED"
|
stop_reason="TTFT_SLO_EXCEEDED"
|
||||||
shape_status="TTFT_BOUNDARY"
|
shape_status="TTFT_BOUNDARY"
|
||||||
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape"
|
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape"
|
||||||
@ -608,6 +622,39 @@ PY
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
adaptive_find_ttft_group_boundary() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local points_jsonl="$3"
|
||||||
|
local threshold_ms="${TTFT_GROUP_SKIP_MS:-0}"
|
||||||
|
if (( threshold_ms == 0 )) || [[ ! -f "$points_jsonl" ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
"$PYTHON" - "$tp" "$dp" "$threshold_ms" "$points_jsonl" <<'PY'
|
||||||
|
import json, sys
|
||||||
|
|
||||||
|
tp, dp = int(sys.argv[1]), int(sys.argv[2])
|
||||||
|
threshold = float(sys.argv[3])
|
||||||
|
best = None
|
||||||
|
with open(sys.argv[4], "r", encoding="utf-8") as f:
|
||||||
|
for line in f:
|
||||||
|
try:
|
||||||
|
point = json.loads(line)
|
||||||
|
if point.get("tp") != tp or point.get("dp") != dp:
|
||||||
|
continue
|
||||||
|
if int(point.get("concurrency", -1)) != 1:
|
||||||
|
continue
|
||||||
|
ttft = float(point["ttft_p95_ms"])
|
||||||
|
except (ValueError, TypeError, KeyError, json.JSONDecodeError):
|
||||||
|
continue
|
||||||
|
if ttft >= threshold and (best is None or ttft > best[0]):
|
||||||
|
best = (ttft, point.get("isl"), point.get("osl"))
|
||||||
|
if best is None:
|
||||||
|
raise SystemExit(1)
|
||||||
|
print(f"isl={best[1]} osl={best[2]} c=1 ttft_p95={best[0]:.3f}ms")
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
adaptive_main() {
|
adaptive_main() {
|
||||||
local resume_mode=0
|
local resume_mode=0
|
||||||
local resume_shapes_jsonl=""
|
local resume_shapes_jsonl=""
|
||||||
@ -683,7 +730,7 @@ adaptive_main() {
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
log "experiment=${EXPERIMENT_NAME} engine=${ENGINE} run_id=${RUN_ID}"
|
log "experiment=${EXPERIMENT_NAME} engine=${ENGINE} run_id=${RUN_ID}"
|
||||||
log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} initial_backoff=[${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-none}] min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}"
|
log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} initial_backoff=[${SEARCH_INITIAL_BACKOFF_CONCURRENCIES:-none}] min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS} ttft_group_skip_ms=${TTFT_GROUP_SKIP_MS:-0}"
|
||||||
log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}"
|
log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}"
|
||||||
|
|
||||||
"$PYTHON" - \
|
"$PYTHON" - \
|
||||||
@ -706,6 +753,7 @@ adaptive_main() {
|
|||||||
"$BENCH_WARMUP_MAX_REQUESTS" \
|
"$BENCH_WARMUP_MAX_REQUESTS" \
|
||||||
"${TTFT_SLO_MS:-4000}" \
|
"${TTFT_SLO_MS:-4000}" \
|
||||||
"${ENABLE_TTFT_SLO_STOP:-1}" \
|
"${ENABLE_TTFT_SLO_STOP:-1}" \
|
||||||
|
"${TTFT_GROUP_SKIP_MS:-0}" \
|
||||||
> "${ADAPTIVE_RUN_ROOT}/run_manifest.json" <<'PY'
|
> "${ADAPTIVE_RUN_ROOT}/run_manifest.json" <<'PY'
|
||||||
import json, sys
|
import json, sys
|
||||||
|
|
||||||
@ -713,7 +761,7 @@ import json, sys
|
|||||||
tokenize_prompt, random_range_ratio, search_start, search_cap,
|
tokenize_prompt, random_range_ratio, search_start, search_cap,
|
||||||
search_multiplier, initial_backoff_concurrencies, prompts_multiplier,
|
search_multiplier, initial_backoff_concurrencies, prompts_multiplier,
|
||||||
min_gain_pct, plateau_patience, warmup_max_requests, ttft_slo_ms,
|
min_gain_pct, plateau_patience, warmup_max_requests, ttft_slo_ms,
|
||||||
enable_ttft_slo_stop) = sys.argv[1:20]
|
enable_ttft_slo_stop, ttft_group_skip_ms) = sys.argv[1:21]
|
||||||
|
|
||||||
print(json.dumps({
|
print(json.dumps({
|
||||||
"experiment": experiment,
|
"experiment": experiment,
|
||||||
@ -738,6 +786,7 @@ print(json.dumps({
|
|||||||
"warmup_max_requests": int(warmup_max_requests),
|
"warmup_max_requests": int(warmup_max_requests),
|
||||||
"ttft_slo_ms": float(ttft_slo_ms),
|
"ttft_slo_ms": float(ttft_slo_ms),
|
||||||
"enable_ttft_slo_stop": int(enable_ttft_slo_stop),
|
"enable_ttft_slo_stop": int(enable_ttft_slo_stop),
|
||||||
|
"ttft_group_skip_ms": float(ttft_group_skip_ms),
|
||||||
},
|
},
|
||||||
}, indent=2, ensure_ascii=False))
|
}, indent=2, ensure_ascii=False))
|
||||||
PY
|
PY
|
||||||
@ -755,6 +804,12 @@ PY
|
|||||||
if ! adaptive_matches_filter "$tp" "$TP_LIST"; then
|
if ! adaptive_matches_filter "$tp" "$TP_LIST"; then
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
local prior_ttft_boundary=""
|
||||||
|
if (( resume_mode == 1 )) && \
|
||||||
|
prior_ttft_boundary="$(adaptive_find_ttft_group_boundary "$tp" "$dp" "$ADAPTIVE_POINTS_JSONL")"; then
|
||||||
|
log "RESUME skip remaining config tp=${tp} dp=${dp}: prior ${prior_ttft_boundary} >= group_skip_threshold=${TTFT_GROUP_SKIP_MS}ms"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
config_root="${ADAPTIVE_RUN_ROOT}/tp${tp}_dp${dp}"
|
config_root="${ADAPTIVE_RUN_ROOT}/tp${tp}_dp${dp}"
|
||||||
mkdir -p "${config_root}/raw_outputs" "${config_root}/metrics" \
|
mkdir -p "${config_root}/raw_outputs" "${config_root}/metrics" \
|
||||||
"${config_root}/logs" "${config_root}/gpu_logs"
|
"${config_root}/logs" "${config_root}/gpu_logs"
|
||||||
@ -816,6 +871,10 @@ PY
|
|||||||
abort_all=1
|
abort_all=1
|
||||||
break
|
break
|
||||||
fi
|
fi
|
||||||
|
if (( ADAPTIVE_SKIP_CONFIG == 1 )); then
|
||||||
|
log "TTFT group-skip threshold reached; skipping remaining shapes for tp=${tp} dp=${dp}"
|
||||||
|
break
|
||||||
|
fi
|
||||||
if (( ADAPTIVE_RESTART_NEEDED == 1 )); then
|
if (( ADAPTIVE_RESTART_NEEDED == 1 )); then
|
||||||
if ! adaptive_restart_server "$tp" "$dp"; then
|
if ! adaptive_restart_server "$tp" "$dp"; then
|
||||||
log "ERROR: server restart failed; skipping remaining shapes for tp=${tp} dp=${dp}"
|
log "ERROR: server restart failed; skipping remaining shapes for tp=${tp} dp=${dp}"
|
||||||
|
|||||||
778
scripts/common/adaptive_bench_lib.sh.bak.20260716-084832
Executable file
778
scripts/common/adaptive_bench_lib.sh.bak.20260716-084832
Executable file
@ -0,0 +1,778 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Shared adaptive-concurrency search loop for serving benchmarks.
|
||||||
|
|
||||||
|
ADAPTIVE_COMMON_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
ADAPTIVE_HELPER="${ADAPTIVE_COMMON_DIR}/adaptive_concurrency.py"
|
||||||
|
|
||||||
|
adaptive_timestamp() {
|
||||||
|
date --iso-8601=seconds
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_json_number_or_null() {
|
||||||
|
local value="${1:-}"
|
||||||
|
if [[ -n "$value" ]]; then
|
||||||
|
printf '%s' "$value"
|
||||||
|
else
|
||||||
|
printf 'null'
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_warmup_request_count() {
|
||||||
|
local concurrency="$1"
|
||||||
|
if (( BENCH_WARMUP_MAX_REQUESTS > 0 && concurrency > BENCH_WARMUP_MAX_REQUESTS )); then
|
||||||
|
printf '%s' "$BENCH_WARMUP_MAX_REQUESTS"
|
||||||
|
else
|
||||||
|
printf '%s' "$concurrency"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
export -f adaptive_warmup_request_count
|
||||||
|
|
||||||
|
adaptive_validate_config() {
|
||||||
|
local name
|
||||||
|
for name in SEARCH_START_CONCURRENCY SEARCH_MAX_CONCURRENCY \
|
||||||
|
NUM_PROMPTS_MULTIPLIER PLATEAU_PATIENCE MAX_POINT_RETRIES \
|
||||||
|
BENCH_WARMUP_MAX_REQUESTS; do
|
||||||
|
if ! [[ "${!name}" =~ ^[0-9]+$ ]]; then
|
||||||
|
log "ERROR: ${name} must be a non-negative integer, got ${!name}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
if (( SEARCH_START_CONCURRENCY < 1 )); then
|
||||||
|
log "ERROR: SEARCH_START_CONCURRENCY must be at least 1"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if (( SEARCH_MAX_CONCURRENCY < SEARCH_START_CONCURRENCY )); then
|
||||||
|
log "ERROR: SEARCH_MAX_CONCURRENCY must be >= SEARCH_START_CONCURRENCY"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ -n "${SEARCH_ADDEND:-}" ]]; then
|
||||||
|
if ! [[ "${SEARCH_ADDEND}" =~ ^[0-9]+$ ]] || (( SEARCH_ADDEND < 1 )); then
|
||||||
|
log "ERROR: SEARCH_ADDEND must be at least 1"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
if ! [[ "${SEARCH_MULTIPLIER:-}" =~ ^[0-9]+$ ]] || (( SEARCH_MULTIPLIER < 2 )); then
|
||||||
|
log "ERROR: SEARCH_MULTIPLIER must be at least 2"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if (( NUM_PROMPTS_MULTIPLIER < 1 || PLATEAU_PATIENCE < 1 )); then
|
||||||
|
log "ERROR: NUM_PROMPTS_MULTIPLIER and PLATEAU_PATIENCE must be at least 1"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if ! [[ "$TPS_MIN_GAIN_PCT" =~ ^[0-9]+([.][0-9]+)?$ ]]; then
|
||||||
|
log "ERROR: TPS_MIN_GAIN_PCT must be a non-negative number"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if ! [[ "$RANDOM_RANGE_RATIO" =~ ^[0-9]+([.][0-9]+)?$ ]] || \
|
||||||
|
! awk -v ratio="$RANDOM_RANGE_RATIO" 'BEGIN { exit !(ratio >= 0 && ratio <= 1) }'; then
|
||||||
|
log "ERROR: RANDOM_RANGE_RATIO must be between 0.0 and 1.0"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if ! [[ "${TTFT_SLO_MS:-4000}" =~ ^[0-9]+$ ]] || (( TTFT_SLO_MS < 1 )); then
|
||||||
|
log "ERROR: TTFT_SLO_MS must be a positive integer, got ${TTFT_SLO_MS}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if [[ "${ENABLE_TTFT_SLO_STOP:-1}" != "0" && "${ENABLE_TTFT_SLO_STOP:-1}" != "1" ]]; then
|
||||||
|
log "ERROR: ENABLE_TTFT_SLO_STOP must be 0 or 1, got ${ENABLE_TTFT_SLO_STOP}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_start_gpu_monitor() {
|
||||||
|
local csv_path="$1"
|
||||||
|
mkdir -p "$(dirname "$csv_path")"
|
||||||
|
nvidia-smi \
|
||||||
|
--query-gpu=timestamp,index,memory.used,memory.total,utilization.gpu \
|
||||||
|
--format=csv \
|
||||||
|
-l "$GPU_MEM_SAMPLE_INTERVAL_S" \
|
||||||
|
> "$csv_path" 2>/dev/null &
|
||||||
|
echo $!
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_stop_gpu_monitor() {
|
||||||
|
local pid="$1"
|
||||||
|
if kill -0 "$pid" 2>/dev/null; then
|
||||||
|
kill "$pid" 2>/dev/null || true
|
||||||
|
sleep 1
|
||||||
|
kill -9 "$pid" 2>/dev/null || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_warmup() {
|
||||||
|
local log_path="$1"
|
||||||
|
log "running synthetic warmup"
|
||||||
|
timeout "$SCENARIO_TIMEOUT_S" bash -c \
|
||||||
|
'engine_run_bench 1024 128 1 1 /dev/null' \
|
||||||
|
>> "$log_path" 2>&1
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_restart_server() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
log "restarting ${ENGINE} server tp=${tp} dp=${dp}"
|
||||||
|
engine_stop_server "$tp" "$dp"
|
||||||
|
sleep "$SERVER_RESTART_COOLDOWN_S"
|
||||||
|
engine_start_server "$tp" "$dp"
|
||||||
|
adaptive_warmup "${ADAPTIVE_LOG_DIR}/${ENGINE}_tp${tp}_dp${dp}.warmup.log"
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_append_point_from_metrics() {
|
||||||
|
local metrics_file="$1"
|
||||||
|
local tp="$2"
|
||||||
|
local dp="$3"
|
||||||
|
local mark="$4"
|
||||||
|
local isl="$5"
|
||||||
|
local osl="$6"
|
||||||
|
local concurrency="$7"
|
||||||
|
local num_prompts="$8"
|
||||||
|
local attempt="$9"
|
||||||
|
local gain_pct="${10:-}"
|
||||||
|
local plateau_streak="${11:-0}"
|
||||||
|
local raw_file="${12}"
|
||||||
|
local detail_log="${13}"
|
||||||
|
local gain_json
|
||||||
|
gain_json="$(adaptive_json_number_or_null "$gain_pct")"
|
||||||
|
|
||||||
|
jq -c \
|
||||||
|
--arg timestamp "$(adaptive_timestamp)" \
|
||||||
|
--arg engine "$ENGINE" \
|
||||||
|
--argjson tp "$tp" \
|
||||||
|
--argjson dp "$dp" \
|
||||||
|
--arg mark "$mark" \
|
||||||
|
--argjson isl "$isl" \
|
||||||
|
--argjson osl "$osl" \
|
||||||
|
--argjson concurrency "$concurrency" \
|
||||||
|
--argjson num_prompts "$num_prompts" \
|
||||||
|
--argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \
|
||||||
|
--argjson attempt "$attempt" \
|
||||||
|
--argjson gain_pct "$gain_json" \
|
||||||
|
--argjson plateau_streak "$plateau_streak" \
|
||||||
|
--arg raw_file "$raw_file" \
|
||||||
|
--arg detail_log "$detail_log" \
|
||||||
|
'. + {
|
||||||
|
timestamp: $timestamp,
|
||||||
|
engine: $engine,
|
||||||
|
tp: $tp,
|
||||||
|
dp: $dp,
|
||||||
|
mark: $mark,
|
||||||
|
isl: $isl,
|
||||||
|
osl: $osl,
|
||||||
|
concurrency: $concurrency,
|
||||||
|
num_prompts: $num_prompts,
|
||||||
|
warmup_requests: $warmup_requests,
|
||||||
|
attempt: $attempt,
|
||||||
|
gain_pct: $gain_pct,
|
||||||
|
plateau_streak: $plateau_streak,
|
||||||
|
error_type: "",
|
||||||
|
raw_file: $raw_file,
|
||||||
|
detail_log: $detail_log
|
||||||
|
}' "$metrics_file" >> "$ADAPTIVE_POINTS_JSONL"
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_append_failed_point() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local mark="$3"
|
||||||
|
local isl="$4"
|
||||||
|
local osl="$5"
|
||||||
|
local concurrency="$6"
|
||||||
|
local num_prompts="$7"
|
||||||
|
local attempt="$8"
|
||||||
|
local status="$9"
|
||||||
|
local error_type="${10}"
|
||||||
|
local raw_file="${11}"
|
||||||
|
local detail_log="${12}"
|
||||||
|
|
||||||
|
jq -nc \
|
||||||
|
--arg timestamp "$(adaptive_timestamp)" \
|
||||||
|
--arg engine "$ENGINE" \
|
||||||
|
--argjson tp "$tp" \
|
||||||
|
--argjson dp "$dp" \
|
||||||
|
--arg mark "$mark" \
|
||||||
|
--argjson isl "$isl" \
|
||||||
|
--argjson osl "$osl" \
|
||||||
|
--argjson concurrency "$concurrency" \
|
||||||
|
--argjson num_prompts "$num_prompts" \
|
||||||
|
--argjson warmup_requests "$(adaptive_warmup_request_count "$concurrency")" \
|
||||||
|
--argjson attempt "$attempt" \
|
||||||
|
--arg status "$status" \
|
||||||
|
--arg error_type "$error_type" \
|
||||||
|
--arg raw_file "$raw_file" \
|
||||||
|
--arg detail_log "$detail_log" \
|
||||||
|
'{
|
||||||
|
timestamp: $timestamp,
|
||||||
|
engine: $engine,
|
||||||
|
tp: $tp,
|
||||||
|
dp: $dp,
|
||||||
|
mark: $mark,
|
||||||
|
isl: $isl,
|
||||||
|
osl: $osl,
|
||||||
|
concurrency: $concurrency,
|
||||||
|
num_prompts: $num_prompts,
|
||||||
|
warmup_requests: $warmup_requests,
|
||||||
|
attempt: $attempt,
|
||||||
|
status: $status,
|
||||||
|
completed: 0,
|
||||||
|
failed: $num_prompts,
|
||||||
|
error_type: $error_type,
|
||||||
|
validation_errors: [],
|
||||||
|
gain_pct: null,
|
||||||
|
plateau_streak: 0,
|
||||||
|
raw_file: $raw_file,
|
||||||
|
detail_log: $detail_log
|
||||||
|
}' >> "$ADAPTIVE_POINTS_JSONL"
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_append_shape_summary() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local mark="$3"
|
||||||
|
local isl="$4"
|
||||||
|
local osl="$5"
|
||||||
|
local status="$6"
|
||||||
|
local stop_reason="$7"
|
||||||
|
local tested_points="$8"
|
||||||
|
local max_successful_concurrency="$9"
|
||||||
|
local saturation_concurrency="${10:-}"
|
||||||
|
local stop_probe_concurrency="${11:-}"
|
||||||
|
local best_tps_concurrency="${12:-}"
|
||||||
|
local best_total_tps="${13:-}"
|
||||||
|
local last_total_tps="${14:-}"
|
||||||
|
local saturation_json stop_probe_json best_c_json best_tps_json last_tps_json
|
||||||
|
saturation_json="$(adaptive_json_number_or_null "$saturation_concurrency")"
|
||||||
|
stop_probe_json="$(adaptive_json_number_or_null "$stop_probe_concurrency")"
|
||||||
|
best_c_json="$(adaptive_json_number_or_null "$best_tps_concurrency")"
|
||||||
|
best_tps_json="$(adaptive_json_number_or_null "$best_total_tps")"
|
||||||
|
last_tps_json="$(adaptive_json_number_or_null "$last_total_tps")"
|
||||||
|
|
||||||
|
jq -nc \
|
||||||
|
--arg timestamp "$(adaptive_timestamp)" \
|
||||||
|
--arg engine "$ENGINE" \
|
||||||
|
--argjson tp "$tp" \
|
||||||
|
--argjson dp "$dp" \
|
||||||
|
--arg mark "$mark" \
|
||||||
|
--argjson isl "$isl" \
|
||||||
|
--argjson osl "$osl" \
|
||||||
|
--arg status "$status" \
|
||||||
|
--arg stop_reason "$stop_reason" \
|
||||||
|
--argjson tested_points "$tested_points" \
|
||||||
|
--argjson search_cap "$SEARCH_MAX_CONCURRENCY" \
|
||||||
|
--argjson max_successful_concurrency "$max_successful_concurrency" \
|
||||||
|
--argjson saturation_concurrency "$saturation_json" \
|
||||||
|
--argjson stop_probe_concurrency "$stop_probe_json" \
|
||||||
|
--argjson best_tps_concurrency "$best_c_json" \
|
||||||
|
--argjson best_total_tps "$best_tps_json" \
|
||||||
|
--argjson last_total_tps "$last_tps_json" \
|
||||||
|
'{
|
||||||
|
timestamp: $timestamp,
|
||||||
|
engine: $engine,
|
||||||
|
tp: $tp,
|
||||||
|
dp: $dp,
|
||||||
|
mark: $mark,
|
||||||
|
isl: $isl,
|
||||||
|
osl: $osl,
|
||||||
|
status: $status,
|
||||||
|
stop_reason: $stop_reason,
|
||||||
|
tested_points: $tested_points,
|
||||||
|
search_cap: $search_cap,
|
||||||
|
max_successful_concurrency: $max_successful_concurrency,
|
||||||
|
saturation_concurrency: $saturation_concurrency,
|
||||||
|
stop_probe_concurrency: $stop_probe_concurrency,
|
||||||
|
best_tps_concurrency: $best_tps_concurrency,
|
||||||
|
best_total_tps: $best_total_tps,
|
||||||
|
last_total_tps: $last_total_tps
|
||||||
|
}' >> "$ADAPTIVE_SHAPES_JSONL"
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_run_point() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local isl="$3"
|
||||||
|
local osl="$4"
|
||||||
|
local concurrency="$5"
|
||||||
|
local num_prompts="$6"
|
||||||
|
local config_root="$7"
|
||||||
|
local attempt bench_rc parse_rc gpu_pid
|
||||||
|
|
||||||
|
POINT_RAW_FILE=""
|
||||||
|
POINT_DETAIL_LOG=""
|
||||||
|
POINT_METRICS_FILE=""
|
||||||
|
POINT_ATTEMPT=0
|
||||||
|
POINT_ERROR_TYPE=""
|
||||||
|
|
||||||
|
for (( attempt = 1; attempt <= MAX_POINT_RETRIES + 1; attempt++ )); do
|
||||||
|
POINT_ATTEMPT="$attempt"
|
||||||
|
POINT_RAW_FILE="${config_root}/raw_outputs/${ENGINE}_adaptive_c${concurrency}_i${isl}_o${osl}_a${attempt}.jsonl"
|
||||||
|
POINT_DETAIL_LOG="${config_root}/logs/${ENGINE}_c${concurrency}_i${isl}_o${osl}_a${attempt}.log"
|
||||||
|
POINT_METRICS_FILE="${config_root}/metrics/${ENGINE}_c${concurrency}_i${isl}_o${osl}_a${attempt}.json"
|
||||||
|
local gpu_csv="${config_root}/gpu_logs/gpu_c${concurrency}_i${isl}_o${osl}_a${attempt}.csv"
|
||||||
|
|
||||||
|
log "probe ${ENGINE} tp=${tp} dp=${dp} isl=${isl} osl=${osl} c=${concurrency} warmup=$(adaptive_warmup_request_count "$concurrency") n=${num_prompts} attempt=${attempt}"
|
||||||
|
gpu_pid="$(adaptive_start_gpu_monitor "$gpu_csv")"
|
||||||
|
bench_rc=0
|
||||||
|
timeout "$SCENARIO_TIMEOUT_S" bash -c \
|
||||||
|
'engine_run_bench "$@"' _ \
|
||||||
|
"$isl" "$osl" "$concurrency" "$num_prompts" "$POINT_RAW_FILE" \
|
||||||
|
> "$POINT_DETAIL_LOG" 2>&1 || bench_rc=$?
|
||||||
|
adaptive_stop_gpu_monitor "$gpu_pid"
|
||||||
|
|
||||||
|
if (( bench_rc == 0 )); then
|
||||||
|
parse_rc=0
|
||||||
|
"$PYTHON" "$ADAPTIVE_HELPER" parse-result \
|
||||||
|
--input "$POINT_RAW_FILE" \
|
||||||
|
--output "$POINT_METRICS_FILE" \
|
||||||
|
--expected-prompts "$num_prompts" \
|
||||||
|
--isl "$isl" \
|
||||||
|
--osl "$osl" \
|
||||||
|
--input-tolerance-pct "$INPUT_LENGTH_TOLERANCE_PCT" \
|
||||||
|
--output-tolerance-pct "$OUTPUT_LENGTH_TOLERANCE_PCT" \
|
||||||
|
>> "$POINT_DETAIL_LOG" 2>&1 || parse_rc=$?
|
||||||
|
if (( parse_rc == 0 )); then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
POINT_ERROR_TYPE="workload validation failed"
|
||||||
|
return 21
|
||||||
|
fi
|
||||||
|
|
||||||
|
if engine_detect_oom "$POINT_DETAIL_LOG" "$tp" "$dp"; then
|
||||||
|
POINT_ERROR_TYPE="detected out of memory"
|
||||||
|
return 20
|
||||||
|
fi
|
||||||
|
|
||||||
|
POINT_ERROR_TYPE="benchmark rc=${bench_rc}"
|
||||||
|
if (( attempt <= MAX_POINT_RETRIES )); then
|
||||||
|
log "probe failed without OOM; restarting server before retry"
|
||||||
|
if ! adaptive_restart_server "$tp" "$dp"; then
|
||||||
|
POINT_ERROR_TYPE="benchmark failed and server restart failed"
|
||||||
|
return 22
|
||||||
|
fi
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
return 22
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_run_shape() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local mark="$3"
|
||||||
|
local isl="$4"
|
||||||
|
local osl="$5"
|
||||||
|
local config_root="$6"
|
||||||
|
local concurrency="$SEARCH_START_CONCURRENCY"
|
||||||
|
local previous_tps=""
|
||||||
|
local current_tps=""
|
||||||
|
local best_tps=""
|
||||||
|
local best_concurrency=""
|
||||||
|
local max_successful_concurrency=0
|
||||||
|
local last_total_tps=""
|
||||||
|
local plateau_streak=0
|
||||||
|
local plateau_start_concurrency=""
|
||||||
|
local saturation_concurrency=""
|
||||||
|
local stop_probe_concurrency=""
|
||||||
|
local stop_reason="SEARCH_CAP_REACHED"
|
||||||
|
local shape_status="COMPLETED"
|
||||||
|
local tested_points=0
|
||||||
|
local gain_pct=""
|
||||||
|
local meaningful_gain=1
|
||||||
|
local point_rc=0
|
||||||
|
|
||||||
|
ADAPTIVE_RESTART_NEEDED=0
|
||||||
|
ADAPTIVE_FATAL_WORKLOAD=0
|
||||||
|
|
||||||
|
while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do
|
||||||
|
local num_prompts=$((concurrency * NUM_PROMPTS_MULTIPLIER))
|
||||||
|
point_rc=0
|
||||||
|
adaptive_run_point "$tp" "$dp" "$isl" "$osl" "$concurrency" "$num_prompts" "$config_root" || point_rc=$?
|
||||||
|
tested_points=$((tested_points + 1))
|
||||||
|
stop_probe_concurrency="$concurrency"
|
||||||
|
|
||||||
|
if (( point_rc == 0 )); then
|
||||||
|
current_tps="$(jq -r '.total_tps' "$POINT_METRICS_FILE")"
|
||||||
|
gain_pct=""
|
||||||
|
meaningful_gain=1
|
||||||
|
if [[ -n "$previous_tps" ]]; then
|
||||||
|
IFS=$'\t' read -r gain_pct meaningful_gain < <(
|
||||||
|
"$PYTHON" "$ADAPTIVE_HELPER" gain \
|
||||||
|
--previous "$previous_tps" \
|
||||||
|
--current "$current_tps" \
|
||||||
|
--threshold-pct "$TPS_MIN_GAIN_PCT"
|
||||||
|
)
|
||||||
|
if (( meaningful_gain == 1 )); then
|
||||||
|
plateau_streak=0
|
||||||
|
plateau_start_concurrency=""
|
||||||
|
else
|
||||||
|
plateau_streak=$((plateau_streak + 1))
|
||||||
|
if [[ -z "$plateau_start_concurrency" ]]; then
|
||||||
|
plateau_start_concurrency="$concurrency"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$best_tps" ]] || awk -v current="$current_tps" -v best="$best_tps" 'BEGIN { exit !(current > best) }'; then
|
||||||
|
best_tps="$current_tps"
|
||||||
|
best_concurrency="$concurrency"
|
||||||
|
fi
|
||||||
|
max_successful_concurrency="$concurrency"
|
||||||
|
last_total_tps="$current_tps"
|
||||||
|
|
||||||
|
# TTFT SLO check: stop searching if TTFT P95 exceeds the SLO threshold
|
||||||
|
local ttft_p95_ms=""
|
||||||
|
if [[ "${ENABLE_TTFT_SLO_STOP:-1}" == "1" ]]; then
|
||||||
|
ttft_p95_ms="$(jq -r '.ttft_p95_ms' "$POINT_METRICS_FILE")"
|
||||||
|
if awk -v ttft="$ttft_p95_ms" -v slo="${TTFT_SLO_MS:-4000}" 'BEGIN { exit !(ttft > slo) }'; then
|
||||||
|
adaptive_append_point_from_metrics \
|
||||||
|
"$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \
|
||||||
|
"$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \
|
||||||
|
"$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG"
|
||||||
|
stop_reason="TTFT_SLO_EXCEEDED"
|
||||||
|
shape_status="TTFT_BOUNDARY"
|
||||||
|
log "probe result c=${concurrency} total_tps=${current_tps} ttft_p95=${ttft_p95_ms}ms > SLO=${TTFT_SLO_MS}ms, stopping shape"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
adaptive_append_point_from_metrics \
|
||||||
|
"$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \
|
||||||
|
"$concurrency" "$num_prompts" "$POINT_ATTEMPT" "$gain_pct" \
|
||||||
|
"$plateau_streak" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG"
|
||||||
|
|
||||||
|
log "probe result c=${concurrency} total_tps=${current_tps} gain_pct=${gain_pct:-first} plateau_streak=${plateau_streak}/${PLATEAU_PATIENCE}"
|
||||||
|
if (( plateau_streak >= PLATEAU_PATIENCE )); then
|
||||||
|
saturation_concurrency="$plateau_start_concurrency"
|
||||||
|
stop_reason="TPS_PLATEAU"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
previous_tps="$current_tps"
|
||||||
|
elif (( point_rc == 20 )); then
|
||||||
|
adaptive_append_failed_point \
|
||||||
|
"$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \
|
||||||
|
"$POINT_ATTEMPT" "OOM" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG"
|
||||||
|
shape_status="OOM_BOUNDARY"
|
||||||
|
stop_reason="OOM"
|
||||||
|
ADAPTIVE_RESTART_NEEDED=1
|
||||||
|
break
|
||||||
|
elif (( point_rc == 21 )); then
|
||||||
|
adaptive_append_point_from_metrics \
|
||||||
|
"$POINT_METRICS_FILE" "$tp" "$dp" "$mark" "$isl" "$osl" \
|
||||||
|
"$concurrency" "$num_prompts" "$POINT_ATTEMPT" "" 0 \
|
||||||
|
"$POINT_RAW_FILE" "$POINT_DETAIL_LOG"
|
||||||
|
shape_status="INVALID_WORKLOAD"
|
||||||
|
stop_reason="INVALID_WORKLOAD"
|
||||||
|
ADAPTIVE_FATAL_WORKLOAD=1
|
||||||
|
break
|
||||||
|
else
|
||||||
|
adaptive_append_failed_point \
|
||||||
|
"$tp" "$dp" "$mark" "$isl" "$osl" "$concurrency" "$num_prompts" \
|
||||||
|
"$POINT_ATTEMPT" "FAILED" "$POINT_ERROR_TYPE" "$POINT_RAW_FILE" "$POINT_DETAIL_LOG"
|
||||||
|
shape_status="FAILED"
|
||||||
|
stop_reason="BENCHMARK_FAILED"
|
||||||
|
ADAPTIVE_RESTART_NEEDED=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
|
||||||
|
if (( concurrency == SEARCH_MAX_CONCURRENCY )); then
|
||||||
|
stop_reason="SEARCH_CAP_REACHED"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
local next_concurrency
|
||||||
|
if [[ -n "${SEARCH_ADDEND:-}" ]]; then
|
||||||
|
next_concurrency=$((concurrency + SEARCH_ADDEND))
|
||||||
|
else
|
||||||
|
next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
|
||||||
|
fi
|
||||||
|
if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then
|
||||||
|
next_concurrency="$SEARCH_MAX_CONCURRENCY"
|
||||||
|
fi
|
||||||
|
concurrency="$next_concurrency"
|
||||||
|
done
|
||||||
|
|
||||||
|
adaptive_append_shape_summary \
|
||||||
|
"$tp" "$dp" "$mark" "$isl" "$osl" "$shape_status" "$stop_reason" \
|
||||||
|
"$tested_points" "$max_successful_concurrency" "$saturation_concurrency" \
|
||||||
|
"$stop_probe_concurrency" "$best_concurrency" "$best_tps" "$last_total_tps"
|
||||||
|
|
||||||
|
log "shape done tp=${tp} dp=${dp} isl=${isl} osl=${osl} stop=${stop_reason} saturation_c=${saturation_concurrency:-none} best_c=${best_concurrency:-none} best_total_tps=${best_tps:-none}"
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_matches_filter() {
|
||||||
|
local value="$1"
|
||||||
|
local filter="${2:-}"
|
||||||
|
[[ -z "$filter" || " $filter " == *" $value "* ]]
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_print_sequence() {
|
||||||
|
local concurrency="$SEARCH_START_CONCURRENCY"
|
||||||
|
local -a sequence=()
|
||||||
|
while (( concurrency <= SEARCH_MAX_CONCURRENCY )); do
|
||||||
|
sequence+=("$concurrency")
|
||||||
|
if (( concurrency == SEARCH_MAX_CONCURRENCY )); then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
local next_concurrency
|
||||||
|
if [[ -n "${SEARCH_ADDEND:-}" ]]; then
|
||||||
|
next_concurrency=$((concurrency + SEARCH_ADDEND))
|
||||||
|
else
|
||||||
|
next_concurrency=$((concurrency * SEARCH_MULTIPLIER))
|
||||||
|
fi
|
||||||
|
if (( next_concurrency > SEARCH_MAX_CONCURRENCY )); then
|
||||||
|
next_concurrency="$SEARCH_MAX_CONCURRENCY"
|
||||||
|
fi
|
||||||
|
concurrency="$next_concurrency"
|
||||||
|
done
|
||||||
|
printf '%s ' "${sequence[@]}"
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_cleanup_active_server() {
|
||||||
|
if [[ -n "${ADAPTIVE_ACTIVE_TP:-}" && -n "${ADAPTIVE_ACTIVE_DP:-}" ]]; then
|
||||||
|
engine_stop_server "$ADAPTIVE_ACTIVE_TP" "$ADAPTIVE_ACTIVE_DP" || true
|
||||||
|
ADAPTIVE_ACTIVE_TP=""
|
||||||
|
ADAPTIVE_ACTIVE_DP=""
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_shape_already_tested() {
|
||||||
|
local tp="$1"
|
||||||
|
local dp="$2"
|
||||||
|
local mark="$3"
|
||||||
|
local isl="$4"
|
||||||
|
local osl="$5"
|
||||||
|
local resume_shapes_jsonl="$6"
|
||||||
|
if [[ ! -f "$resume_shapes_jsonl" ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
local count
|
||||||
|
count="$(jq -s -r --argjson tp "$tp" --argjson dp "$dp" --arg mark "$mark" --argjson isl "$isl" --argjson osl "$osl" '
|
||||||
|
[ .[] | select(.tp == $tp and .dp == $dp and .mark == $mark and .isl == $isl and .osl == $osl) ] | length
|
||||||
|
' "$resume_shapes_jsonl" 2>/dev/null)"
|
||||||
|
if [[ -z "$count" ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
if (( count > 0 )); then
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
adaptive_main() {
|
||||||
|
local resume_mode=0
|
||||||
|
local resume_shapes_jsonl=""
|
||||||
|
if [[ -n "${RESUME_RUN_ID:-}" ]]; then
|
||||||
|
ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RESUME_RUN_ID}"
|
||||||
|
if [[ ! -d "$ADAPTIVE_RUN_ROOT" ]]; then
|
||||||
|
log "ERROR: RESUME_RUN_ID=${RESUME_RUN_ID} does not exist under ${RESULT_BASE}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
resume_mode=1
|
||||||
|
RUN_ID="$RESUME_RUN_ID"
|
||||||
|
resume_shapes_jsonl="${ADAPTIVE_RUN_ROOT}/adaptive_shapes.jsonl"
|
||||||
|
if [[ ! -f "$resume_shapes_jsonl" ]]; then
|
||||||
|
log "WARNING: no adaptive_shapes.jsonl found in ${ADAPTIVE_RUN_ROOT}; will resume with empty history"
|
||||||
|
fi
|
||||||
|
log "RESUME mode: continuing run_id=${RESUME_RUN_ID}"
|
||||||
|
else
|
||||||
|
RUN_ID="${RUN_ID:-adaptive_$(date '+%Y%m%d-%H%M%S')}"
|
||||||
|
ADAPTIVE_RUN_ROOT="${RESULT_BASE}/${RUN_ID}"
|
||||||
|
fi
|
||||||
|
RESULT_BASE="${RESULT_BASE:-${SCRIPT_DIR}/adaptive_results}"
|
||||||
|
MATRIX_FILE="${MATRIX_FILE:-${SCRIPT_DIR}/matrix.json}"
|
||||||
|
MATRIX_MODE="${MATRIX_MODE:-Y}"
|
||||||
|
ADAPTIVE_LOG_DIR="${ADAPTIVE_RUN_ROOT}/logs"
|
||||||
|
ADAPTIVE_POINTS_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_points.jsonl"
|
||||||
|
ADAPTIVE_SHAPES_JSONL="${ADAPTIVE_RUN_ROOT}/adaptive_shapes.jsonl"
|
||||||
|
local shapes_tsv="${ADAPTIVE_RUN_ROOT}/shapes.tsv"
|
||||||
|
mkdir -p "$ADAPTIVE_LOG_DIR"
|
||||||
|
if (( resume_mode == 0 )); then
|
||||||
|
: > "$ADAPTIVE_POINTS_JSONL"
|
||||||
|
: > "$ADAPTIVE_SHAPES_JSONL"
|
||||||
|
fi
|
||||||
|
log_init "${ADAPTIVE_LOG_DIR}/orchestrator.log"
|
||||||
|
|
||||||
|
adaptive_validate_config
|
||||||
|
"$PYTHON" "$ADAPTIVE_HELPER" shapes \
|
||||||
|
--matrix "$MATRIX_FILE" \
|
||||||
|
--mode "$MATRIX_MODE" \
|
||||||
|
> "$shapes_tsv"
|
||||||
|
|
||||||
|
local tokenize_prompt_json=true
|
||||||
|
if [[ "$BENCH_DATASET_NAME" == "random" ]]; then
|
||||||
|
local required_dataset_rows=$((SEARCH_MAX_CONCURRENCY * NUM_PROMPTS_MULTIPLIER))
|
||||||
|
local dataset_capacity=0
|
||||||
|
local dataset_error=""
|
||||||
|
if [[ ! -f "$DATASET_PATH" ]]; then
|
||||||
|
dataset_error="DATASET_PATH does not exist: ${DATASET_PATH}"
|
||||||
|
elif ! dataset_capacity="$(
|
||||||
|
"$PYTHON" "$ADAPTIVE_HELPER" dataset-capacity --path "$DATASET_PATH" 2>/dev/null
|
||||||
|
)"; then
|
||||||
|
dataset_error="DATASET_PATH is not a valid ShareGPT JSON file: ${DATASET_PATH}"
|
||||||
|
fi
|
||||||
|
if [[ -n "$dataset_error" ]]; then
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
log "WARNING: ${dataset_error}"
|
||||||
|
else
|
||||||
|
log "ERROR: ${dataset_error}"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if [[ -z "$dataset_error" ]] && (( dataset_capacity < required_dataset_rows )); then
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
log "WARNING: DATASET_PATH has ${dataset_capacity} usable conversations, but a real run may request ${required_dataset_rows}."
|
||||||
|
else
|
||||||
|
log "ERROR: DATASET_PATH has ${dataset_capacity} usable conversations, but adaptive search may request ${required_dataset_rows}. Set DATASET_PATH to a larger ShareGPT file or explicitly use BENCH_DATASET_NAME=random-ids."
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
tokenize_prompt_json=false
|
||||||
|
elif [[ "$BENCH_DATASET_NAME" != random* ]]; then
|
||||||
|
log "ERROR: BENCH_DATASET_NAME must be random or start with random"
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
log "experiment=${EXPERIMENT_NAME} engine=${ENGINE} run_id=${RUN_ID}"
|
||||||
|
log "search_start=${SEARCH_START_CONCURRENCY} search_cap=${SEARCH_MAX_CONCURRENCY} multiplier=${SEARCH_MULTIPLIER} min_gain_pct=${TPS_MIN_GAIN_PCT} patience=${PLATEAU_PATIENCE} prompts_multiplier=${NUM_PROMPTS_MULTIPLIER} warmup_cap=${BENCH_WARMUP_MAX_REQUESTS}"
|
||||||
|
log "dataset=${BENCH_DATASET_NAME} random_range_ratio=${RANDOM_RANGE_RATIO} tokenize_prompt=${tokenize_prompt_json} matrix=${MATRIX_FILE} mode=${MATRIX_MODE}"
|
||||||
|
|
||||||
|
jq -n \
|
||||||
|
--arg experiment "$EXPERIMENT_NAME" \
|
||||||
|
--arg engine "$ENGINE" \
|
||||||
|
--arg run_id "$RUN_ID" \
|
||||||
|
--arg model "$MODEL_PATH" \
|
||||||
|
--arg hardware "$HARDWARE" \
|
||||||
|
--arg matrix "$MATRIX_FILE" \
|
||||||
|
--arg dataset "$BENCH_DATASET_NAME" \
|
||||||
|
--argjson tokenize_prompt "$tokenize_prompt_json" \
|
||||||
|
--argjson random_range_ratio "$RANDOM_RANGE_RATIO" \
|
||||||
|
--argjson search_start "$SEARCH_START_CONCURRENCY" \
|
||||||
|
--argjson search_cap "$SEARCH_MAX_CONCURRENCY" \
|
||||||
|
--argjson search_multiplier "$SEARCH_MULTIPLIER" \
|
||||||
|
--argjson prompts_multiplier "$NUM_PROMPTS_MULTIPLIER" \
|
||||||
|
--argjson min_gain_pct "$TPS_MIN_GAIN_PCT" \
|
||||||
|
--argjson plateau_patience "$PLATEAU_PATIENCE" \
|
||||||
|
--argjson warmup_max_requests "$BENCH_WARMUP_MAX_REQUESTS" \
|
||||||
|
--argjson ttft_slo_ms "${TTFT_SLO_MS:-4000}" \
|
||||||
|
--argjson enable_ttft_slo_stop "${ENABLE_TTFT_SLO_STOP:-1}" \
|
||||||
|
'{
|
||||||
|
experiment: $experiment,
|
||||||
|
engine: $engine,
|
||||||
|
run_id: $run_id,
|
||||||
|
model: $model,
|
||||||
|
hardware: $hardware,
|
||||||
|
matrix: $matrix,
|
||||||
|
dataset: $dataset,
|
||||||
|
tokenize_prompt: $tokenize_prompt,
|
||||||
|
random_range_ratio: $random_range_ratio,
|
||||||
|
search: {
|
||||||
|
start_concurrency: $search_start,
|
||||||
|
max_concurrency: $search_cap,
|
||||||
|
multiplier: $search_multiplier,
|
||||||
|
num_prompts_multiplier: $prompts_multiplier,
|
||||||
|
min_tps_gain_pct: $min_gain_pct,
|
||||||
|
plateau_patience: $plateau_patience,
|
||||||
|
warmup_max_requests: $warmup_max_requests,
|
||||||
|
ttft_slo_ms: $ttft_slo_ms,
|
||||||
|
enable_ttft_slo_stop: $enable_ttft_slo_stop
|
||||||
|
}
|
||||||
|
}' > "${ADAPTIVE_RUN_ROOT}/run_manifest.json"
|
||||||
|
|
||||||
|
ADAPTIVE_ACTIVE_TP=""
|
||||||
|
ADAPTIVE_ACTIVE_DP=""
|
||||||
|
trap adaptive_cleanup_active_server EXIT
|
||||||
|
trap 'adaptive_cleanup_active_server; exit 130' INT
|
||||||
|
trap 'adaptive_cleanup_active_server; exit 143' TERM
|
||||||
|
|
||||||
|
local abort_all=0
|
||||||
|
local cfg tp dp config_root server_args shape_count mark isl osl
|
||||||
|
for cfg in "${PARALLEL_CONFIGS[@]}"; do
|
||||||
|
read -r tp dp <<< "$cfg"
|
||||||
|
if ! adaptive_matches_filter "$tp" "$TP_LIST"; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
config_root="${ADAPTIVE_RUN_ROOT}/tp${tp}_dp${dp}"
|
||||||
|
mkdir -p "${config_root}/raw_outputs" "${config_root}/metrics" \
|
||||||
|
"${config_root}/logs" "${config_root}/gpu_logs"
|
||||||
|
server_args="$(engine_build_server_args "$tp" "$dp")"
|
||||||
|
printf '%s\n' "$server_args" > "${config_root}/server_cmd.txt"
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == "1" ]]; then
|
||||||
|
log "DRY_RUN server tp=${tp} dp=${dp}: ${server_args}"
|
||||||
|
shape_count=0
|
||||||
|
while IFS=$'\t' read -r mark isl osl; do
|
||||||
|
[[ "$mark" == "mark" ]] && continue
|
||||||
|
adaptive_matches_filter "$isl" "$ISL_LIST" || continue
|
||||||
|
adaptive_matches_filter "$osl" "$OSL_LIST" || continue
|
||||||
|
if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
shape_count=$((shape_count + 1))
|
||||||
|
if (( resume_mode == 1 )) && adaptive_shape_already_tested "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl"; then
|
||||||
|
log "DRY_RUN RESUME skip already tested shape tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
log "DRY_RUN tp=${tp} dp=${dp} isl=${isl} osl=${osl} C=[$(adaptive_print_sequence)]"
|
||||||
|
done < "$shapes_tsv"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
|
ADAPTIVE_ACTIVE_TP="$tp"
|
||||||
|
ADAPTIVE_ACTIVE_DP="$dp"
|
||||||
|
if ! engine_start_server "$tp" "$dp"; then
|
||||||
|
log "ERROR: failed to start ${ENGINE} tp=${tp} dp=${dp}; skipping config"
|
||||||
|
adaptive_cleanup_active_server
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
adaptive_warmup "${ADAPTIVE_LOG_DIR}/${ENGINE}_tp${tp}_dp${dp}.warmup.log" || {
|
||||||
|
log "ERROR: warmup failed for tp=${tp} dp=${dp}; skipping config"
|
||||||
|
adaptive_cleanup_active_server
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
shape_count=0
|
||||||
|
while IFS=$'\t' read -r mark isl osl; do
|
||||||
|
[[ "$mark" == "mark" ]] && continue
|
||||||
|
adaptive_matches_filter "$isl" "$ISL_LIST" || continue
|
||||||
|
adaptive_matches_filter "$osl" "$OSL_LIST" || continue
|
||||||
|
if (( GRID_LIMIT > 0 && shape_count >= GRID_LIMIT )); then
|
||||||
|
log "GRID_LIMIT=${GRID_LIMIT} reached for tp=${tp} dp=${dp}"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
shape_count=$((shape_count + 1))
|
||||||
|
if (( resume_mode == 1 )) && adaptive_shape_already_tested "$tp" "$dp" "$mark" "$isl" "$osl" "$resume_shapes_jsonl"; then
|
||||||
|
log "RESUME skip already tested shape tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark}"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
log "===== shape START tp=${tp} dp=${dp} isl=${isl} osl=${osl} mark=${mark} ====="
|
||||||
|
adaptive_run_shape "$tp" "$dp" "$mark" "$isl" "$osl" "$config_root"
|
||||||
|
|
||||||
|
if (( ADAPTIVE_FATAL_WORKLOAD == 1 )); then
|
||||||
|
log "ERROR: invalid synthetic workload; aborting remaining searches"
|
||||||
|
abort_all=1
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
if (( ADAPTIVE_RESTART_NEEDED == 1 )); then
|
||||||
|
if ! adaptive_restart_server "$tp" "$dp"; then
|
||||||
|
log "ERROR: server restart failed; skipping remaining shapes for tp=${tp} dp=${dp}"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
done < "$shapes_tsv"
|
||||||
|
|
||||||
|
adaptive_cleanup_active_server
|
||||||
|
if (( abort_all == 1 )); then
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
"$PYTHON" "$ADAPTIVE_HELPER" summarize \
|
||||||
|
--points "$ADAPTIVE_POINTS_JSONL" \
|
||||||
|
--shapes "$ADAPTIVE_SHAPES_JSONL" \
|
||||||
|
--output-dir "$ADAPTIVE_RUN_ROOT" \
|
||||||
|
>> "${ADAPTIVE_LOG_DIR}/summarize.log" 2>&1
|
||||||
|
log "adaptive results saved to ${ADAPTIVE_RUN_ROOT}"
|
||||||
|
|
||||||
|
trap - EXIT INT TERM
|
||||||
|
adaptive_cleanup_active_server
|
||||||
|
if (( abort_all == 1 )); then
|
||||||
|
return 2
|
||||||
|
fi
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user