From 20cb4e35c261351cd61f676f6b60d99d1ed928f7 Mon Sep 17 00:00:00 2001 From: SSKJ Dev Date: Thu, 16 Jul 2026 06:25:00 +0000 Subject: [PATCH] update --- .../dsv4_h20_sglang_tp_dp_matrix/run_bench.sh | 10 ++++--- .../adaptive_points.jsonl | 16 ++++++++++ .../adaptive_shapes.jsonl | 5 ++++ .../run_manifest.json | 22 ++++++++++++++ .../adaptive_20260716-051910/shapes.tsv | 29 +++++++++++++++++++ .../metrics/vllm_c112_i1024_o2048_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c16_i1024_o1024_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c16_i1024_o128_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c16_i1024_o2048_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c16_i1024_o256_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c16_i1024_o512_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c32_i1024_o1024_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c32_i1024_o2048_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c32_i1024_o256_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c48_i1024_o1024_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c48_i1024_o2048_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c64_i1024_o1024_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c64_i1024_o2048_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c80_i1024_o1024_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c80_i1024_o2048_a1.json | 25 ++++++++++++++++ .../metrics/vllm_c96_i1024_o2048_a1.json | 25 ++++++++++++++++ .../tp2_dp4/server_cmd.txt | 1 + .../h20/dsv4_h20_vllm_tp_dp_matrix/compare.py | 4 +-- 23 files changed, 481 insertions(+), 6 deletions(-) create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_points.jsonl create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_shapes.jsonl create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/run_manifest.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/shapes.tsv create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c112_i1024_o2048_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o1024_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o128_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o2048_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o256_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o512_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o1024_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o2048_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o256_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o1024_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o2048_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o1024_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o2048_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o1024_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o2048_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c96_i1024_o2048_a1.json create mode 100644 experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/server_cmd.txt diff --git a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh index c2d3a37..3fbb7b3 100755 --- a/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh +++ b/experiments/h20/dsv4_h20_sglang_tp_dp_matrix/run_bench.sh @@ -162,7 +162,7 @@ run_warmup() { local input_len="$1" local output_len="$2" - log "warming up (input=${input_len}, output=${output_len}, num=1)" + log "warming up (input=${input_len}, output=${output_len}, num=5)" bash -c ' run_bench_serving \ --backend sglang \ @@ -172,14 +172,16 @@ run_warmup() { --dataset-path "'"$DATASET_PATH"'" \ --random-input-len "'"$input_len"'" \ --random-output-len "'"$output_len"'" \ - --num-prompts 1 \ + --num-prompts 5 \ --max-concurrency 1 \ - --request-rate 10000 \ + --request-rate 1 \ --output-file /dev/null \ --output-details \ >> "'"${log_dir_global}/warmup.log"'" 2>&1 ' - log "warmup completed" + log "warmup completed, waiting for TileLang compilation to settle..." + sleep 15 + log "warmup settle done" } scenario_already_completed() { diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_points.jsonl b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_points.jsonl new file mode 100644 index 0000000..f86e45a --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_points.jsonl @@ -0,0 +1,16 @@ +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":25.27050317999965,"request_tps":3.165746223182292,"input_tps":3241.724132538667,"output_tps":405.2155165673334,"total_tps":3646.9396491060006,"mean_input_tokens":1024,"mean_output_tokens":128,"ttft_p50_ms":1051.9641595019493,"ttft_p95_ms":7164.641508652858,"ttft_p99_ms":7186.009613565839,"tpot_p50_ms":18.72120111024348,"tpot_p95_ms":65.18447974448227,"tpot_p99_ms":66.09945481066043,"e2e_p50_ms":3493.628591997549,"e2e_p95_ms":9117.589514059364,"e2e_p99_ms":9138.18436251444,"itl_p50_ms":15.28507299371995,"itl_p95_ms":16.032998800073983,"itl_p99_ms":21.76707776481644,"validation_errors":[],"timestamp":"2026-07-16T05:31:54+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":128,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o128_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i1024_o128_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":25.495733696006937,"request_tps":3.137779871482159,"input_tps":3213.086588397731,"output_tps":803.2716470994327,"total_tps":4016.3582354971636,"mean_input_tokens":1024,"mean_output_tokens":256,"ttft_p50_ms":839.0470390004339,"ttft_p95_ms":1269.5357547512679,"ttft_p99_ms":1545.07869652909,"tpot_p50_ms":16.542784652961465,"tpot_p95_ms":18.45956203372243,"tpot_p99_ms":19.536147781054822,"e2e_p50_ms":5053.470186998311,"e2e_p95_ms":5453.463989494776,"e2e_p99_ms":5468.152181307523,"itl_p50_ms":15.194430503470358,"itl_p95_ms":15.815716297220208,"itl_p99_ms":16.660370378667714,"validation_errors":[],"timestamp":"2026-07-16T05:33:35+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":58.333280092992936,"request_tps":2.742859646241964,"input_tps":2808.6882777517712,"output_tps":702.1720694379428,"total_tps":3510.8603471897136,"mean_input_tokens":1024,"mean_output_tokens":256,"ttft_p50_ms":1302.453588003118,"ttft_p95_ms":25865.006712951435,"ttft_p99_ms":25871.150527912396,"tpot_p50_ms":22.315067135302037,"tpot_p95_ms":116.87548523570477,"tpot_p99_ms":116.9072278467824,"e2e_p50_ms":7041.645481498563,"e2e_p95_ms":30671.992690498882,"e2e_p99_ms":30678.4747339561,"itl_p50_ms":18.8346959912451,"itl_p95_ms":20.653201197274026,"itl_p99_ms":25.224988315603593,"validation_errors":[],"timestamp":"2026-07-16T05:35:22+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":-12.585976,"plateau_streak":1,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o256_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i1024_o256_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":63.555366341999616,"request_tps":1.2587450061967969,"input_tps":1288.95488634552,"output_tps":644.47744317276,"total_tps":1933.4323295182799,"mean_input_tokens":1024,"mean_output_tokens":512,"ttft_p50_ms":651.5386349929031,"ttft_p95_ms":16920.61746209365,"ttft_p99_ms":16942.941599875485,"tpot_p50_ms":15.848476987276518,"tpot_p95_ms":23.578188225724393,"tpot_p99_ms":29.306536829046802,"e2e_p50_ms":8805.075798496546,"e2e_p95_ms":24636.574015000224,"e2e_p99_ms":24650.812979813927,"itl_p50_ms":15.365997998742387,"itl_p95_ms":16.487579100066792,"itl_p99_ms":18.37696666130795,"validation_errors":[],"timestamp":"2026-07-16T05:37:07+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o512_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i1024_o512_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":82.26222992698604,"request_tps":0.9724997738452514,"input_tps":995.8397684175375,"output_tps":995.8397684175375,"total_tps":1991.679536835075,"mean_input_tokens":1024,"mean_output_tokens":1024,"ttft_p50_ms":544.7934150070068,"ttft_p95_ms":881.7404834531771,"ttft_p99_ms":897.7118983241962,"tpot_p50_ms":15.400015403226577,"tpot_p95_ms":15.83289659242619,"tpot_p99_ms":16.04854842089475,"e2e_p50_ms":16476.469576002273,"e2e_p95_ms":16701.17062768695,"e2e_p99_ms":16705.017296750884,"itl_p50_ms":15.259214495017659,"itl_p95_ms":16.13290750174201,"itl_p99_ms":17.04316050017951,"validation_errors":[],"timestamp":"2026-07-16T05:39:13+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":99.20981571500306,"request_tps":1.6127436468547327,"input_tps":1651.4494943792463,"output_tps":1651.4494943792463,"total_tps":3302.8989887584926,"mean_input_tokens":1024,"mean_output_tokens":1024,"ttft_p50_ms":1029.0807624987792,"ttft_p95_ms":1823.7564543480403,"ttft_p99_ms":1851.6841998742893,"tpot_p50_ms":18.9562778137816,"tpot_p95_ms":19.887580249860058,"tpot_p99_ms":19.937121249460077,"e2e_p50_ms":20452.989998499106,"e2e_p95_ms":20839.25912065315,"e2e_p99_ms":20861.500803281087,"itl_p50_ms":18.491245500626974,"itl_p95_ms":19.586568658996843,"itl_p99_ms":20.498206740885504,"validation_errors":[],"timestamp":"2026-07-16T05:41:39+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":65.834861,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":108.07686458299577,"request_tps":2.2206417712617497,"input_tps":2273.9371737720317,"output_tps":2273.9371737720317,"total_tps":4547.874347544063,"mean_input_tokens":1024,"mean_output_tokens":1024,"ttft_p50_ms":1411.8448265071493,"ttft_p95_ms":2348.3333344942366,"ttft_p99_ms":2881.5906707521935,"tpot_p50_ms":19.57261614369442,"tpot_p95_ms":20.474906701757522,"tpot_p99_ms":20.999123584376957,"e2e_p50_ms":21663.715992996003,"e2e_p95_ms":22356.912894507695,"e2e_p99_ms":22380.086564826343,"itl_p50_ms":18.948694996652193,"itl_p95_ms":20.273402400198393,"itl_p99_ms":21.75828296574764,"validation_errors":[],"timestamp":"2026-07-16T05:44:14+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":37.693413,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i1024_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":123.36385090599651,"request_tps":2.593952747501702,"input_tps":2656.207613441743,"output_tps":2656.207613441743,"total_tps":5312.415226883486,"mean_input_tokens":1024,"mean_output_tokens":1024,"ttft_p50_ms":1434.1024970053695,"ttft_p95_ms":3039.2205538046255,"ttft_p99_ms":3390.209743604064,"tpot_p50_ms":22.264150537145387,"tpot_p95_ms":23.93735160308324,"tpot_p99_ms":24.057023045675336,"e2e_p50_ms":24811.180071505078,"e2e_p95_ms":25911.275261299306,"e2e_p99_ms":25928.425582967902,"itl_p50_ms":21.702708501834422,"itl_p95_ms":23.22826191157219,"itl_p99_ms":25.138318116369188,"validation_errors":[],"timestamp":"2026-07-16T05:47:00+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":16.81095,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c64_i1024_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c64_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":143.43459790300403,"request_tps":2.7887274468499945,"input_tps":2855.6569055743944,"output_tps":2855.6569055743944,"total_tps":5711.313811148789,"mean_input_tokens":1024,"mean_output_tokens":1024,"ttft_p50_ms":1577.1677885059034,"ttft_p95_ms":5509.251923993023,"ttft_p99_ms":20218.74289565443,"tpot_p50_ms":22.94427521847951,"tpot_p95_ms":40.54245714501829,"tpot_p99_ms":40.55054783283226,"e2e_p50_ms":24977.686929501942,"e2e_p95_ms":42717.079425647535,"e2e_p99_ms":42743.186112658295,"itl_p50_ms":21.95263899920974,"itl_p95_ms":23.614524496224476,"itl_p99_ms":25.837959441560088,"validation_errors":[],"timestamp":"2026-07-16T05:50:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":7.508799,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o1024_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c80_i1024_o1024_a1.log"} +{"status":"COMPLETED","completed":80,"failed":0,"duration_s":160.96535958899767,"request_tps":0.4970013436696486,"input_tps":508.92937591772017,"output_tps":1017.8587518354403,"total_tps":1526.7881277531606,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":408.2802780030761,"ttft_p95_ms":804.0736109498539,"ttft_p99_ms":871.2221907320782,"tpot_p50_ms":15.45796976209212,"tpot_p95_ms":15.706438855319314,"tpot_p99_ms":15.828019318818155,"e2e_p50_ms":32192.607425007736,"e2e_p95_ms":32558.569978662126,"e2e_p99_ms":32571.827183209243,"itl_p50_ms":15.350678993854672,"itl_p95_ms":16.041272398433648,"itl_p99_ms":16.693045923602767,"validation_errors":[],"timestamp":"2026-07-16T05:53:33+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":16,"num_prompts":80,"warmup_requests":16,"attempt":1,"gain_pct":null,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c16_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c16_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":160,"failed":0,"duration_s":197.1833312999952,"request_tps":0.8114276138107009,"input_tps":830.9018765421578,"output_tps":1661.8037530843155,"total_tps":2492.705629626473,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":656.5913640006329,"ttft_p95_ms":1290.7357379466703,"ttft_p99_ms":1304.3246922609978,"tpot_p50_ms":18.853423953588848,"tpot_p95_ms":19.17381441135775,"tpot_p99_ms":19.217155080916477,"e2e_p50_ms":39465.7350474954,"e2e_p95_ms":39581.901998698595,"e2e_p99_ms":39587.70705576069,"itl_p50_ms":18.70039399364032,"itl_p95_ms":19.59088460134808,"itl_p99_ms":20.509281637496308,"validation_errors":[],"timestamp":"2026-07-16T05:57:35+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":32,"num_prompts":160,"warmup_requests":32,"attempt":1,"gain_pct":63.264672,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c32_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c32_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":240,"failed":0,"duration_s":200.68862108800386,"request_tps":1.195882450628617,"input_tps":1224.5836294437038,"output_tps":2449.1672588874076,"total_tps":3673.7508883311116,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":833.4129540016875,"ttft_p95_ms":1647.5724648524192,"ttft_p99_ms":1656.5784132984118,"tpot_p50_ms":19.151686465800864,"tpot_p95_ms":19.465789479505947,"tpot_p99_ms":19.511620899212012,"e2e_p50_ms":40142.91101550043,"e2e_p95_ms":40502.659226708056,"e2e_p99_ms":40515.6316231635,"itl_p50_ms":18.947302000015043,"itl_p95_ms":20.123915805015713,"itl_p99_ms":21.517608713475045,"validation_errors":[],"timestamp":"2026-07-16T06:01:41+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":48,"num_prompts":240,"warmup_requests":48,"attempt":1,"gain_pct":47.380053,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c48_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c48_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":320,"failed":0,"duration_s":231.29285078399698,"request_tps":1.3835274152024963,"input_tps":1416.7320731673562,"output_tps":2833.4641463347125,"total_tps":4250.196219502069,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":1245.4625594982645,"ttft_p95_ms":2417.803988052765,"ttft_p99_ms":2683.46135836342,"tpot_p50_ms":21.843712060574735,"tpot_p95_ms":22.34440391123519,"tpot_p99_ms":22.46978759702825,"e2e_p50_ms":46248.58569000207,"e2e_p95_ms":46951.28065479075,"e2e_p99_ms":46979.07613164149,"itl_p50_ms":21.61537500069244,"itl_p95_ms":23.96329630355467,"itl_p99_ms":25.871719614806352,"validation_errors":[],"timestamp":"2026-07-16T06:06:16+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":64,"num_prompts":320,"warmup_requests":64,"attempt":1,"gain_pct":15.69092,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c64_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c64_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":400,"failed":0,"duration_s":235.45233723000274,"request_tps":1.6988576316796469,"input_tps":1739.6302148399584,"output_tps":3479.260429679917,"total_tps":5218.890644519875,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":1586.3785705005284,"ttft_p95_ms":3133.5033947099873,"ttft_p99_ms":3142.967526005523,"tpot_p50_ms":22.12632127210705,"tpot_p95_ms":22.91146984701974,"tpot_p99_ms":22.976990950927,"e2e_p50_ms":47054.46670300444,"e2e_p95_ms":48105.06032910416,"e2e_p99_ms":48135.96432496226,"itl_p50_ms":21.907794500293676,"itl_p95_ms":23.54792225087294,"itl_p99_ms":25.59503832002519,"validation_errors":[],"timestamp":"2026-07-16T06:11:01+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":80,"num_prompts":400,"warmup_requests":80,"attempt":1,"gain_pct":22.791758,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c80_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c80_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":480,"failed":0,"duration_s":261.48785026300175,"request_tps":1.8356493409434549,"input_tps":1879.7049251260978,"output_tps":3759.4098502521956,"total_tps":5639.114775378293,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":2183.7680815006024,"ttft_p95_ms":3805.7011090502783,"ttft_p99_ms":4558.676817088854,"tpot_p50_ms":24.876903371278917,"tpot_p95_ms":25.891116515634604,"tpot_p99_ms":26.181815688893472,"e2e_p50_ms":53079.94003850763,"e2e_p95_ms":54830.756290954014,"e2e_p99_ms":54843.78045572128,"itl_p50_ms":24.47853601188399,"itl_p95_ms":26.07234639581293,"itl_p99_ms":27.980867308797315,"validation_errors":[],"timestamp":"2026-07-16T06:16:08+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":96,"num_prompts":480,"warmup_requests":96,"attempt":1,"gain_pct":8.051982,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c96_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c96_i1024_o2048_a1.log"} +{"status":"COMPLETED","completed":560,"failed":0,"duration_s":271.93914603099984,"request_tps":2.059284248602305,"input_tps":2108.70707056876,"output_tps":4217.41414113752,"total_tps":6326.121211706281,"mean_input_tokens":1024,"mean_output_tokens":2048,"ttft_p50_ms":2360.8657174918335,"ttft_p95_ms":4785.738604894141,"ttft_p99_ms":5083.200499021332,"tpot_p50_ms":25.078506466781775,"tpot_p95_ms":26.261017000906556,"tpot_p99_ms":26.82069243732855,"e2e_p50_ms":54802.16980399564,"e2e_p95_ms":55957.59202675035,"e2e_p99_ms":55983.62555622443,"itl_p50_ms":24.729957003728487,"itl_p95_ms":26.741590403253216,"itl_p99_ms":29.055896277423017,"validation_errors":[],"timestamp":"2026-07-16T06:21:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"concurrency":112,"num_prompts":560,"warmup_requests":112,"attempt":1,"gain_pct":12.182877,"plateau_streak":0,"error_type":"","raw_file":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/raw_outputs/vllm_adaptive_c112_i1024_o2048_a1.jsonl","detail_log":"/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/logs/vllm_c112_i1024_o2048_a1.log"} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_shapes.jsonl b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_shapes.jsonl new file mode 100644 index 0000000..945ee47 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/adaptive_shapes.jsonl @@ -0,0 +1,5 @@ +{"timestamp":"2026-07-16T05:31:54+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":128,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":3646.9396491060006,"last_total_tps":3646.9396491060006} +{"timestamp":"2026-07-16T05:35:22+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":256,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":2,"search_cap":512,"max_successful_concurrency":32,"saturation_concurrency":null,"stop_probe_concurrency":32,"best_tps_concurrency":16,"best_total_tps":4016.3582354971636,"last_total_tps":3510.8603471897136} +{"timestamp":"2026-07-16T05:37:07+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":512,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":1,"search_cap":512,"max_successful_concurrency":16,"saturation_concurrency":null,"stop_probe_concurrency":16,"best_tps_concurrency":16,"best_total_tps":1933.4323295182799,"last_total_tps":1933.4323295182799} +{"timestamp":"2026-07-16T05:50:06+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":1024,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":5,"search_cap":512,"max_successful_concurrency":80,"saturation_concurrency":null,"stop_probe_concurrency":80,"best_tps_concurrency":80,"best_total_tps":5711.313811148789,"last_total_tps":5711.313811148789} +{"timestamp":"2026-07-16T06:21:25+00:00","engine":"vllm","tp":2,"dp":4,"mark":"Y","isl":1024,"osl":2048,"status":"TTFT_BOUNDARY","stop_reason":"TTFT_SLO_EXCEEDED","tested_points":7,"search_cap":512,"max_successful_concurrency":112,"saturation_concurrency":null,"stop_probe_concurrency":112,"best_tps_concurrency":112,"best_total_tps":6326.121211706281,"last_total_tps":6326.121211706281} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/run_manifest.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/run_manifest.json new file mode 100644 index 0000000..ae2922b --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/run_manifest.json @@ -0,0 +1,22 @@ +{ + "experiment": "dsv4_h20_vllm_tp_dp_matrix", + "engine": "vllm", + "run_id": "adaptive_20260716-051910", + "model": "/data1/hf_models/DeepSeek-V4-Flash", + "hardware": "8x NVIDIA H20 96GB", + "matrix": "/data1/yy/sskj/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/matrix.json", + "dataset": "random", + "tokenize_prompt": false, + "random_range_ratio": 1, + "search": { + "start_concurrency": 16, + "max_concurrency": 512, + "multiplier": 2, + "num_prompts_multiplier": 5, + "min_tps_gain_pct": 2, + "plateau_patience": 2, + "warmup_max_requests": 0, + "ttft_slo_ms": 4000, + "enable_ttft_slo_stop": 1 + } +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/shapes.tsv b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/shapes.tsv new file mode 100644 index 0000000..121a236 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/shapes.tsv @@ -0,0 +1,29 @@ +mark input_len output_len +Y 1024 128 +Y 1024 256 +Y 1024 512 +Y 1024 1024 +Y 1024 2048 +Y 1024 4096 +Y 4096 128 +Y 4096 256 +Y 4096 512 +Y 4096 1024 +Y 4096 2048 +Y 4096 4096 +Y 16384 128 +Y 16384 256 +Y 16384 512 +Y 16384 1024 +Y 16384 2048 +Y 65536 128 +Y 65536 256 +Y 65536 512 +Y 65536 1024 +Y 131072 128 +Y 131072 256 +Y 131072 512 +Y 262144 128 +Y 262144 256 +Y 524288 128 +Y 1048576 128 diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c112_i1024_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c112_i1024_o2048_a1.json new file mode 100644 index 0000000..cbfe5b3 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c112_i1024_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 560, + "failed": 0, + "duration_s": 271.93914603099984, + "request_tps": 2.059284248602305, + "input_tps": 2108.70707056876, + "output_tps": 4217.41414113752, + "total_tps": 6326.121211706281, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 2360.8657174918335, + "ttft_p95_ms": 4785.738604894141, + "ttft_p99_ms": 5083.200499021332, + "tpot_p50_ms": 25.078506466781775, + "tpot_p95_ms": 26.261017000906556, + "tpot_p99_ms": 26.82069243732855, + "e2e_p50_ms": 54802.16980399564, + "e2e_p95_ms": 55957.59202675035, + "e2e_p99_ms": 55983.62555622443, + "itl_p50_ms": 24.729957003728487, + "itl_p95_ms": 26.741590403253216, + "itl_p99_ms": 29.055896277423017, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o1024_a1.json new file mode 100644 index 0000000..a45c7e1 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 82.26222992698604, + "request_tps": 0.9724997738452514, + "input_tps": 995.8397684175375, + "output_tps": 995.8397684175375, + "total_tps": 1991.679536835075, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 544.7934150070068, + "ttft_p95_ms": 881.7404834531771, + "ttft_p99_ms": 897.7118983241962, + "tpot_p50_ms": 15.400015403226577, + "tpot_p95_ms": 15.83289659242619, + "tpot_p99_ms": 16.04854842089475, + "e2e_p50_ms": 16476.469576002273, + "e2e_p95_ms": 16701.17062768695, + "e2e_p99_ms": 16705.017296750884, + "itl_p50_ms": 15.259214495017659, + "itl_p95_ms": 16.13290750174201, + "itl_p99_ms": 17.04316050017951, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o128_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o128_a1.json new file mode 100644 index 0000000..69b9be5 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o128_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 25.27050317999965, + "request_tps": 3.165746223182292, + "input_tps": 3241.724132538667, + "output_tps": 405.2155165673334, + "total_tps": 3646.9396491060006, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 128.0, + "ttft_p50_ms": 1051.9641595019493, + "ttft_p95_ms": 7164.641508652858, + "ttft_p99_ms": 7186.009613565839, + "tpot_p50_ms": 18.72120111024348, + "tpot_p95_ms": 65.18447974448227, + "tpot_p99_ms": 66.09945481066043, + "e2e_p50_ms": 3493.628591997549, + "e2e_p95_ms": 9117.589514059364, + "e2e_p99_ms": 9138.18436251444, + "itl_p50_ms": 15.28507299371995, + "itl_p95_ms": 16.032998800073983, + "itl_p99_ms": 21.76707776481644, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o2048_a1.json new file mode 100644 index 0000000..92046a3 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 160.96535958899767, + "request_tps": 0.4970013436696486, + "input_tps": 508.92937591772017, + "output_tps": 1017.8587518354403, + "total_tps": 1526.7881277531606, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 408.2802780030761, + "ttft_p95_ms": 804.0736109498539, + "ttft_p99_ms": 871.2221907320782, + "tpot_p50_ms": 15.45796976209212, + "tpot_p95_ms": 15.706438855319314, + "tpot_p99_ms": 15.828019318818155, + "e2e_p50_ms": 32192.607425007736, + "e2e_p95_ms": 32558.569978662126, + "e2e_p99_ms": 32571.827183209243, + "itl_p50_ms": 15.350678993854672, + "itl_p95_ms": 16.041272398433648, + "itl_p99_ms": 16.693045923602767, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o256_a1.json new file mode 100644 index 0000000..89da427 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 25.495733696006937, + "request_tps": 3.137779871482159, + "input_tps": 3213.086588397731, + "output_tps": 803.2716470994327, + "total_tps": 4016.3582354971636, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 839.0470390004339, + "ttft_p95_ms": 1269.5357547512679, + "ttft_p99_ms": 1545.07869652909, + "tpot_p50_ms": 16.542784652961465, + "tpot_p95_ms": 18.45956203372243, + "tpot_p99_ms": 19.536147781054822, + "e2e_p50_ms": 5053.470186998311, + "e2e_p95_ms": 5453.463989494776, + "e2e_p99_ms": 5468.152181307523, + "itl_p50_ms": 15.194430503470358, + "itl_p95_ms": 15.815716297220208, + "itl_p99_ms": 16.660370378667714, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o512_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o512_a1.json new file mode 100644 index 0000000..e37d0f6 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c16_i1024_o512_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 80, + "failed": 0, + "duration_s": 63.555366341999616, + "request_tps": 1.2587450061967969, + "input_tps": 1288.95488634552, + "output_tps": 644.47744317276, + "total_tps": 1933.4323295182799, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 512.0, + "ttft_p50_ms": 651.5386349929031, + "ttft_p95_ms": 16920.61746209365, + "ttft_p99_ms": 16942.941599875485, + "tpot_p50_ms": 15.848476987276518, + "tpot_p95_ms": 23.578188225724393, + "tpot_p99_ms": 29.306536829046802, + "e2e_p50_ms": 8805.075798496546, + "e2e_p95_ms": 24636.574015000224, + "e2e_p99_ms": 24650.812979813927, + "itl_p50_ms": 15.365997998742387, + "itl_p95_ms": 16.487579100066792, + "itl_p99_ms": 18.37696666130795, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o1024_a1.json new file mode 100644 index 0000000..92b50e3 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 160, + "failed": 0, + "duration_s": 99.20981571500306, + "request_tps": 1.6127436468547327, + "input_tps": 1651.4494943792463, + "output_tps": 1651.4494943792463, + "total_tps": 3302.8989887584926, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 1029.0807624987792, + "ttft_p95_ms": 1823.7564543480403, + "ttft_p99_ms": 1851.6841998742893, + "tpot_p50_ms": 18.9562778137816, + "tpot_p95_ms": 19.887580249860058, + "tpot_p99_ms": 19.937121249460077, + "e2e_p50_ms": 20452.989998499106, + "e2e_p95_ms": 20839.25912065315, + "e2e_p99_ms": 20861.500803281087, + "itl_p50_ms": 18.491245500626974, + "itl_p95_ms": 19.586568658996843, + "itl_p99_ms": 20.498206740885504, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o2048_a1.json new file mode 100644 index 0000000..1087187 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 160, + "failed": 0, + "duration_s": 197.1833312999952, + "request_tps": 0.8114276138107009, + "input_tps": 830.9018765421578, + "output_tps": 1661.8037530843155, + "total_tps": 2492.705629626473, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 656.5913640006329, + "ttft_p95_ms": 1290.7357379466703, + "ttft_p99_ms": 1304.3246922609978, + "tpot_p50_ms": 18.853423953588848, + "tpot_p95_ms": 19.17381441135775, + "tpot_p99_ms": 19.217155080916477, + "e2e_p50_ms": 39465.7350474954, + "e2e_p95_ms": 39581.901998698595, + "e2e_p99_ms": 39587.70705576069, + "itl_p50_ms": 18.70039399364032, + "itl_p95_ms": 19.59088460134808, + "itl_p99_ms": 20.509281637496308, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o256_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o256_a1.json new file mode 100644 index 0000000..e27818f --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c32_i1024_o256_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 160, + "failed": 0, + "duration_s": 58.333280092992936, + "request_tps": 2.742859646241964, + "input_tps": 2808.6882777517712, + "output_tps": 702.1720694379428, + "total_tps": 3510.8603471897136, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 256.0, + "ttft_p50_ms": 1302.453588003118, + "ttft_p95_ms": 25865.006712951435, + "ttft_p99_ms": 25871.150527912396, + "tpot_p50_ms": 22.315067135302037, + "tpot_p95_ms": 116.87548523570477, + "tpot_p99_ms": 116.9072278467824, + "e2e_p50_ms": 7041.645481498563, + "e2e_p95_ms": 30671.992690498882, + "e2e_p99_ms": 30678.4747339561, + "itl_p50_ms": 18.8346959912451, + "itl_p95_ms": 20.653201197274026, + "itl_p99_ms": 25.224988315603593, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o1024_a1.json new file mode 100644 index 0000000..1594b18 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 240, + "failed": 0, + "duration_s": 108.07686458299577, + "request_tps": 2.2206417712617497, + "input_tps": 2273.9371737720317, + "output_tps": 2273.9371737720317, + "total_tps": 4547.874347544063, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 1411.8448265071493, + "ttft_p95_ms": 2348.3333344942366, + "ttft_p99_ms": 2881.5906707521935, + "tpot_p50_ms": 19.57261614369442, + "tpot_p95_ms": 20.474906701757522, + "tpot_p99_ms": 20.999123584376957, + "e2e_p50_ms": 21663.715992996003, + "e2e_p95_ms": 22356.912894507695, + "e2e_p99_ms": 22380.086564826343, + "itl_p50_ms": 18.948694996652193, + "itl_p95_ms": 20.273402400198393, + "itl_p99_ms": 21.75828296574764, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o2048_a1.json new file mode 100644 index 0000000..944a408 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c48_i1024_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 240, + "failed": 0, + "duration_s": 200.68862108800386, + "request_tps": 1.195882450628617, + "input_tps": 1224.5836294437038, + "output_tps": 2449.1672588874076, + "total_tps": 3673.7508883311116, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 833.4129540016875, + "ttft_p95_ms": 1647.5724648524192, + "ttft_p99_ms": 1656.5784132984118, + "tpot_p50_ms": 19.151686465800864, + "tpot_p95_ms": 19.465789479505947, + "tpot_p99_ms": 19.511620899212012, + "e2e_p50_ms": 40142.91101550043, + "e2e_p95_ms": 40502.659226708056, + "e2e_p99_ms": 40515.6316231635, + "itl_p50_ms": 18.947302000015043, + "itl_p95_ms": 20.123915805015713, + "itl_p99_ms": 21.517608713475045, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o1024_a1.json new file mode 100644 index 0000000..526872a --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 320, + "failed": 0, + "duration_s": 123.36385090599651, + "request_tps": 2.593952747501702, + "input_tps": 2656.207613441743, + "output_tps": 2656.207613441743, + "total_tps": 5312.415226883486, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 1434.1024970053695, + "ttft_p95_ms": 3039.2205538046255, + "ttft_p99_ms": 3390.209743604064, + "tpot_p50_ms": 22.264150537145387, + "tpot_p95_ms": 23.93735160308324, + "tpot_p99_ms": 24.057023045675336, + "e2e_p50_ms": 24811.180071505078, + "e2e_p95_ms": 25911.275261299306, + "e2e_p99_ms": 25928.425582967902, + "itl_p50_ms": 21.702708501834422, + "itl_p95_ms": 23.22826191157219, + "itl_p99_ms": 25.138318116369188, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o2048_a1.json new file mode 100644 index 0000000..efb8af9 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c64_i1024_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 320, + "failed": 0, + "duration_s": 231.29285078399698, + "request_tps": 1.3835274152024963, + "input_tps": 1416.7320731673562, + "output_tps": 2833.4641463347125, + "total_tps": 4250.196219502069, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 1245.4625594982645, + "ttft_p95_ms": 2417.803988052765, + "ttft_p99_ms": 2683.46135836342, + "tpot_p50_ms": 21.843712060574735, + "tpot_p95_ms": 22.34440391123519, + "tpot_p99_ms": 22.46978759702825, + "e2e_p50_ms": 46248.58569000207, + "e2e_p95_ms": 46951.28065479075, + "e2e_p99_ms": 46979.07613164149, + "itl_p50_ms": 21.61537500069244, + "itl_p95_ms": 23.96329630355467, + "itl_p99_ms": 25.871719614806352, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o1024_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o1024_a1.json new file mode 100644 index 0000000..103bee8 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o1024_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 400, + "failed": 0, + "duration_s": 143.43459790300403, + "request_tps": 2.7887274468499945, + "input_tps": 2855.6569055743944, + "output_tps": 2855.6569055743944, + "total_tps": 5711.313811148789, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 1024.0, + "ttft_p50_ms": 1577.1677885059034, + "ttft_p95_ms": 5509.251923993023, + "ttft_p99_ms": 20218.74289565443, + "tpot_p50_ms": 22.94427521847951, + "tpot_p95_ms": 40.54245714501829, + "tpot_p99_ms": 40.55054783283226, + "e2e_p50_ms": 24977.686929501942, + "e2e_p95_ms": 42717.079425647535, + "e2e_p99_ms": 42743.186112658295, + "itl_p50_ms": 21.95263899920974, + "itl_p95_ms": 23.614524496224476, + "itl_p99_ms": 25.837959441560088, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o2048_a1.json new file mode 100644 index 0000000..741eea5 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c80_i1024_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 400, + "failed": 0, + "duration_s": 235.45233723000274, + "request_tps": 1.6988576316796469, + "input_tps": 1739.6302148399584, + "output_tps": 3479.260429679917, + "total_tps": 5218.890644519875, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 1586.3785705005284, + "ttft_p95_ms": 3133.5033947099873, + "ttft_p99_ms": 3142.967526005523, + "tpot_p50_ms": 22.12632127210705, + "tpot_p95_ms": 22.91146984701974, + "tpot_p99_ms": 22.976990950927, + "e2e_p50_ms": 47054.46670300444, + "e2e_p95_ms": 48105.06032910416, + "e2e_p99_ms": 48135.96432496226, + "itl_p50_ms": 21.907794500293676, + "itl_p95_ms": 23.54792225087294, + "itl_p99_ms": 25.59503832002519, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c96_i1024_o2048_a1.json b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c96_i1024_o2048_a1.json new file mode 100644 index 0000000..9f5a88b --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/metrics/vllm_c96_i1024_o2048_a1.json @@ -0,0 +1,25 @@ +{ + "status": "COMPLETED", + "completed": 480, + "failed": 0, + "duration_s": 261.48785026300175, + "request_tps": 1.8356493409434549, + "input_tps": 1879.7049251260978, + "output_tps": 3759.4098502521956, + "total_tps": 5639.114775378293, + "mean_input_tokens": 1024.0, + "mean_output_tokens": 2048.0, + "ttft_p50_ms": 2183.7680815006024, + "ttft_p95_ms": 3805.7011090502783, + "ttft_p99_ms": 4558.676817088854, + "tpot_p50_ms": 24.876903371278917, + "tpot_p95_ms": 25.891116515634604, + "tpot_p99_ms": 26.181815688893472, + "e2e_p50_ms": 53079.94003850763, + "e2e_p95_ms": 54830.756290954014, + "e2e_p99_ms": 54843.78045572128, + "itl_p50_ms": 24.47853601188399, + "itl_p95_ms": 26.07234639581293, + "itl_p99_ms": 27.980867308797315, + "validation_errors": [] +} diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/server_cmd.txt b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/server_cmd.txt new file mode 100644 index 0000000..0fd28a6 --- /dev/null +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/adaptive_results/adaptive_20260716-051910/tp2_dp4/server_cmd.txt @@ -0,0 +1 @@ +vllm serve /data1/hf_models/DeepSeek-V4-Flash --trust-remote-code --kv-cache-dtype fp8 --block-size 256 --tensor-parallel-size 2 --gpu-memory-utilization 0.9 --max-model-len 1048576 --max-num-seqs 128 --no-enable-flashinfer-autotune --host 0.0.0.0 --port 30030 --data-parallel-size 4 diff --git a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/compare.py b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/compare.py index 15f277c..002fa0c 100644 --- a/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/compare.py +++ b/experiments/h20/dsv4_h20_vllm_tp_dp_matrix/compare.py @@ -18,7 +18,7 @@ def load_result(result_root: Path) -> dict: def slo_status(ttft_p95_ms: float, tpot_mean_ms: float, - ttft_limit_ms: float = 4000.0, tpot_limit_ms: float = 50.0) -> str: + ttft_limit_ms: float = 3000.0, tpot_limit_ms: float = 50.0) -> str: ttft_ok = ttft_p95_ms < ttft_limit_ms tpot_ok = tpot_mean_ms < tpot_limit_ms if ttft_ok and tpot_ok: @@ -42,7 +42,7 @@ def main(): parser = argparse.ArgumentParser() parser.add_argument("--run-root", type=Path, required=True) parser.add_argument("-o", "--output", type=Path, default=Path("comparison.md")) - parser.add_argument("--ttft-limit", type=float, default=4000.0) + parser.add_argument("--ttft-limit", type=float, default=3000.0) parser.add_argument("--tpot-limit", type=float, default=50.0) args = parser.parse_args()