156 lines
5.1 KiB
JSON
156 lines
5.1 KiB
JSON
[
|
|
{
|
|
"name": "sharegpt_concurrency_1",
|
|
"args": "--max-concurrency 1",
|
|
"duration_s": 1715.45,
|
|
"request_throughput": 1.17,
|
|
"input_token_throughput": 335.83,
|
|
"output_token_throughput": 243.51,
|
|
"total_token_throughput": 579.34,
|
|
"mean_ttft_ms": 128.09,
|
|
"mean_tpot_ms": 4.0,
|
|
"mean_e2e_latency_ms": 856.77,
|
|
"accept_length": 2.62,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_1.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_concurrency_8",
|
|
"args": "--max-concurrency 8",
|
|
"duration_s": 448.91,
|
|
"request_throughput": 4.46,
|
|
"input_token_throughput": 1283.33,
|
|
"output_token_throughput": 930.54,
|
|
"total_token_throughput": 2213.87,
|
|
"mean_ttft_ms": 146.76,
|
|
"mean_tpot_ms": 8.41,
|
|
"mean_e2e_latency_ms": 1790.65,
|
|
"accept_length": 2.61,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_8.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_concurrency_16",
|
|
"args": "--max-concurrency 16",
|
|
"duration_s": 331.79,
|
|
"request_throughput": 6.03,
|
|
"input_token_throughput": 1736.31,
|
|
"output_token_throughput": 1259.0,
|
|
"total_token_throughput": 2995.3,
|
|
"mean_ttft_ms": 168.95,
|
|
"mean_tpot_ms": 12.67,
|
|
"mean_e2e_latency_ms": 2642.1,
|
|
"accept_length": 2.61,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_16.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_concurrency_32",
|
|
"args": "--max-concurrency 32",
|
|
"duration_s": 267.72,
|
|
"request_throughput": 7.47,
|
|
"input_token_throughput": 2151.87,
|
|
"output_token_throughput": 1560.32,
|
|
"total_token_throughput": 3712.19,
|
|
"mean_ttft_ms": 206.24,
|
|
"mean_tpot_ms": 20.72,
|
|
"mean_e2e_latency_ms": 4249.71,
|
|
"accept_length": 2.61,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_32.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_concurrency_64",
|
|
"args": "--max-concurrency 64",
|
|
"duration_s": 207.57,
|
|
"request_throughput": 9.64,
|
|
"input_token_throughput": 2775.39,
|
|
"output_token_throughput": 2012.43,
|
|
"total_token_throughput": 4787.82,
|
|
"mean_ttft_ms": 248.41,
|
|
"mean_tpot_ms": 32.61,
|
|
"mean_e2e_latency_ms": 6557.39,
|
|
"accept_length": 2.61,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_64.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_concurrency_128",
|
|
"args": "--max-concurrency 128",
|
|
"duration_s": 139.41,
|
|
"request_throughput": 14.35,
|
|
"input_token_throughput": 4132.36,
|
|
"output_token_throughput": 2996.37,
|
|
"total_token_throughput": 7128.73,
|
|
"mean_ttft_ms": 330.4,
|
|
"mean_tpot_ms": 46.15,
|
|
"mean_e2e_latency_ms": 8596.12,
|
|
"accept_length": 2.61,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_concurrency_128.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_rps_1",
|
|
"args": "--request-rate 1",
|
|
"duration_s": 2011.46,
|
|
"request_throughput": 0.99,
|
|
"input_token_throughput": 286.41,
|
|
"output_token_throughput": 207.67,
|
|
"total_token_throughput": 494.08,
|
|
"mean_ttft_ms": 146.87,
|
|
"mean_tpot_ms": 4.91,
|
|
"mean_e2e_latency_ms": 1007.62,
|
|
"accept_length": 2.61,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_1.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_rps_2",
|
|
"args": "--request-rate 2",
|
|
"duration_s": 1007.21,
|
|
"request_throughput": 1.99,
|
|
"input_token_throughput": 571.97,
|
|
"output_token_throughput": 414.74,
|
|
"total_token_throughput": 986.71,
|
|
"mean_ttft_ms": 160.63,
|
|
"mean_tpot_ms": 5.99,
|
|
"mean_e2e_latency_ms": 1205.24,
|
|
"accept_length": 2.61,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_2.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_rps_4",
|
|
"args": "--request-rate 4",
|
|
"duration_s": 505.49,
|
|
"request_throughput": 3.96,
|
|
"input_token_throughput": 1139.69,
|
|
"output_token_throughput": 826.39,
|
|
"total_token_throughput": 1966.08,
|
|
"mean_ttft_ms": 174.67,
|
|
"mean_tpot_ms": 9.02,
|
|
"mean_e2e_latency_ms": 1743.4,
|
|
"accept_length": 2.6,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_4.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_rps_8",
|
|
"args": "--request-rate 8",
|
|
"duration_s": 256.46,
|
|
"request_throughput": 7.8,
|
|
"input_token_throughput": 2246.34,
|
|
"output_token_throughput": 1628.82,
|
|
"total_token_throughput": 3875.15,
|
|
"mean_ttft_ms": 216.65,
|
|
"mean_tpot_ms": 25.35,
|
|
"mean_e2e_latency_ms": 4542.64,
|
|
"accept_length": 2.6,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_8.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_rps_16",
|
|
"args": "--request-rate 16",
|
|
"duration_s": 137.92,
|
|
"request_throughput": 14.5,
|
|
"input_token_throughput": 4176.94,
|
|
"output_token_throughput": 3028.7,
|
|
"total_token_throughput": 7205.64,
|
|
"mean_ttft_ms": 377.28,
|
|
"mean_tpot_ms": 87.16,
|
|
"mean_e2e_latency_ms": 15048.09,
|
|
"accept_length": 2.61,
|
|
"output_file": "bench_results/sglang_8card_systematic_20260704_120819/sharegpt_rps_16.json"
|
|
}
|
|
] |