58 lines
1.9 KiB
JSON
58 lines
1.9 KiB
JSON
[
|
|
{
|
|
"name": "sharegpt_concurrency_128",
|
|
"args": "--max-concurrency 128",
|
|
"duration_s": 155.26,
|
|
"request_throughput": 12.88,
|
|
"input_token_throughput": 3710.5,
|
|
"output_token_throughput": 2690.48,
|
|
"total_token_throughput": 6400.98,
|
|
"mean_ttft_ms": 410.55,
|
|
"mean_tpot_ms": 48.45,
|
|
"mean_e2e_latency_ms": 9696.87,
|
|
"accept_length": 2.6,
|
|
"output_file": "bench_results/sglang_8card_max_throughput_20260705_030839/sharegpt_concurrency_128.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_concurrency_256",
|
|
"args": "--max-concurrency 256",
|
|
"duration_s": 90.65,
|
|
"request_throughput": 22.06,
|
|
"input_token_throughput": 6355.48,
|
|
"output_token_throughput": 4608.35,
|
|
"total_token_throughput": 10963.84,
|
|
"mean_ttft_ms": 509.73,
|
|
"mean_tpot_ms": 62.71,
|
|
"mean_e2e_latency_ms": 10869.76,
|
|
"accept_length": 2.6,
|
|
"output_file": "bench_results/sglang_8card_max_throughput_20260705_030839/sharegpt_concurrency_256.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_concurrency_512",
|
|
"args": "--max-concurrency 512",
|
|
"duration_s": 71.39,
|
|
"request_throughput": 28.02,
|
|
"input_token_throughput": 8070.04,
|
|
"output_token_throughput": 5851.58,
|
|
"total_token_throughput": 13921.61,
|
|
"mean_ttft_ms": 1343.73,
|
|
"mean_tpot_ms": 121.2,
|
|
"mean_e2e_latency_ms": 16176.14,
|
|
"accept_length": 2.6,
|
|
"output_file": "bench_results/sglang_8card_max_throughput_20260705_030839/sharegpt_concurrency_512.json"
|
|
},
|
|
{
|
|
"name": "sharegpt_concurrency_1024",
|
|
"args": "--max-concurrency 1024",
|
|
"duration_s": 68.21,
|
|
"request_throughput": 29.32,
|
|
"input_token_throughput": 8445.53,
|
|
"output_token_throughput": 6123.84,
|
|
"total_token_throughput": 14569.37,
|
|
"mean_ttft_ms": 13265.46,
|
|
"mean_tpot_ms": 105.36,
|
|
"mean_e2e_latency_ms": 27566.99,
|
|
"accept_length": 2.6,
|
|
"output_file": "bench_results/sglang_8card_max_throughput_20260705_030839/sharegpt_concurrency_1024.json"
|
|
}
|
|
] |