From f15003d2f5781f9dd4b39490060e9712f2be0c27 Mon Sep 17 00:00:00 2001 From: yy-fighting <2351884576@qq.com> Date: Wed, 9 Sep 2026 17:11:37 +0800 Subject: [PATCH] hit90 bench: add aggregated summaries (41 scored points) + md5 manifest (70 raw logs on 60.8) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 原始 .log 按仓库惯例留 60.8:/root/bench_logs/hit90exp/（md5s.txt 可核验）， 仓库入聚合 SUMMARY 数据：A1/A3/A6 全套主扫+cap+决赛、A2 sanity、种子遍。 --- .../results/20260909/all_summaries.json | 2339 +++++++++++++++++ .../results/20260909/md5s.txt | 70 + 2 files changed, 2409 insertions(+) create mode 100644 experiments/pro6000/glm53_nvfp4_hit90_dp_dcp_bench/results/20260909/all_summaries.json create mode 100644 experiments/pro6000/glm53_nvfp4_hit90_dp_dcp_bench/results/20260909/md5s.txt diff --git a/experiments/pro6000/glm53_nvfp4_hit90_dp_dcp_bench/results/20260909/all_summaries.json b/experiments/pro6000/glm53_nvfp4_hit90_dp_dcp_bench/results/20260909/all_summaries.json new file mode 100644 index 0000000..fb418e4 --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_hit90_dp_dcp_bench/results/20260909/all_summaries.json @@ -0,0 +1,2339 @@ +{ + "arm1_tp2pp4_cap2": { + "concurrency": 2, + "num_requests": 2, + "run_id": 9102, + "corpus_window": { + "start": 18500000, + "end": 18762144 + }, + "ok": 2, + "failed": 0, + "wall_s": 27.87, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 1024, + "output_throughput_tok_s": 36.74, + "input_throughput_tok_s": 9404.85, + "ttft_s": { + "mean": 0.4625, + "p50": 0.4814, + "max": 0.4814, + "min": 0.4436 + }, + "tpot_s": { + "mean": 0.0536, + "p50": 0.0537, + "max": 0.0537, + "min": 0.0535 + }, + "e2e_s": { + "mean": 27.8471, + "p50": 27.8592, + "max": 27.8592, + "min": 27.8349 + }, + "per_req_out_tok_s_e2e": { + "mean": 18.3861, + "p50": 18.3942, + "max": 18.3942, + "min": 18.3781 + }, + "per_req_decode_tok_s": { + "mean": 18.6969, + "p50": 18.7181, + "max": 18.7181, + "min": 18.6757 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 8, + "new_tokens": 512, + "cached_tokens": 1048064, + "hit_rate": 0.9995 + } + }, + "arm1_tp2pp4_cap2_seed": { + "concurrency": 1, + "num_requests": 2, + "run_id": 9993, + "corpus_window": { + "start": 18500000, + "end": 18762144 + }, + "ok": 2, + "failed": 0, + "wall_s": 32.52, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 16, + "output_throughput_tok_s": 0.49, + "input_throughput_tok_s": 8061.18, + "ttft_s": { + "mean": 15.6546, + "p50": 15.7625, + "max": 15.7625, + "min": 15.5466 + }, + "tpot_s": { + "mean": 0.0863, + "p50": 0.0866, + "max": 0.0866, + "min": 0.086 + }, + "e2e_s": { + "mean": 16.2589, + "p50": 16.3691, + "max": 16.3691, + "min": 16.1487 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.4921, + "p50": 0.4954, + "max": 0.4954, + "min": 0.4887 + }, + "per_req_decode_tok_s": { + "mean": 13.2421, + "p50": 13.2904, + "max": 13.2904, + "min": 13.1938 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 128, + "new_tokens": 1048576, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm1_tp2pp4_cap4": { + "concurrency": 4, + "num_requests": 4, + "run_id": 9104, + "corpus_window": { + "start": 18800000, + "end": 19324288 + }, + "ok": 4, + "failed": 0, + "wall_s": 24.41, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 2048, + "output_throughput_tok_s": 83.89, + "input_throughput_tok_s": 21475.34, + "ttft_s": { + "mean": 0.4643, + "p50": 0.4848, + "max": 0.4902, + "min": 0.4295 + }, + "tpot_s": { + "mean": 0.0467, + "p50": 0.0467, + "max": 0.0469, + "min": 0.0466 + }, + "e2e_s": { + "mean": 24.3464, + "p50": 24.3656, + "max": 24.3892, + "min": 24.3023 + }, + "per_req_out_tok_s_e2e": { + "mean": 21.0298, + "p50": 21.0452, + "max": 21.068, + "min": 20.9929 + }, + "per_req_decode_tok_s": { + "mean": 21.4389, + "p50": 21.4449, + "max": 21.497, + "min": 21.3694 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 12, + "new_tokens": 1024, + "cached_tokens": 2096128, + "hit_rate": 0.9995 + } + }, + "arm1_tp2pp4_cap4_seed": { + "concurrency": 1, + "num_requests": 4, + "run_id": 9993, + "corpus_window": { + "start": 18800000, + "end": 19324288 + }, + "ok": 4, + "failed": 0, + "wall_s": 64.86, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 32, + "output_throughput_tok_s": 0.49, + "input_throughput_tok_s": 8083.19, + "ttft_s": { + "mean": 15.6096, + "p50": 15.5571, + "max": 15.8253, + "min": 15.5229 + }, + "tpot_s": { + "mean": 0.0864, + "p50": 0.0858, + "max": 0.0889, + "min": 0.0855 + }, + "e2e_s": { + "mean": 16.2149, + "p50": 16.1581, + "max": 16.448, + "min": 16.1218 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.4934, + "p50": 0.4959, + "max": 0.4962, + "min": 0.4864 + }, + "per_req_decode_tok_s": { + "mean": 13.2245, + "p50": 13.362, + "max": 13.3691, + "min": 12.8529 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 256, + "new_tokens": 2097152, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm1_tp2pp4_cap6": { + "concurrency": 6, + "num_requests": 6, + "run_id": 9106, + "corpus_window": { + "start": 19400000, + "end": 20186432 + }, + "ok": 6, + "failed": 0, + "wall_s": 30.06, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 3072, + "output_throughput_tok_s": 102.18, + "input_throughput_tok_s": 26158.59, + "ttft_s": { + "mean": 1.113, + "p50": 1.1227, + "max": 1.1662, + "min": 1.0771 + }, + "tpot_s": { + "mean": 0.0564, + "p50": 0.0565, + "max": 0.0566, + "min": 0.0563 + }, + "e2e_s": { + "mean": 29.955, + "p50": 29.9619, + "max": 30.0192, + "min": 29.8818 + }, + "per_req_out_tok_s_e2e": { + "mean": 17.0924, + "p50": 17.0947, + "max": 17.1342, + "min": 17.0557 + }, + "per_req_decode_tok_s": { + "mean": 17.7522, + "p50": 17.787, + "max": 17.7876, + "min": 17.6907 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 12, + "new_tokens": 1536, + "cached_tokens": 3144192, + "hit_rate": 0.9995 + } + }, + "arm1_tp2pp4_cap6_seed": { + "concurrency": 1, + "num_requests": 6, + "run_id": 9993, + "corpus_window": { + "start": 19400000, + "end": 20186432 + }, + "ok": 6, + "failed": 0, + "wall_s": 96.99, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 48, + "output_throughput_tok_s": 0.49, + "input_throughput_tok_s": 8108.07, + "ttft_s": { + "mean": 15.5653, + "p50": 15.5386, + "max": 15.7702, + "min": 15.4927 + }, + "tpot_s": { + "mean": 0.0857, + "p50": 0.0855, + "max": 0.0873, + "min": 0.0848 + }, + "e2e_s": { + "mean": 16.1653, + "p50": 16.1351, + "max": 16.3817, + "min": 16.0914 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.4949, + "p50": 0.4964, + "max": 0.4972, + "min": 0.4883 + }, + "per_req_decode_tok_s": { + "mean": 13.3362, + "p50": 13.3755, + "max": 13.4795, + "min": 13.0855 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 384, + "new_tokens": 3145728, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm1_tp2pp4_cap8": { + "concurrency": 8, + "num_requests": 8, + "run_id": 9108, + "corpus_window": { + "start": 20200000, + "end": 21248576 + }, + "ok": 8, + "failed": 0, + "wall_s": 60.59, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 4096, + "output_throughput_tok_s": 67.6, + "input_throughput_tok_s": 17305.18, + "ttft_s": { + "mean": 9.5673, + "p50": 1.7305, + "max": 33.2324, + "min": 1.5966 + }, + "tpot_s": { + "mean": 0.0587, + "p50": 0.0606, + "max": 0.0608, + "min": 0.0531 + }, + "e2e_s": { + "mean": 39.5529, + "p50": 32.6736, + "max": 60.3788, + "min": 32.5433 + }, + "per_req_out_tok_s_e2e": { + "mean": 13.8941, + "p50": 15.689, + "max": 15.7329, + "min": 8.4798 + }, + "per_req_decode_tok_s": { + "mean": 17.1298, + "p50": 16.5474, + "max": 18.8609, + "min": 16.4729 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 16, + "new_tokens": 2048, + "cached_tokens": 4192256, + "hit_rate": 0.9995 + } + }, + "arm1_tp2pp4_cap8_seed": { + "concurrency": 1, + "num_requests": 8, + "run_id": 9993, + "corpus_window": { + "start": 20200000, + "end": 21248576 + }, + "ok": 8, + "failed": 0, + "wall_s": 129.3, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 64, + "output_throughput_tok_s": 0.49, + "input_throughput_tok_s": 8109.79, + "ttft_s": { + "mean": 15.5621, + "p50": 15.5385, + "max": 15.8195, + "min": 15.4877 + }, + "tpot_s": { + "mean": 0.0857, + "p50": 0.0854, + "max": 0.0886, + "min": 0.0847 + }, + "e2e_s": { + "mean": 16.1618, + "p50": 16.1363, + "max": 16.44, + "min": 16.0805 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.495, + "p50": 0.4958, + "max": 0.4975, + "min": 0.4866 + }, + "per_req_decode_tok_s": { + "mean": 13.3454, + "p50": 13.4067, + "max": 13.4988, + "min": 12.8977 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 512, + "new_tokens": 4194304, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm1_tp2pp4_hit90_cc16": { + "concurrency": 16, + "num_requests": 32, + "run_id": 9016, + "corpus_window": { + "start": 16100000, + "end": 16519328 + }, + "ok": 32, + "failed": 0, + "wall_s": 132.04, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 16384, + "output_throughput_tok_s": 124.09, + "input_throughput_tok_s": 31765.82, + "ttft_s": { + "mean": 16.3308, + "p50": 16.7472, + "max": 28.5969, + "min": 4.6926 + }, + "tpot_s": { + "mean": 0.0968, + "p50": 0.0992, + "max": 0.1204, + "min": 0.0722 + }, + "e2e_s": { + "mean": 65.798, + "p50": 65.8067, + "max": 66.4522, + "min": 65.421 + }, + "per_req_out_tok_s_e2e": { + "mean": 7.7815, + "p50": 7.7865, + "max": 7.8262, + "min": 7.7048 + }, + "per_req_decode_tok_s": { + "mean": 10.5988, + "p50": 10.4112, + "max": 13.8858, + "min": 8.3204 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 232, + "new_tokens": 1679360, + "cached_tokens": 15097856, + "hit_rate": 0.8999 + } + }, + "arm1_tp2pp4_hit90_cc4": { + "concurrency": 4, + "num_requests": 8, + "run_id": 9004, + "corpus_window": { + "start": 15500000, + "end": 15604832 + }, + "ok": 8, + "failed": 0, + "wall_s": 61.23, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 4096, + "output_throughput_tok_s": 66.9, + "input_throughput_tok_s": 17126.33, + "ttft_s": { + "mean": 6.7165, + "p50": 7.2602, + "max": 8.5903, + "min": 4.6314 + }, + "tpot_s": { + "mean": 0.0467, + "p50": 0.0471, + "max": 0.0526, + "min": 0.0414 + }, + "e2e_s": { + "mean": 30.5694, + "p50": 31.3807, + "max": 31.5266, + "min": 29.6247 + }, + "per_req_out_tok_s_e2e": { + "mean": 16.7628, + "p50": 17.2141, + "max": 17.2829, + "min": 16.2403 + }, + "per_req_decode_tok_s": { + "mean": 21.5721, + "p50": 21.9225, + "max": 24.1983, + "min": 19.0533 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 56, + "new_tokens": 419840, + "cached_tokens": 3774464, + "hit_rate": 0.8999 + } + }, + "arm1_tp2pp4_hit90_cc8": { + "concurrency": 8, + "num_requests": 16, + "run_id": 9008, + "corpus_window": { + "start": 15700000, + "end": 15909664 + }, + "ok": 16, + "failed": 0, + "wall_s": 88.49, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 8192, + "output_throughput_tok_s": 92.58, + "input_throughput_tok_s": 23700.36, + "ttft_s": { + "mean": 9.9075, + "p50": 10.978, + "max": 14.613, + "min": 4.6124 + }, + "tpot_s": { + "mean": 0.067, + "p50": 0.0676, + "max": 0.078, + "min": 0.057 + }, + "e2e_s": { + "mean": 44.1488, + "p50": 44.3813, + "max": 44.6607, + "min": 43.6772 + }, + "per_req_out_tok_s_e2e": { + "mean": 11.5981, + "p50": 11.6791, + "max": 11.7224, + "min": 11.4642 + }, + "per_req_decode_tok_s": { + "mean": 15.1051, + "p50": 15.3066, + "max": 17.5772, + "min": 12.8382 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 104, + "new_tokens": 839680, + "cached_tokens": 7548928, + "hit_rate": 0.8999 + } + }, + "arm1_tp2pp4_jitwarm": { + "concurrency": 2, + "num_requests": 2, + "run_id": 9992, + "corpus_window": { + "start": 18000000, + "end": 18026208 + }, + "ok": 2, + "failed": 0, + "wall_s": 26.88, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 1024, + "output_throughput_tok_s": 38.09, + "input_throughput_tok_s": 9751.77, + "ttft_s": { + "mean": 4.9791, + "p50": 5.5048, + "max": 5.5048, + "min": 4.4534 + }, + "tpot_s": { + "mean": 0.0428, + "p50": 0.0439, + "max": 0.0439, + "min": 0.0418 + }, + "e2e_s": { + "mean": 26.8534, + "p50": 26.8636, + "max": 26.8636, + "min": 26.8432 + }, + "per_req_out_tok_s_e2e": { + "mean": 19.0665, + "p50": 19.0737, + "max": 19.0737, + "min": 19.0593 + }, + "per_req_decode_tok_s": { + "mean": 23.4208, + "p50": 23.9946, + "max": 23.9946, + "min": 22.847 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 16, + "new_tokens": 104960, + "cached_tokens": 943616, + "hit_rate": 0.8999 + } + }, + "arm2_accept_sanity": { + "concurrency": 1, + "num_requests": 1, + "run_id": 9994, + "corpus_window": { + "start": 18000000, + "end": 18013104 + }, + "ok": 1, + "failed": 0, + "wall_s": 38.52, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 512, + "output_throughput_tok_s": 13.29, + "input_throughput_tok_s": 3402.92, + "ttft_s": { + "mean": 4.2434, + "p50": 4.2434, + "max": 4.2434, + "min": 4.2434 + }, + "tpot_s": { + "mean": 0.0671, + "p50": 0.0671, + "max": 0.0671, + "min": 0.0671 + }, + "e2e_s": { + "mean": 38.5166, + "p50": 38.5166, + "max": 38.5166, + "min": 38.5166 + }, + "per_req_out_tok_s_e2e": { + "mean": 13.293, + "p50": 13.293, + "max": 13.293, + "min": 13.293 + }, + "per_req_decode_tok_s": { + "mean": 14.939, + "p50": 14.939, + "max": 14.939, + "min": 14.939 + }, + "spec_accept_length_mean": 2.073, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 8, + "new_tokens": 52480, + "cached_tokens": 471808, + "hit_rate": 0.8999 + } + }, + "arm3_a16eagle_cap2": { + "concurrency": 2, + "num_requests": 2, + "run_id": 9102, + "corpus_window": { + "start": 18500000, + "end": 18762144 + }, + "ok": 2, + "failed": 0, + "wall_s": 11.44, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 1024, + "output_throughput_tok_s": 89.47, + "input_throughput_tok_s": 22905.4, + "ttft_s": { + "mean": 0.7252, + "p50": 0.7278, + "max": 0.7278, + "min": 0.7226 + }, + "tpot_s": { + "mean": 0.0203, + "p50": 0.021, + "max": 0.021, + "min": 0.0197 + }, + "e2e_s": { + "mean": 11.1243, + "p50": 11.4437, + "max": 11.4437, + "min": 10.8049 + }, + "per_req_out_tok_s_e2e": { + "mean": 46.0633, + "p50": 47.3861, + "max": 47.3861, + "min": 44.7406 + }, + "per_req_decode_tok_s": { + "mean": 49.2822, + "p50": 50.7836, + "max": 50.7836, + "min": 47.7808 + }, + "spec_accept_length_mean": 2.288, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 2, + "new_tokens": 128, + "cached_tokens": 262016, + "hit_rate": 0.9995 + } + }, + "arm3_a16eagle_cap2_seed": { + "concurrency": 1, + "num_requests": 2, + "run_id": 9993, + "corpus_window": { + "start": 18500000, + "end": 18762144 + }, + "ok": 2, + "failed": 0, + "wall_s": 77.22, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 16, + "output_throughput_tok_s": 0.21, + "input_throughput_tok_s": 3394.6, + "ttft_s": { + "mean": 37.9204, + "p50": 38.0978, + "max": 38.0978, + "min": 37.743 + }, + "tpot_s": { + "mean": 0.0987, + "p50": 0.1049, + "max": 0.1049, + "min": 0.0924 + }, + "e2e_s": { + "mean": 38.6112, + "p50": 38.8326, + "max": 38.8326, + "min": 38.3899 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.2072, + "p50": 0.2084, + "max": 0.2084, + "min": 0.206 + }, + "per_req_decode_tok_s": { + "mean": 11.6311, + "p50": 12.371, + "max": 12.371, + "min": 10.8912 + }, + "spec_accept_length_mean": 3, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 32, + "new_tokens": 262144, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm3_a16eagle_cap4": { + "concurrency": 4, + "num_requests": 4, + "run_id": 9104, + "corpus_window": { + "start": 18800000, + "end": 19324288 + }, + "ok": 4, + "failed": 0, + "wall_s": 26.24, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 2048, + "output_throughput_tok_s": 78.04, + "input_throughput_tok_s": 19978.52, + "ttft_s": { + "mean": 8.4174, + "p50": 13.6476, + "max": 15.4796, + "min": 2.1269 + }, + "tpot_s": { + "mean": 0.0215, + "p50": 0.0221, + "max": 0.0231, + "min": 0.0201 + }, + "e2e_s": { + "mean": 19.4279, + "p50": 24.9606, + "max": 26.1265, + "min": 12.408 + }, + "per_req_out_tok_s_e2e": { + "mean": 29.3469, + "p50": 36.0149, + "max": 41.2636, + "min": 19.5969 + }, + "per_req_decode_tok_s": { + "mean": 46.6345, + "p50": 48.0904, + "max": 49.8013, + "min": 43.3876 + }, + "spec_accept_length_mean": 2.306, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 4, + "new_tokens": 256, + "cached_tokens": 524032, + "hit_rate": 0.9995 + } + }, + "arm3_a16eagle_cap4_seed": { + "concurrency": 1, + "num_requests": 4, + "run_id": 9993, + "corpus_window": { + "start": 18800000, + "end": 19324288 + }, + "ok": 4, + "failed": 0, + "wall_s": 154.16, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 32, + "output_throughput_tok_s": 0.21, + "input_throughput_tok_s": 3401, + "ttft_s": { + "mean": 37.8493, + "p50": 37.7735, + "max": 38.1405, + "min": 37.7162 + }, + "tpot_s": { + "mean": 0.0985, + "p50": 0.0982, + "max": 0.1051, + "min": 0.0928 + }, + "e2e_s": { + "mean": 38.5388, + "p50": 38.503, + "max": 38.828, + "min": 38.4005 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.2076, + "p50": 0.2082, + "max": 0.2083, + "min": 0.206 + }, + "per_req_decode_tok_s": { + "mean": 11.6287, + "p50": 11.6934, + "max": 12.31, + "min": 10.8706 + }, + "spec_accept_length_mean": 2.833, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 64, + "new_tokens": 524288, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm3_a16eagle_cap6": { + "concurrency": 6, + "num_requests": 6, + "run_id": 9106, + "corpus_window": { + "start": 19400000, + "end": 20186432 + }, + "ok": 6, + "failed": 0, + "wall_s": 35.61, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 3072, + "output_throughput_tok_s": 86.27, + "input_throughput_tok_s": 22085.65, + "ttft_s": { + "mean": 14.1606, + "p50": 15.5251, + "max": 28.6514, + "min": 2.0322 + }, + "tpot_s": { + "mean": 0.0189, + "p50": 0.0196, + "max": 0.0241, + "min": 0.0133 + }, + "e2e_s": { + "mean": 23.7986, + "p50": 27.4988, + "max": 35.4232, + "min": 12.3895 + }, + "per_req_out_tok_s_e2e": { + "mean": 24.9974, + "p50": 23.6421, + "max": 41.3252, + "min": 14.4538 + }, + "per_req_decode_tok_s": { + "mean": 55.5678, + "p50": 59.725, + "max": 75.6113, + "min": 41.5621 + }, + "spec_accept_length_mean": 2.807, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 6, + "new_tokens": 384, + "cached_tokens": 786048, + "hit_rate": 0.9995 + } + }, + "arm3_a16eagle_cap6_seed": { + "concurrency": 1, + "num_requests": 6, + "run_id": 9993, + "corpus_window": { + "start": 19400000, + "end": 20186432 + }, + "ok": 6, + "failed": 0, + "wall_s": 230.84, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 48, + "output_throughput_tok_s": 0.21, + "input_throughput_tok_s": 3406.78, + "ttft_s": { + "mean": 37.7973, + "p50": 37.7371, + "max": 38.1364, + "min": 37.709 + }, + "tpot_s": { + "mean": 0.0966, + "p50": 0.0978, + "max": 0.0994, + "min": 0.0933 + }, + "e2e_s": { + "mean": 38.4735, + "p50": 38.422, + "max": 38.823, + "min": 38.3846 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.2079, + "p50": 0.2084, + "max": 0.2084, + "min": 0.2061 + }, + "per_req_decode_tok_s": { + "mean": 11.8411, + "p50": 11.7373, + "max": 12.2513, + "min": 11.4933 + }, + "spec_accept_length_mean": 3.111, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 96, + "new_tokens": 786432, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm3_a16eagle_cap8": { + "concurrency": 8, + "num_requests": 8, + "run_id": 9108, + "corpus_window": { + "start": 20200000, + "end": 21248576 + }, + "ok": 8, + "failed": 0, + "wall_s": 269.64, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 4096, + "output_throughput_tok_s": 15.19, + "input_throughput_tok_s": 3888.86, + "ttft_s": { + "mean": 145.7322, + "p50": 157.0707, + "max": 262.0321, + "min": 39.5137 + }, + "tpot_s": { + "mean": 0.0638, + "p50": 0.0714, + "max": 0.1421, + "min": 0.0118 + }, + "e2e_s": { + "mean": 178.3299, + "p50": 193.547, + "max": 269.1834, + "min": 73.0412 + }, + "per_req_out_tok_s_e2e": { + "mean": 3.4688, + "p50": 3.2304, + "max": 7.0097, + "min": 1.902 + }, + "per_req_decode_tok_s": { + "mean": 35.6398, + "p50": 14.5358, + "max": 84.9834, + "min": 7.0499 + }, + "spec_accept_length_mean": 3.259, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 101, + "new_tokens": 794880, + "cached_tokens": 253696, + "hit_rate": 0.2419 + } + }, + "arm3_a16eagle_cap8_seed": { + "concurrency": 1, + "num_requests": 8, + "run_id": 9993, + "corpus_window": { + "start": 20200000, + "end": 21248576 + }, + "ok": 8, + "failed": 0, + "wall_s": 307.77, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 64, + "output_throughput_tok_s": 0.21, + "input_throughput_tok_s": 3407.02, + "ttft_s": { + "mean": 37.8073, + "p50": 37.7662, + "max": 38.1528, + "min": 37.734 + }, + "tpot_s": { + "mean": 0.0947, + "p50": 0.0943, + "max": 0.1002, + "min": 0.0919 + }, + "e2e_s": { + "mean": 38.4708, + "p50": 38.4272, + "max": 38.7991, + "min": 38.3823 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.208, + "p50": 0.2082, + "max": 0.2084, + "min": 0.2062 + }, + "per_req_decode_tok_s": { + "mean": 12.073, + "p50": 12.2106, + "max": 12.4364, + "min": 11.4025 + }, + "spec_accept_length_mean": 3.667, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 128, + "new_tokens": 1048576, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm3_a16eagle_hit90_cc16": { + "concurrency": 16, + "num_requests": 32, + "run_id": 9016, + "corpus_window": { + "start": 16100000, + "end": 16519328 + }, + "ok": 32, + "failed": 0, + "wall_s": 199.88, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 16384, + "output_throughput_tok_s": 81.97, + "input_throughput_tok_s": 20984.61, + "ttft_s": { + "mean": 41.2462, + "p50": 35.6094, + "max": 94.9258, + "min": 7.9708 + }, + "tpot_s": { + "mean": 0.1021, + "p50": 0.1021, + "max": 0.2022, + "min": 0.0326 + }, + "e2e_s": { + "mean": 93.4162, + "p50": 89.8749, + "max": 172.5144, + "min": 45.3026 + }, + "per_req_out_tok_s_e2e": { + "mean": 6.0754, + "p50": 5.814, + "max": 11.3018, + "min": 2.9679 + }, + "per_req_decode_tok_s": { + "mean": 11.9887, + "p50": 10.263, + "max": 30.7319, + "min": 4.9563 + }, + "spec_accept_length_mean": 2.558, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 59, + "new_tokens": 419840, + "cached_tokens": 3774464, + "hit_rate": 0.8999 + } + }, + "arm3_a16eagle_hit90_cc4": { + "concurrency": 4, + "num_requests": 8, + "run_id": 9004, + "corpus_window": { + "start": 15500000, + "end": 15604832 + }, + "ok": 8, + "failed": 0, + "wall_s": 56.88, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 4096, + "output_throughput_tok_s": 72.01, + "input_throughput_tok_s": 18435.18, + "ttft_s": { + "mean": 9.4373, + "p50": 7.9555, + "max": 18.6278, + "min": 4.6074 + }, + "tpot_s": { + "mean": 0.0364, + "p50": 0.0381, + "max": 0.0672, + "min": 0.0164 + }, + "e2e_s": { + "mean": 28.0354, + "p50": 27.5458, + "max": 42.3104, + "min": 13.8828 + }, + "per_req_out_tok_s_e2e": { + "mean": 20.4503, + "p50": 18.9695, + "max": 36.8803, + "min": 12.101 + }, + "per_req_decode_tok_s": { + "mean": 33.5317, + "p50": 31.7499, + "max": 61.2262, + "min": 14.9034 + }, + "spec_accept_length_mean": 2.563, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 15, + "new_tokens": 104960, + "cached_tokens": 943616, + "hit_rate": 0.8999 + } + }, + "arm3_a16eagle_hit90_cc8": { + "concurrency": 8, + "num_requests": 16, + "run_id": 9008, + "corpus_window": { + "start": 15700000, + "end": 15909664 + }, + "ok": 16, + "failed": 0, + "wall_s": 103.39, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 8192, + "output_throughput_tok_s": 79.24, + "input_throughput_tok_s": 20284.56, + "ttft_s": { + "mean": 16.1443, + "p50": 13.3034, + "max": 37.6856, + "min": 4.675 + }, + "tpot_s": { + "mean": 0.0683, + "p50": 0.0673, + "max": 0.1309, + "min": 0.0259 + }, + "e2e_s": { + "mean": 51.0548, + "p50": 50.9761, + "max": 85.2322, + "min": 17.9352 + }, + "per_req_out_tok_s_e2e": { + "mean": 11.7522, + "p50": 10.6141, + "max": 28.5472, + "min": 6.0071 + }, + "per_req_decode_tok_s": { + "mean": 17.1222, + "p50": 14.9056, + "max": 38.6207, + "min": 7.657 + }, + "spec_accept_length_mean": 2.563, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 28, + "new_tokens": 209920, + "cached_tokens": 1887232, + "hit_rate": 0.8999 + } + }, + "arm3_a16eagle_jitwarm": { + "concurrency": 2, + "num_requests": 2, + "run_id": 9992, + "corpus_window": { + "start": 18000000, + "end": 18026208 + }, + "ok": 2, + "failed": 0, + "wall_s": 19.78, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 1024, + "output_throughput_tok_s": 51.76, + "input_throughput_tok_s": 13251.67, + "ttft_s": { + "mean": 9.1246, + "p50": 9.9966, + "max": 9.9966, + "min": 8.2527 + }, + "tpot_s": { + "mean": 0.0205, + "p50": 0.0219, + "max": 0.0219, + "min": 0.0191 + }, + "e2e_s": { + "mean": 19.5969, + "p50": 19.741, + "max": 19.741, + "min": 19.4528 + }, + "per_req_out_tok_s_e2e": { + "mean": 26.128, + "p50": 26.3201, + "max": 26.3201, + "min": 25.9358 + }, + "per_req_decode_tok_s": { + "mean": 49.1296, + "p50": 52.5442, + "max": 52.5442, + "min": 45.715 + }, + "spec_accept_length_mean": 2.318, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 4, + "new_tokens": 26240, + "cached_tokens": 235904, + "hit_rate": 0.8999 + } + }, + "arm6_hit90_cc1": { + "concurrency": 1, + "num_requests": 2, + "run_id": 9001, + "corpus_window": { + "start": 17000000, + "end": 17026208 + }, + "ok": 2, + "failed": 0, + "wall_s": 35.57, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 1024, + "output_throughput_tok_s": 28.79, + "input_throughput_tok_s": 7369.09, + "ttft_s": { + "mean": 3.5677, + "p50": 3.5785, + "max": 3.5785, + "min": 3.5569 + }, + "tpot_s": { + "mean": 0.0278, + "p50": 0.0278, + "max": 0.0278, + "min": 0.0278 + }, + "e2e_s": { + "mean": 17.7863, + "p50": 17.7954, + "max": 17.7954, + "min": 17.7772 + }, + "per_req_out_tok_s_e2e": { + "mean": 28.7863, + "p50": 28.801, + "max": 28.801, + "min": 28.7716 + }, + "per_req_decode_tok_s": { + "mean": 36.0099, + "p50": 36.0144, + "max": 36.0144, + "min": 36.0055 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 8, + "new_tokens": 52480, + "cached_tokens": 471808, + "hit_rate": 0.8999 + } + }, + "arm6_hit90_cc2": { + "concurrency": 2, + "num_requests": 4, + "run_id": 9002, + "corpus_window": { + "start": 17300000, + "end": 17352416 + }, + "ok": 4, + "failed": 0, + "wall_s": 43.08, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 2048, + "output_throughput_tok_s": 47.53, + "input_throughput_tok_s": 12168.78, + "ttft_s": { + "mean": 5.094, + "p50": 5.9677, + "max": 5.9747, + "min": 4.2149 + }, + "tpot_s": { + "mean": 0.0322, + "p50": 0.0333, + "max": 0.0345, + "min": 0.0298 + }, + "e2e_s": { + "mean": 21.532, + "p50": 21.8295, + "max": 21.832, + "min": 21.2159 + }, + "per_req_out_tok_s_e2e": { + "mean": 23.7831, + "p50": 24.0932, + "max": 24.1329, + "min": 23.4518 + }, + "per_req_decode_tok_s": { + "mean": 31.2494, + "p50": 32.2795, + "max": 33.5938, + "min": 29.0631 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 16, + "new_tokens": 104960, + "cached_tokens": 943616, + "hit_rate": 0.8999 + } + }, + "arm6_hit90_cc3": { + "concurrency": 3, + "num_requests": 6, + "run_id": 9003, + "corpus_window": { + "start": 17600000, + "end": 17678624 + }, + "ok": 6, + "failed": 0, + "wall_s": 48.08, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 3072, + "output_throughput_tok_s": 63.89, + "input_throughput_tok_s": 16355.86, + "ttft_s": { + "mean": 6.4701, + "p50": 7.0797, + "max": 8.1096, + "min": 4.2319 + }, + "tpot_s": { + "mean": 0.0343, + "p50": 0.0334, + "max": 0.039, + "min": 0.0309 + }, + "e2e_s": { + "mean": 24.0159, + "p50": 24.0953, + "max": 24.142, + "min": 23.8786 + }, + "per_req_out_tok_s_e2e": { + "mean": 21.3197, + "p50": 21.3456, + "max": 21.4418, + "min": 21.2078 + }, + "per_req_decode_tok_s": { + "mean": 29.436, + "p50": 30.4693, + "max": 32.4695, + "min": 25.716 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 20, + "new_tokens": 157440, + "cached_tokens": 1415424, + "hit_rate": 0.8999 + } + }, + "arm6_single512k": { + "concurrency": 1, + "num_requests": 1, + "run_id": 9601, + "corpus_window": { + "start": 10000000, + "end": 10523776 + }, + "ok": 1, + "failed": 0, + "wall_s": 151.88, + "input_len": 523776, + "shared_len": 0, + "unique_len": 523776, + "output_len": 512, + "output_tokens_total": 512, + "output_throughput_tok_s": 3.37, + "input_throughput_tok_s": 3448.53, + "ttft_s": { + "mean": 135.8167, + "p50": 135.8167, + "max": 135.8167, + "min": 135.8167 + }, + "tpot_s": { + "mean": 0.0314, + "p50": 0.0314, + "max": 0.0314, + "min": 0.0314 + }, + "e2e_s": { + "mean": 151.8827, + "p50": 151.8827, + "max": 151.8827, + "min": 151.8827 + }, + "per_req_out_tok_s_e2e": { + "mean": 3.371, + "p50": 3.371, + "max": 3.371, + "min": 3.371 + }, + "per_req_decode_tok_s": { + "mean": 31.8693, + "p50": 31.8693, + "max": 31.8693, + "min": 31.8693 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 128, + "new_tokens": 1047552, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm6_tp4pp2_090_cap2": { + "concurrency": 2, + "num_requests": 2, + "run_id": 9102, + "corpus_window": { + "start": 18500000, + "end": 18762144 + }, + "ok": 2, + "failed": 0, + "wall_s": 16.28, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 1024, + "output_throughput_tok_s": 62.91, + "input_throughput_tok_s": 16104.69, + "ttft_s": { + "mean": 0.555, + "p50": 0.6163, + "max": 0.6163, + "min": 0.4938 + }, + "tpot_s": { + "mean": 0.0307, + "p50": 0.0308, + "max": 0.0308, + "min": 0.0306 + }, + "e2e_s": { + "mean": 16.2461, + "p50": 16.2554, + "max": 16.2554, + "min": 16.2368 + }, + "per_req_out_tok_s_e2e": { + "mean": 31.5153, + "p50": 31.5333, + "max": 31.5333, + "min": 31.4973 + }, + "per_req_decode_tok_s": { + "mean": 32.6316, + "p50": 32.7781, + "max": 32.7781, + "min": 32.4851 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 4, + "new_tokens": 256, + "cached_tokens": 524032, + "hit_rate": 0.9995 + } + }, + "arm6_tp4pp2_090_cap2_seed": { + "concurrency": 1, + "num_requests": 2, + "run_id": 9993, + "corpus_window": { + "start": 18500000, + "end": 18762144 + }, + "ok": 2, + "failed": 0, + "wall_s": 43.71, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 16, + "output_throughput_tok_s": 0.37, + "input_throughput_tok_s": 5997.03, + "ttft_s": { + "mean": 21.1128, + "p50": 21.2786, + "max": 21.2786, + "min": 20.947 + }, + "tpot_s": { + "mean": 0.106, + "p50": 0.1061, + "max": 0.1061, + "min": 0.106 + }, + "e2e_s": { + "mean": 21.8554, + "p50": 22.0214, + "max": 22.0214, + "min": 21.6894 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.3661, + "p50": 0.3688, + "max": 0.3688, + "min": 0.3633 + }, + "per_req_decode_tok_s": { + "mean": 10.7767, + "p50": 10.779, + "max": 10.779, + "min": 10.7745 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 64, + "new_tokens": 524288, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm6_tp4pp2_090_cap4": { + "concurrency": 4, + "num_requests": 4, + "run_id": 9104, + "corpus_window": { + "start": 18800000, + "end": 19324288 + }, + "ok": 4, + "failed": 0, + "wall_s": 20.8, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 2048, + "output_throughput_tok_s": 98.47, + "input_throughput_tok_s": 25207.25, + "ttft_s": { + "mean": 1.329, + "p50": 1.3761, + "max": 1.4146, + "min": 1.1855 + }, + "tpot_s": { + "mean": 0.038, + "p50": 0.0379, + "max": 0.0383, + "min": 0.0379 + }, + "e2e_s": { + "mean": 20.7327, + "p50": 20.7596, + "max": 20.7663, + "min": 20.6842 + }, + "per_req_out_tok_s_e2e": { + "mean": 24.6953, + "p50": 24.7095, + "max": 24.7532, + "min": 24.6554 + }, + "per_req_decode_tok_s": { + "mean": 26.3879, + "p50": 26.4676, + "max": 26.4681, + "min": 26.1486 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 4, + "new_tokens": 512, + "cached_tokens": 1048064, + "hit_rate": 0.9995 + } + }, + "arm6_tp4pp2_090_cap4_seed": { + "concurrency": 1, + "num_requests": 4, + "run_id": 9993, + "corpus_window": { + "start": 18800000, + "end": 19324288 + }, + "ok": 4, + "failed": 0, + "wall_s": 87.09, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 32, + "output_throughput_tok_s": 0.37, + "input_throughput_tok_s": 6019.96, + "ttft_s": { + "mean": 21.0321, + "p50": 20.9467, + "max": 21.3435, + "min": 20.9114 + }, + "tpot_s": { + "mean": 0.1057, + "p50": 0.1059, + "max": 0.1066, + "min": 0.1052 + }, + "e2e_s": { + "mean": 21.7725, + "p50": 21.6882, + "max": 22.0898, + "min": 21.648 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.3675, + "p50": 0.3693, + "max": 0.3695, + "min": 0.3622 + }, + "per_req_decode_tok_s": { + "mean": 10.8085, + "p50": 10.8586, + "max": 10.8619, + "min": 10.7214 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 128, + "new_tokens": 1048576, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm6_tp4pp2_090_cap6": { + "concurrency": 6, + "num_requests": 6, + "run_id": 9106, + "corpus_window": { + "start": 19400000, + "end": 20186432 + }, + "ok": 6, + "failed": 0, + "wall_s": 38.01, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 3072, + "output_throughput_tok_s": 80.83, + "input_throughput_tok_s": 20692.52, + "ttft_s": { + "mean": 9.0646, + "p50": 2.5909, + "max": 22.8354, + "min": 1.0891 + }, + "tpot_s": { + "mean": 0.0351, + "p50": 0.0372, + "max": 0.0402, + "min": 0.0294 + }, + "e2e_s": { + "mean": 26.998, + "p50": 21.6134, + "max": 37.8551, + "min": 21.529 + }, + "per_req_out_tok_s_e2e": { + "mean": 20.3292, + "p50": 23.6998, + "max": 23.7818, + "min": 13.5252 + }, + "per_req_decode_tok_s": { + "mean": 28.9862, + "p50": 26.931, + "max": 34.0898, + "min": 24.9466 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 6, + "new_tokens": 768, + "cached_tokens": 1572096, + "hit_rate": 0.9995 + } + }, + "arm6_tp4pp2_090_cap6_seed": { + "concurrency": 1, + "num_requests": 6, + "run_id": 9993, + "corpus_window": { + "start": 19400000, + "end": 20186432 + }, + "ok": 6, + "failed": 0, + "wall_s": 130.37, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 48, + "output_throughput_tok_s": 0.37, + "input_throughput_tok_s": 6032.14, + "ttft_s": { + "mean": 20.9911, + "p50": 20.9267, + "max": 21.3366, + "min": 20.8952 + }, + "tpot_s": { + "mean": 0.1053, + "p50": 0.1053, + "max": 0.1069, + "min": 0.1046 + }, + "e2e_s": { + "mean": 21.7287, + "p50": 21.6607, + "max": 22.0851, + "min": 21.6364 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.3682, + "p50": 0.3694, + "max": 0.3697, + "min": 0.3622 + }, + "per_req_decode_tok_s": { + "mean": 10.8503, + "p50": 10.9178, + "max": 10.9286, + "min": 10.6903 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 192, + "new_tokens": 1572864, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm6_tp4pp2_090_cap8": { + "concurrency": 8, + "num_requests": 8, + "run_id": 9108, + "corpus_window": { + "start": 20200000, + "end": 21248576 + }, + "ok": 8, + "failed": 0, + "wall_s": 42.45, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 512, + "output_tokens_total": 4096, + "output_throughput_tok_s": 96.5, + "input_throughput_tok_s": 24703.17, + "ttft_s": { + "mean": 13.1278, + "p50": 24.015, + "max": 24.1504, + "min": 1.1047 + }, + "tpot_s": { + "mean": 0.0368, + "p50": 0.0372, + "max": 0.0402, + "min": 0.0356 + }, + "e2e_s": { + "mean": 31.9182, + "p50": 42.198, + "max": 42.3335, + "min": 21.5054 + }, + "per_req_out_tok_s_e2e": { + "mean": 17.9214, + "p50": 23.6791, + "max": 23.8079, + "min": 12.0944 + }, + "per_req_decode_tok_s": { + "mean": 27.2909, + "p50": 28.1571, + "max": 28.1589, + "min": 24.9545 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 6, + "new_tokens": 1024, + "cached_tokens": 2096128, + "hit_rate": 0.9995 + } + }, + "arm6_tp4pp2_090_cap8_seed": { + "concurrency": 1, + "num_requests": 8, + "run_id": 9993, + "corpus_window": { + "start": 20200000, + "end": 21248576 + }, + "ok": 8, + "failed": 0, + "wall_s": 173.79, + "input_len": 131072, + "shared_len": 0, + "unique_len": 131072, + "output_len": 8, + "output_tokens_total": 64, + "output_throughput_tok_s": 0.37, + "input_throughput_tok_s": 6033.65, + "ttft_s": { + "mean": 20.9848, + "p50": 20.9319, + "max": 21.3372, + "min": 20.9241 + }, + "tpot_s": { + "mean": 0.1055, + "p50": 0.1056, + "max": 0.1063, + "min": 0.1046 + }, + "e2e_s": { + "mean": 21.7232, + "p50": 21.6729, + "max": 22.0816, + "min": 21.6587 + }, + "per_req_out_tok_s_e2e": { + "mean": 0.3683, + "p50": 0.3692, + "max": 0.3694, + "min": 0.3623 + }, + "per_req_decode_tok_s": { + "mean": 10.837, + "p50": 10.8447, + "max": 10.921, + "min": 10.75 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 256, + "new_tokens": 2097152, + "cached_tokens": 0, + "hit_rate": 0 + } + }, + "arm6_tp4pp2_090_hit90_cc16": { + "concurrency": 16, + "num_requests": 32, + "run_id": 9016, + "corpus_window": { + "start": 16100000, + "end": 16519328 + }, + "ok": 32, + "failed": 0, + "wall_s": 130.37, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 16384, + "output_throughput_tok_s": 125.67, + "input_throughput_tok_s": 32172.03, + "ttft_s": { + "mean": 21.4126, + "p50": 22.4265, + "max": 38.7426, + "min": 4.3355 + }, + "tpot_s": { + "mean": 0.0853, + "p50": 0.088, + "max": 0.1193, + "min": 0.0506 + }, + "e2e_s": { + "mean": 64.9865, + "p50": 64.9936, + "max": 65.5692, + "min": 64.4982 + }, + "per_req_out_tok_s_e2e": { + "mean": 7.8787, + "p50": 7.8844, + "max": 7.9382, + "min": 7.8085 + }, + "per_req_decode_tok_s": { + "mean": 12.5048, + "p50": 12.0257, + "max": 19.7836, + "min": 8.3972 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 108, + "new_tokens": 839680, + "cached_tokens": 7548928, + "hit_rate": 0.8999 + } + }, + "arm6_tp4pp2_090_hit90_cc4": { + "concurrency": 4, + "num_requests": 8, + "run_id": 9004, + "corpus_window": { + "start": 15500000, + "end": 15604832 + }, + "ok": 8, + "failed": 0, + "wall_s": 53.74, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 4096, + "output_throughput_tok_s": 76.21, + "input_throughput_tok_s": 19510.89, + "ttft_s": { + "mean": 7.5782, + "p50": 8.4526, + "max": 10.5113, + "min": 4.2662 + }, + "tpot_s": { + "mean": 0.0377, + "p50": 0.0384, + "max": 0.0445, + "min": 0.0317 + }, + "e2e_s": { + "mean": 26.8365, + "p50": 26.9123, + "max": 26.9937, + "min": 26.6817 + }, + "per_req_out_tok_s_e2e": { + "mean": 19.0789, + "p50": 19.1344, + "max": 19.1892, + "min": 18.9674 + }, + "per_req_decode_tok_s": { + "mean": 26.9585, + "p50": 27.6615, + "max": 31.5934, + "min": 22.5281 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 28, + "new_tokens": 209920, + "cached_tokens": 1887232, + "hit_rate": 0.8999 + } + }, + "arm6_tp4pp2_090_hit90_cc8": { + "concurrency": 8, + "num_requests": 16, + "run_id": 9008, + "corpus_window": { + "start": 15700000, + "end": 15909664 + }, + "ok": 16, + "failed": 0, + "wall_s": 77.02, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 8192, + "output_throughput_tok_s": 106.36, + "input_throughput_tok_s": 27228.84, + "ttft_s": { + "mean": 12.3826, + "p50": 14.1784, + "max": 19.509, + "min": 4.3089 + }, + "tpot_s": { + "mean": 0.051, + "p50": 0.0525, + "max": 0.0674, + "min": 0.0364 + }, + "e2e_s": { + "mean": 38.4274, + "p50": 38.488, + "max": 38.7584, + "min": 38.1166 + }, + "per_req_out_tok_s_e2e": { + "mean": 13.3243, + "p50": 13.3696, + "max": 13.4325, + "min": 13.2101 + }, + "per_req_decode_tok_s": { + "mean": 20.4664, + "p50": 20.9385, + "max": 27.5161, + "min": 14.8642 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 52, + "new_tokens": 419840, + "cached_tokens": 3774464, + "hit_rate": 0.8999 + } + }, + "arm6_tp4pp2_090_jitwarm": { + "concurrency": 2, + "num_requests": 2, + "run_id": 9992, + "corpus_window": { + "start": 18000000, + "end": 18026208 + }, + "ok": 2, + "failed": 0, + "wall_s": 22.47, + "input_len": 131072, + "shared_len": 117968, + "unique_len": 13104, + "output_len": 512, + "output_tokens_total": 1024, + "output_throughput_tok_s": 45.58, + "input_throughput_tok_s": 11668.2, + "ttft_s": { + "mean": 5.6609, + "p50": 6.5403, + "max": 6.5403, + "min": 4.7816 + }, + "tpot_s": { + "mean": 0.0328, + "p50": 0.0346, + "max": 0.0346, + "min": 0.0311 + }, + "e2e_s": { + "mean": 22.4461, + "p50": 22.465, + "max": 22.465, + "min": 22.4273 + }, + "per_req_out_tok_s_e2e": { + "mean": 22.8102, + "p50": 22.8294, + "max": 22.8294, + "min": 22.791 + }, + "per_req_decode_tok_s": { + "mean": 30.5912, + "p50": 32.2282, + "max": 32.2282, + "min": 28.9541 + }, + "spec_accept_length_mean": null, + "retractions_total": 0, + "cache_hit_from_logs": { + "prefill_batches": 8, + "new_tokens": 52480, + "cached_tokens": 471808, + "hit_rate": 0.8999 + } + } +} \ No newline at end of file diff --git a/experiments/pro6000/glm53_nvfp4_hit90_dp_dcp_bench/results/20260909/md5s.txt b/experiments/pro6000/glm53_nvfp4_hit90_dp_dcp_bench/results/20260909/md5s.txt new file mode 100644 index 0000000..34fd31a --- /dev/null +++ b/experiments/pro6000/glm53_nvfp4_hit90_dp_dcp_bench/results/20260909/md5s.txt @@ -0,0 +1,70 @@ +e5b7d3a343bd8346ed6fcc81095becbc arm1_tp2pp4_cap2.log +bdd5dcab52597dc741bd5683b7a5176e arm1_tp2pp4_cap2_seed.log +47db2486446f4a0524c969d737dd44a5 arm1_tp2pp4_cap4.log +85c967cac4ea37da8026e8ab006f165b arm1_tp2pp4_cap4_seed.log +4b0e5a685e2aaf85edbd040f1cea2439 arm1_tp2pp4_cap6.log +594c8cfc4d949ed079a29d6b2bb455ad arm1_tp2pp4_cap6_seed.log +ea702eaf752b5a25873922cac2dd1d3a arm1_tp2pp4_cap8.log +dcad88481cb38e0851aae5a0cd882766 arm1_tp2pp4_cap8_seed.log +e8fd1877885551cd4d4d2d8598e35662 arm1_tp2pp4_hit90_cc16.log +7a2b304688b5c8fdd4c532ec83a5cffd arm1_tp2pp4_hit90_cc4.log +cf1a006b9a86d0d9decd73cc0cfefd15 arm1_tp2pp4_hit90_cc8.log +7b61f63dfc6e1ce002c40f3be2822dc3 arm1_tp2pp4_jitwarm.log +edf793a9d41df95b00dfab9979896fd8 arm1_tp2pp4_progress.log +6d55108f37947a3113f26ec7b4f92519 arm1_tp2pp4_warm_cc16.log +d9f5da2406d8cf04a87f2f0d2a323e49 arm1_tp2pp4_warm_cc4.log +f0f9e1055bebcfa758fe9a00f82e0bd7 arm1_tp2pp4_warm_cc8.log +d9f5da2406d8cf04a87f2f0d2a323e49 arm1_tp2pp4_warm_jit.log +7cb4a9905748e464c1d2714d45c8e07a arm2_accept_sanity.log +b598e201c5f8fd0002fa57ef2fa49883 arm2_deploy.log +43896d9059067f07bd0731da45fc07f6 arm3_a16eagle_cap2.log +9b88f8b7e494a89deda08a492c010eee arm3_a16eagle_cap2_seed.log +d4c6172a955a06f749cb0cc6631d0e32 arm3_a16eagle_cap4.log +9ac940ca7a3c0cfc6603275f7c7dc55d arm3_a16eagle_cap4_seed.log +b4ddc816549d372d10020af4f8d22510 arm3_a16eagle_cap6.log +dba518055e2881cc9609cdd6a6414bbd arm3_a16eagle_cap6_seed.log +8777cb5f5b0028cbb937478e62658fbf arm3_a16eagle_cap8.log +015950db7193e82f34069b1d205e3060 arm3_a16eagle_cap8_seed.log +1c7182dcb9cdee4ceb09a544954225bf arm3_a16eagle_hit90_cc16.log +998b501f91427c536dc7aefe5459db24 arm3_a16eagle_hit90_cc4.log +089549e4f9672896c05399a61158b4ce arm3_a16eagle_hit90_cc8.log +4ac490e8af609deffd16b70dd39197e7 arm3_a16eagle_jitwarm.log +2a99e0b288c8727e3156564349cd012b arm3_a16eagle_progress.log +04c1c3722652d937cd81b5c735843450 arm3_a16eagle_warm_cc16.log +5e516f01f698198178a3a38e5f0dac99 arm3_a16eagle_warm_cc4.log +fc9e8f330ac4e383e848f1040c5e1a93 arm3_a16eagle_warm_cc8.log +b275936cd0381cf7b69086da3c35b9ce arm3_a16eagle_warm_jit.log +ec6042c6c8ee8f03d36652a081993c24 arm3_deploy.log +2c0316f0ac5447ec181d997c6c3b56d3 arm3_docker_full.log +f18086ecfc47acc2166c939bbd634231 arm3_docker_tail.log +8704d920e3761462ea482c80ffff0153 arm4_deploy.log +d41d8cd98f00b204e9800998ecf8427e arm5_crash2_evidence.txt +95b56b9928a39063a11d8fec5a7780e4 arm5_deploy.log +18f935521ee113d09624f9db34551314 arm6_deploy.log +2b402b8e06753167a815f7c77afbb07a arm6_finals_progress.log +c3a1e8ae5535917404936b1b42f40b98 arm6_hit90_cc1.log +20c7acb9d73d0e30527afe79b8bc0816 arm6_hit90_cc2.log +d77e5bf6947a7965d696a515285f5fb4 arm6_hit90_cc3.log +72a70343690bdc8b77b3fa9bd7e9869b arm6_quality_gate.log +5e802480ee7cb075461e1f7cab3cd1ea arm6_single512k.log +fd9060e6a4935b65f8fc206a4b2e7d3c arm6_tp4pp2_090_cap2.log +7340b9fe77db6248bab4c5493f0455ed arm6_tp4pp2_090_cap2_seed.log +9b13afd7621a20e06d41005f527e0d8c arm6_tp4pp2_090_cap4.log +4c6e5f60e8566fdb0f4432ac902b779d arm6_tp4pp2_090_cap4_seed.log +207cd2c664afd33b61a44a3b35d18b1e arm6_tp4pp2_090_cap6.log +a66677ad740162d5fe2599dac5a230cf arm6_tp4pp2_090_cap6_seed.log +cb4313fee0e26825ed8a3d82c560d129 arm6_tp4pp2_090_cap8.log +cf97a5aee7b83557f7ab149a5230157f arm6_tp4pp2_090_cap8_seed.log +58dbd55fb5777957a0e2ae1c64fe22ca arm6_tp4pp2_090_hit90_cc16.log +fcb2fd02117cfa89befe4fd5460ebf4f arm6_tp4pp2_090_hit90_cc4.log +f9fd40271de7a4f65680c30f79ef7220 arm6_tp4pp2_090_hit90_cc8.log +16b945669bbea0ba88a5cc0991b8c156 arm6_tp4pp2_090_jitwarm.log +eefd4a27f72f55ffe183d2a39af35699 arm6_tp4pp2_090_progress.log +bd842523b5c435b56081e56d428db13a arm6_tp4pp2_090_warm_cc16.log +bd842523b5c435b56081e56d428db13a arm6_tp4pp2_090_warm_cc4.log +385fc9a8642f183fb320bead6a5b3eff arm6_tp4pp2_090_warm_cc8.log +adf66baeff95e281e516400f08c8eb74 arm6_tp4pp2_090_warm_jit.log +40210b382e0aa2c05b7289035d881cfa arm6_warm_cc1.log +d14e524d30186912be3f17bd1d98a0e6 arm6_warm_cc2.log +75ba4578ebdc405c45b8e6236402fc68 arm6_warm_cc3.log +5ee6109bb6820a33b1896486762e2733 dp_arms_failure_record.md