Quantong Qiu 3694614dd9 feat(p800): add max context length exploration experiment
Add experiments/dsv4_p800_max_context_length/ to find the longest input
context that P800 SGLang INT8 can serve for DeepSeek-V4-Flash-INT8.

- Docker-based server start with dynamic --context-length.
- Single-request, single-output-token probes for each candidate length.
- Records success/failure, server args, and metrics per length.
- Generates results.json and report.md summarizing the max supported length.
- Smoke-tested at 4096 tokens successfully.
2026-07-09 03:30:09 +00:00

2482 lines
96 KiB
JSON

{
"metadata": {
"experiment": "dsv4_p800_sglang",
"run_id": "20260708-051316",
"timestamp": "2026-07-08T05:13:16+00:00",
"model": "/data1/models/DeepSeek-V4-Flash-INT8",
"backend": "sglang",
"engine": "sglang-xpu",
"hardware": "8x Kunlun P800 XPU",
"accelerator": "Kunlun P800 XPU",
"chip": "kunlun_p800",
"script": "experiments/dsv4_p800_sglang/run_bench.sh",
"env": "",
"git_commit": "2c332ff",
"git_dirty": "dirty",
"description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8"
},
"config": {},
"scenarios": [
{
"name": "c32_i512_o256",
"concurrency": 32,
"input_len": 512,
"output_len": 256,
"success": 10,
"failed": 0,
"duration_s": 5.35,
"request_throughput": 1.87,
"input_token_throughput": 470.87,
"output_token_throughput": 262.65,
"total_token_throughput": 733.53,
"accept_length": null,
"latencies": {
"e2e_ms": {
"mean": 3372.781996097183,
"p50": 3498.675397480838,
"p90": 5125.400093087228,
"p95": null,
"p99": 5314.590020017349
},
"ttft_ms": {
"mean": 478.8548628857825,
"p50": 506.4399079710711,
"p90": null,
"p95": null,
"p99": 507.2712521097855
},
"tpot_ms": {
"mean": 22.334078483797782,
"p50": 21.948128755653826,
"p90": null,
"p95": null,
"p99": 29.154062995221466
},
"itl_ms": {
"mean": 20.75981864563044,
"p50": 18.560815330905218,
"p90": null,
"p95": 30.416983005125076,
"p99": 76.09861597185954
}
},
"raw_requests": [
{
"tag": null,
"backend": "sglang",
"dataset_name": "random",
"request_rate": Infinity,
"max_concurrency": 32,
"sharegpt_output_len": null,
"random_input_len": 512,
"random_output_len": 256,
"random_range_ratio": 0.0,
"server_info": {
"model_path": "/data1/models/DeepSeek-V4-Flash-INT8",
"tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8",
"tokenizer_mode": "auto",
"tokenizer_worker_num": 1,
"skip_tokenizer_init": false,
"load_format": "auto",
"model_loader_extra_config": "{}",
"trust_remote_code": true,
"context_length": null,
"is_embedding": false,
"enable_multimodal": null,
"revision": null,
"model_impl": "auto",
"host": "0.0.0.0",
"port": 30000,
"fastapi_root_path": "",
"grpc_mode": false,
"skip_server_warmup": false,
"warmups": null,
"nccl_port": null,
"checkpoint_engine_wait_weights_before_ready": false,
"ssl_keyfile": null,
"ssl_certfile": null,
"ssl_ca_certs": null,
"ssl_keyfile_password": null,
"enable_ssl_refresh": false,
"dtype": "auto",
"quantization": "w8a8_int8",
"quantization_param_path": null,
"kv_cache_dtype": "float16",
"enable_fp32_lm_head": false,
"modelopt_quant": null,
"modelopt_checkpoint_restore_path": null,
"modelopt_checkpoint_save_path": null,
"modelopt_export_path": null,
"quantize_and_serve": false,
"rl_quant_profile": null,
"mem_fraction_static": 0.8,
"max_running_requests": 64,
"max_queued_requests": null,
"max_total_tokens": null,
"chunked_prefill_size": 8192,
"enable_dynamic_chunking": false,
"max_prefill_tokens": 16384,
"prefill_max_requests": null,
"custom_max_context_len": null,
"schedule_policy": "fcfs",
"enable_priority_scheduling": false,
"disable_priority_preemption": false,
"default_priority_value": null,
"abort_on_priority_when_disabled": false,
"schedule_low_priority_values_first": false,
"priority_scheduling_preemption_threshold": 10,
"schedule_conservativeness": 1.0,
"page_size": 256,
"swa_full_tokens_ratio": 0.1,
"disable_hybrid_swa_memory": false,
"radix_eviction_policy": "lru",
"enable_prefill_delayer": false,
"prefill_delayer_max_delay_passes": 30,
"prefill_delayer_token_usage_low_watermark": null,
"prefill_delayer_forward_passes_buckets": null,
"prefill_delayer_wait_seconds_buckets": null,
"device": "cuda",
"tp_size": 8,
"pp_size": 1,
"pp_max_micro_batch_size": null,
"pp_async_batch_depth": 0,
"stream_interval": 1,
"stream_response_default_include_usage": false,
"incremental_streaming_output": false,
"enable_streaming_session": false,
"random_seed": 228424360,
"constrained_json_whitespace_pattern": null,
"constrained_json_disable_any_whitespace": true,
"watchdog_timeout": 3000000.0,
"soft_watchdog_timeout": null,
"dist_timeout": null,
"download_dir": null,
"model_checksum": null,
"base_gpu_id": 0,
"gpu_id_step": 1,
"sleep_on_idle": false,
"use_ray": false,
"custom_sigquit_handler": null,
"log_level": "info",
"log_level_http": null,
"log_requests": false,
"log_requests_level": 2,
"log_requests_format": "text",
"log_requests_target": null,
"uvicorn_access_log_exclude_prefixes": [],
"crash_dump_folder": null,
"show_time_cost": false,
"enable_metrics": true,
"enable_mfu_metrics": false,
"enable_metrics_for_all_schedulers": false,
"tokenizer_metrics_custom_labels_header": "x-custom-labels",
"tokenizer_metrics_allowed_custom_labels": null,
"extra_metric_labels": null,
"bucket_time_to_first_token": null,
"bucket_inter_token_latency": null,
"bucket_e2e_request_latency": null,
"collect_tokens_histogram": false,
"prompt_tokens_buckets": null,
"generation_tokens_buckets": null,
"gc_warning_threshold_secs": 0.0,
"decode_log_interval": 40,
"enable_request_time_stats_logging": true,
"kv_events_config": null,
"enable_trace": false,
"otlp_traces_endpoint": "localhost:4317",
"export_metrics_to_file": false,
"export_metrics_to_file_dir": null,
"api_key": null,
"admin_api_key": null,
"served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8",
"weight_version": "default",
"chat_template": null,
"hf_chat_template_name": null,
"completion_template": null,
"file_storage_path": "sglang_storage",
"enable_cache_report": false,
"reasoning_parser": "deepseek-v4",
"tool_call_parser": "deepseekv4",
"tool_server": null,
"sampling_defaults": "model",
"dp_size": 1,
"load_balance_method": "round_robin",
"attn_cp_size": 1,
"moe_dp_size": 1,
"dist_init_addr": null,
"nnodes": 1,
"node_rank": 0,
"json_model_override_args": "{}",
"preferred_sampling_params": null,
"enable_lora": null,
"enable_lora_overlap_loading": null,
"max_lora_rank": null,
"lora_target_modules": null,
"lora_paths": null,
"max_loaded_loras": null,
"max_loras_per_batch": 8,
"lora_eviction_policy": "lru",
"lora_backend": "csgmv",
"max_lora_chunk_size": 16,
"experts_shared_outer_loras": null,
"attention_backend": "compressed",
"decode_attention_backend": null,
"prefill_attention_backend": null,
"sampling_backend": "flashinfer",
"grammar_backend": "xgrammar",
"mm_attention_backend": null,
"fp8_gemm_runner_backend": "auto",
"fp4_gemm_runner_backend": "auto",
"nsa_prefill_backend": "klxdsa",
"nsa_decode_backend": "klxdsa",
"disable_flashinfer_autotune": false,
"mamba_backend": "triton",
"speculative_algorithm": "EAGLE",
"speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8",
"speculative_draft_model_revision": null,
"speculative_draft_load_format": null,
"speculative_num_steps": 3,
"speculative_eagle_topk": 1,
"speculative_num_draft_tokens": 4,
"speculative_accept_threshold_single": 1.0,
"speculative_accept_threshold_acc": 1.0,
"speculative_token_map": null,
"speculative_attention_mode": "prefill",
"speculative_draft_attention_backend": null,
"speculative_moe_runner_backend": "auto",
"speculative_moe_a2a_backend": null,
"speculative_draft_model_quantization": "w8a8_int8",
"speculative_ngram_min_bfs_breadth": 1,
"speculative_ngram_max_bfs_breadth": 10,
"speculative_ngram_match_type": "BFS",
"speculative_ngram_max_trie_depth": 18,
"speculative_ngram_capacity": 10000000,
"enable_multi_layer_eagle": false,
"ep_size": 8,
"moe_a2a_backend": "none",
"moe_runner_backend": "auto",
"flashinfer_mxfp4_moe_precision": "default",
"enable_hisparse": false,
"enable_flashinfer_allreduce_fusion": false,
"enforce_disable_flashinfer_allreduce_fusion": false,
"enable_aiter_allreduce_fusion": false,
"deepep_mode": "auto",
"ep_num_redundant_experts": 0,
"ep_dispatch_algorithm": null,
"init_expert_location": "trivial",
"enable_eplb": false,
"eplb_algorithm": "auto",
"eplb_rebalance_num_iterations": 1000,
"eplb_rebalance_layers_per_chunk": null,
"eplb_min_rebalancing_utilization_threshold": 1.0,
"expert_distribution_recorder_mode": null,
"expert_distribution_recorder_buffer_size": 1000,
"enable_expert_distribution_metrics": false,
"deepep_config": null,
"moe_dense_tp_size": null,
"elastic_ep_backend": null,
"enable_elastic_expert_backup": false,
"mooncake_ib_device": null,
"max_mamba_cache_size": null,
"mamba_ssm_dtype": null,
"mamba_full_memory_ratio": 0.9,
"mamba_scheduler_strategy": "no_buffer",
"mamba_track_interval": 256,
"linear_attn_backend": "triton",
"linear_attn_decode_backend": null,
"linear_attn_prefill_backend": null,
"enable_hierarchical_cache": false,
"hicache_ratio": 2.0,
"hicache_size": 0,
"hicache_write_policy": "write_through",
"hicache_io_backend": "kernel",
"hicache_mem_layout": "page_first",
"hicache_storage_backend": null,
"hicache_storage_prefetch_policy": "best_effort",
"hicache_storage_backend_extra_config": null,
"hisparse_config": null,
"enable_lmcache": false,
"kt_weight_path": null,
"kt_method": "AMXINT4",
"kt_cpuinfer": null,
"kt_threadpool_count": 2,
"kt_num_gpu_experts": null,
"kt_max_deferred_experts_per_token": null,
"dllm_algorithm": null,
"dllm_algorithm_config": null,
"enable_double_sparsity": false,
"ds_channel_config_path": null,
"ds_heavy_channel_num": 32,
"ds_heavy_token_num": 256,
"ds_heavy_channel_type": "qk",
"ds_sparse_decode_threshold": 4096,
"cpu_offload_gb": 0,
"offload_group_size": -1,
"offload_num_in_group": 1,
"offload_prefetch_step": 1,
"offload_mode": "cpu",
"multi_item_scoring_delimiter": null,
"disable_radix_cache": false,
"cuda_graph_max_bs": 32,
"cuda_graph_bs": [
1,
2,
3,
4,
5,
6,
7,
8,
10,
12,
14,
16,
18,
20,
22,
24,
26,
28,
30,
32
],
"disable_cuda_graph": false,
"disable_cuda_graph_padding": false,
"enable_profile_cuda_graph": false,
"enable_cudagraph_gc": false,
"enable_layerwise_nvtx_marker": false,
"enable_nccl_nvls": false,
"enable_symm_mem": false,
"disable_flashinfer_cutlass_moe_fp4_allgather": false,
"enable_tokenizer_batch_encode": false,
"disable_tokenizer_batch_decode": false,
"disable_outlines_disk_cache": false,
"disable_custom_all_reduce": true,
"enable_mscclpp": false,
"enable_torch_symm_mem": false,
"pre_warm_nccl": false,
"disable_overlap_schedule": false,
"enable_mixed_chunk": false,
"enable_dp_attention": false,
"enable_dp_lm_head": false,
"enable_two_batch_overlap": false,
"enable_single_batch_overlap": false,
"tbo_token_distribution_threshold": 0.48,
"enable_torch_compile": false,
"disable_piecewise_cuda_graph": true,
"enforce_piecewise_cuda_graph": false,
"enable_torch_compile_debug_mode": false,
"torch_compile_max_bs": 32,
"piecewise_cuda_graph_max_tokens": 8192,
"piecewise_cuda_graph_tokens": [
4,
8,
12,
16,
20,
24,
28,
32,
48,
64,
80,
96,
112,
128,
144,
160,
176,
192,
208,
224,
240,
256,
288,
320,
352,
384,
416,
448,
480,
512,
576,
640,
704,
768,
832,
896,
960,
1024,
1280,
1536,
1792,
2048,
2304,
2560,
2816,
3072,
3328,
3584,
3840,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192
],
"piecewise_cuda_graph_compiler": "eager",
"torchao_config": "",
"enable_nan_detection": false,
"enable_p2p_check": false,
"triton_attention_reduce_in_fp32": false,
"triton_attention_num_kv_splits": 8,
"triton_attention_split_tile_size": null,
"num_continuous_decode_steps": 1,
"delete_ckpt_after_loading": false,
"enable_memory_saver": false,
"enable_weights_cpu_backup": false,
"enable_draft_weights_cpu_backup": false,
"allow_auto_truncate": false,
"enable_custom_logit_processor": false,
"flashinfer_mla_disable_ragged": false,
"disable_shared_experts_fusion": true,
"disable_chunked_prefix_cache": false,
"disable_fast_image_processor": false,
"keep_mm_feature_on_device": false,
"enable_return_hidden_states": false,
"enable_return_routed_experts": false,
"enable_return_indexer_topk": false,
"scheduler_recv_interval": 1,
"numa_node": null,
"enable_deterministic_inference": false,
"rl_on_policy_target": null,
"enable_attn_tp_input_scattered": false,
"gc_threshold": null,
"enable_nsa_prefill_context_parallel": false,
"nsa_prefill_cp_mode": "in-seq-split",
"enable_fused_qk_norm_rope": false,
"enable_precise_embedding_interpolation": false,
"enable_fused_moe_sum_all_reduce": false,
"enable_prefill_context_parallel": false,
"prefill_cp_mode": "in-seq-split",
"enable_dynamic_batch_tokenizer": false,
"dynamic_batch_tokenizer_batch_size": 32,
"dynamic_batch_tokenizer_batch_timeout": 0.002,
"debug_tensor_dump_output_folder": null,
"debug_tensor_dump_layers": null,
"debug_tensor_dump_input_file": null,
"debug_tensor_dump_inject": false,
"disaggregation_mode": "null",
"disaggregation_transfer_backend": "mooncake",
"disaggregation_bootstrap_port": 8998,
"disaggregation_ib_device": null,
"disaggregation_decode_enable_offload_kvcache": false,
"num_reserved_decode_tokens": 512,
"disaggregation_decode_polling_interval": 1,
"encoder_only": false,
"language_only": false,
"encoder_transfer_backend": "zmq_to_scheduler",
"encoder_urls": [],
"enable_adaptive_dispatch_to_encoder": false,
"enable_over_encoding": false,
"enable_kv_mirror": false,
"custom_weight_loader": [],
"weight_loader_disable_mmap": false,
"remote_instance_weight_loader_seed_instance_ip": null,
"remote_instance_weight_loader_seed_instance_service_port": null,
"remote_instance_weight_loader_send_weights_group_ports": null,
"remote_instance_weight_loader_backend": "nccl",
"remote_instance_weight_loader_start_seed_via_transfer_engine": false,
"engine_info_bootstrap_port": 6789,
"modelexpress_config": null,
"enable_pdmux": false,
"pdmux_config_path": null,
"sm_group_num": 8,
"enable_broadcast_mm_inputs_process": false,
"enable_prefix_mm_cache": false,
"mm_enable_dp_encoder": false,
"mm_process_config": {},
"limit_mm_data_per_request": null,
"enable_mm_global_cache": false,
"decrypted_config_file": null,
"decrypted_draft_config_file": null,
"forward_hooks": null,
"index_cache_topk_freq": 1,
"index_cache_pattern": null,
"status": "ready",
"max_total_num_tokens": 1593344,
"max_req_input_len": 1048570,
"internal_states": [
{
"model_path": "/data1/models/DeepSeek-V4-Flash-INT8",
"tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8",
"tokenizer_mode": "auto",
"tokenizer_worker_num": 1,
"skip_tokenizer_init": false,
"load_format": "auto",
"model_loader_extra_config": "{}",
"trust_remote_code": true,
"context_length": 1048576,
"is_embedding": false,
"enable_multimodal": null,
"revision": null,
"model_impl": "auto",
"host": "0.0.0.0",
"port": 30000,
"fastapi_root_path": "",
"grpc_mode": false,
"skip_server_warmup": false,
"warmups": null,
"nccl_port": null,
"checkpoint_engine_wait_weights_before_ready": false,
"ssl_keyfile": null,
"ssl_certfile": null,
"ssl_ca_certs": null,
"ssl_keyfile_password": null,
"enable_ssl_refresh": false,
"dtype": "auto",
"quantization": "w8a8_int8",
"quantization_param_path": null,
"kv_cache_dtype": "float16",
"enable_fp32_lm_head": false,
"modelopt_quant": null,
"modelopt_checkpoint_restore_path": null,
"modelopt_checkpoint_save_path": null,
"modelopt_export_path": null,
"quantize_and_serve": false,
"rl_quant_profile": null,
"mem_fraction_static": 0.8,
"max_running_requests": 64,
"max_queued_requests": null,
"max_total_tokens": null,
"chunked_prefill_size": 8192,
"enable_dynamic_chunking": false,
"max_prefill_tokens": 16384,
"prefill_max_requests": null,
"custom_max_context_len": null,
"schedule_policy": "fcfs",
"enable_priority_scheduling": false,
"disable_priority_preemption": false,
"default_priority_value": null,
"abort_on_priority_when_disabled": false,
"schedule_low_priority_values_first": false,
"priority_scheduling_preemption_threshold": 10,
"schedule_conservativeness": 1.0,
"page_size": 256,
"swa_full_tokens_ratio": 0.1,
"disable_hybrid_swa_memory": false,
"radix_eviction_policy": "lru",
"enable_prefill_delayer": false,
"prefill_delayer_max_delay_passes": 30,
"prefill_delayer_token_usage_low_watermark": null,
"prefill_delayer_forward_passes_buckets": null,
"prefill_delayer_wait_seconds_buckets": null,
"device": "cuda",
"tp_size": 8,
"pp_size": 1,
"pp_max_micro_batch_size": 64,
"pp_async_batch_depth": 0,
"stream_interval": 1,
"stream_response_default_include_usage": false,
"incremental_streaming_output": false,
"enable_streaming_session": false,
"random_seed": 228424360,
"constrained_json_whitespace_pattern": null,
"constrained_json_disable_any_whitespace": true,
"watchdog_timeout": 3000000.0,
"soft_watchdog_timeout": null,
"dist_timeout": null,
"download_dir": null,
"model_checksum": null,
"base_gpu_id": 0,
"gpu_id_step": 1,
"sleep_on_idle": false,
"use_ray": false,
"custom_sigquit_handler": null,
"log_level": "info",
"log_level_http": null,
"log_requests": false,
"log_requests_level": 2,
"log_requests_format": "text",
"log_requests_target": null,
"uvicorn_access_log_exclude_prefixes": [],
"crash_dump_folder": null,
"show_time_cost": false,
"enable_metrics": true,
"enable_mfu_metrics": false,
"enable_metrics_for_all_schedulers": false,
"tokenizer_metrics_custom_labels_header": "x-custom-labels",
"tokenizer_metrics_allowed_custom_labels": null,
"extra_metric_labels": null,
"bucket_time_to_first_token": null,
"bucket_inter_token_latency": null,
"bucket_e2e_request_latency": null,
"collect_tokens_histogram": false,
"prompt_tokens_buckets": null,
"generation_tokens_buckets": null,
"gc_warning_threshold_secs": 0.0,
"decode_log_interval": 40,
"enable_request_time_stats_logging": true,
"kv_events_config": null,
"enable_trace": false,
"otlp_traces_endpoint": "localhost:4317",
"export_metrics_to_file": false,
"export_metrics_to_file_dir": null,
"api_key": null,
"admin_api_key": null,
"served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8",
"weight_version": "default",
"chat_template": null,
"hf_chat_template_name": null,
"completion_template": null,
"file_storage_path": "sglang_storage",
"enable_cache_report": false,
"reasoning_parser": "deepseek-v4",
"tool_call_parser": "deepseekv4",
"tool_server": null,
"sampling_defaults": "model",
"dp_size": 1,
"load_balance_method": "round_robin",
"attn_cp_size": 1,
"moe_dp_size": 1,
"dist_init_addr": null,
"nnodes": 1,
"node_rank": 0,
"json_model_override_args": "{}",
"preferred_sampling_params": null,
"enable_lora": null,
"enable_lora_overlap_loading": null,
"max_lora_rank": null,
"lora_target_modules": null,
"lora_paths": null,
"max_loaded_loras": null,
"max_loras_per_batch": 8,
"lora_eviction_policy": "lru",
"lora_backend": "csgmv",
"max_lora_chunk_size": 16,
"experts_shared_outer_loras": null,
"attention_backend": "compressed",
"decode_attention_backend": "compressed",
"prefill_attention_backend": "compressed",
"sampling_backend": "flashinfer",
"grammar_backend": "xgrammar",
"mm_attention_backend": null,
"fp8_gemm_runner_backend": "auto",
"fp4_gemm_runner_backend": "auto",
"nsa_prefill_backend": "klxdsa",
"nsa_decode_backend": "klxdsa",
"disable_flashinfer_autotune": false,
"mamba_backend": "triton",
"speculative_algorithm": "EAGLE",
"speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8",
"speculative_draft_model_revision": null,
"speculative_draft_load_format": null,
"speculative_num_steps": 3,
"speculative_eagle_topk": 1,
"speculative_num_draft_tokens": 4,
"speculative_accept_threshold_single": 1.0,
"speculative_accept_threshold_acc": 1.0,
"speculative_token_map": null,
"speculative_attention_mode": "prefill",
"speculative_draft_attention_backend": null,
"speculative_moe_runner_backend": "auto",
"speculative_moe_a2a_backend": null,
"speculative_draft_model_quantization": "w8a8_int8",
"speculative_ngram_min_bfs_breadth": 1,
"speculative_ngram_max_bfs_breadth": 10,
"speculative_ngram_match_type": "BFS",
"speculative_ngram_max_trie_depth": 18,
"speculative_ngram_capacity": 10000000,
"enable_multi_layer_eagle": false,
"ep_size": 8,
"moe_a2a_backend": "none",
"moe_runner_backend": "auto",
"flashinfer_mxfp4_moe_precision": "default",
"enable_hisparse": false,
"enable_flashinfer_allreduce_fusion": false,
"enforce_disable_flashinfer_allreduce_fusion": false,
"enable_aiter_allreduce_fusion": false,
"deepep_mode": "auto",
"ep_num_redundant_experts": 0,
"ep_dispatch_algorithm": null,
"init_expert_location": "trivial",
"enable_eplb": false,
"eplb_algorithm": "auto",
"eplb_rebalance_num_iterations": 1000,
"eplb_rebalance_layers_per_chunk": null,
"eplb_min_rebalancing_utilization_threshold": 1.0,
"expert_distribution_recorder_mode": null,
"expert_distribution_recorder_buffer_size": 1000,
"enable_expert_distribution_metrics": false,
"deepep_config": null,
"moe_dense_tp_size": null,
"elastic_ep_backend": null,
"enable_elastic_expert_backup": false,
"mooncake_ib_device": null,
"max_mamba_cache_size": null,
"mamba_ssm_dtype": null,
"mamba_full_memory_ratio": 0.9,
"mamba_scheduler_strategy": "no_buffer",
"mamba_track_interval": 256,
"linear_attn_backend": "triton",
"linear_attn_decode_backend": null,
"linear_attn_prefill_backend": null,
"enable_hierarchical_cache": false,
"hicache_ratio": 2.0,
"hicache_size": 0,
"hicache_write_policy": "write_through",
"hicache_io_backend": "kernel",
"hicache_mem_layout": "page_first",
"hicache_storage_backend": null,
"hicache_storage_prefetch_policy": "best_effort",
"hicache_storage_backend_extra_config": null,
"hisparse_config": null,
"enable_lmcache": false,
"kt_weight_path": null,
"kt_method": "AMXINT4",
"kt_cpuinfer": null,
"kt_threadpool_count": 2,
"kt_num_gpu_experts": null,
"kt_max_deferred_experts_per_token": null,
"dllm_algorithm": null,
"dllm_algorithm_config": null,
"enable_double_sparsity": false,
"ds_channel_config_path": null,
"ds_heavy_channel_num": 32,
"ds_heavy_token_num": 256,
"ds_heavy_channel_type": "qk",
"ds_sparse_decode_threshold": 4096,
"cpu_offload_gb": 0,
"offload_group_size": -1,
"offload_num_in_group": 1,
"offload_prefetch_step": 1,
"offload_mode": "cpu",
"multi_item_scoring_delimiter": null,
"disable_radix_cache": false,
"cuda_graph_max_bs": 32,
"cuda_graph_bs": [
1,
2,
3,
4,
5,
6,
7,
8,
10,
12,
14,
16,
18,
20,
22,
24,
26,
28,
30,
32
],
"disable_cuda_graph": false,
"disable_cuda_graph_padding": false,
"enable_profile_cuda_graph": false,
"enable_cudagraph_gc": false,
"enable_layerwise_nvtx_marker": false,
"enable_nccl_nvls": false,
"enable_symm_mem": false,
"disable_flashinfer_cutlass_moe_fp4_allgather": false,
"enable_tokenizer_batch_encode": false,
"disable_tokenizer_batch_decode": false,
"disable_outlines_disk_cache": false,
"disable_custom_all_reduce": true,
"enable_mscclpp": false,
"enable_torch_symm_mem": false,
"pre_warm_nccl": false,
"disable_overlap_schedule": false,
"enable_mixed_chunk": false,
"enable_dp_attention": false,
"enable_dp_lm_head": false,
"enable_two_batch_overlap": false,
"enable_single_batch_overlap": false,
"tbo_token_distribution_threshold": 0.48,
"enable_torch_compile": false,
"disable_piecewise_cuda_graph": true,
"enforce_piecewise_cuda_graph": false,
"enable_torch_compile_debug_mode": false,
"torch_compile_max_bs": 32,
"piecewise_cuda_graph_max_tokens": 8192,
"piecewise_cuda_graph_tokens": [
4,
8,
12,
16,
20,
24,
28,
32,
48,
64,
80,
96,
112,
128,
144,
160,
176,
192,
208,
224,
240,
256,
288,
320,
352,
384,
416,
448,
480,
512,
576,
640,
704,
768,
832,
896,
960,
1024,
1280,
1536,
1792,
2048,
2304,
2560,
2816,
3072,
3328,
3584,
3840,
4096,
4608,
5120,
5632,
6144,
6656,
7168,
7680,
8192
],
"piecewise_cuda_graph_compiler": "eager",
"torchao_config": "",
"enable_nan_detection": false,
"enable_p2p_check": false,
"triton_attention_reduce_in_fp32": false,
"triton_attention_num_kv_splits": 8,
"triton_attention_split_tile_size": null,
"num_continuous_decode_steps": 1,
"delete_ckpt_after_loading": false,
"enable_memory_saver": false,
"enable_weights_cpu_backup": false,
"enable_draft_weights_cpu_backup": false,
"allow_auto_truncate": false,
"enable_custom_logit_processor": false,
"flashinfer_mla_disable_ragged": false,
"disable_shared_experts_fusion": true,
"disable_chunked_prefix_cache": true,
"disable_fast_image_processor": false,
"keep_mm_feature_on_device": false,
"enable_return_hidden_states": false,
"enable_return_routed_experts": false,
"enable_return_indexer_topk": false,
"scheduler_recv_interval": 1,
"numa_node": null,
"enable_deterministic_inference": false,
"rl_on_policy_target": null,
"enable_attn_tp_input_scattered": false,
"gc_threshold": null,
"enable_nsa_prefill_context_parallel": false,
"nsa_prefill_cp_mode": "in-seq-split",
"enable_fused_qk_norm_rope": false,
"enable_precise_embedding_interpolation": false,
"enable_fused_moe_sum_all_reduce": false,
"enable_prefill_context_parallel": false,
"prefill_cp_mode": "in-seq-split",
"enable_dynamic_batch_tokenizer": false,
"dynamic_batch_tokenizer_batch_size": 32,
"dynamic_batch_tokenizer_batch_timeout": 0.002,
"debug_tensor_dump_output_folder": null,
"debug_tensor_dump_layers": null,
"debug_tensor_dump_input_file": null,
"debug_tensor_dump_inject": false,
"disaggregation_mode": "null",
"disaggregation_transfer_backend": "mooncake",
"disaggregation_bootstrap_port": 8998,
"disaggregation_ib_device": null,
"disaggregation_decode_enable_offload_kvcache": false,
"num_reserved_decode_tokens": 512,
"disaggregation_decode_polling_interval": 1,
"encoder_only": false,
"language_only": false,
"encoder_transfer_backend": "zmq_to_scheduler",
"encoder_urls": [],
"enable_adaptive_dispatch_to_encoder": false,
"enable_over_encoding": false,
"enable_kv_mirror": false,
"custom_weight_loader": [],
"weight_loader_disable_mmap": false,
"remote_instance_weight_loader_seed_instance_ip": null,
"remote_instance_weight_loader_seed_instance_service_port": null,
"remote_instance_weight_loader_send_weights_group_ports": null,
"remote_instance_weight_loader_backend": "nccl",
"remote_instance_weight_loader_start_seed_via_transfer_engine": false,
"engine_info_bootstrap_port": 6789,
"modelexpress_config": null,
"enable_pdmux": false,
"pdmux_config_path": null,
"sm_group_num": 8,
"enable_broadcast_mm_inputs_process": false,
"enable_prefix_mm_cache": false,
"mm_enable_dp_encoder": false,
"mm_process_config": {},
"limit_mm_data_per_request": null,
"enable_mm_global_cache": false,
"decrypted_config_file": null,
"decrypted_draft_config_file": null,
"forward_hooks": null,
"index_cache_topk_freq": 1,
"index_cache_pattern": null,
"use_mla_backend": false,
"_mx_config_cache": {},
"_draft_pool_config": {
"full_max_total_num_tokens": 1593344,
"swa_max_total_num_tokens": 159232
},
"last_gen_throughput": 304.3209280340091,
"memory_usage": {
"weight": 37.94,
"kvcache": 0,
"token_capacity": 1593344,
"graph": 1.2
},
"effective_max_running_requests_per_dp": 64,
"avg_spec_accept_length": 2.8933333333333335
}
],
"version": "0.5.10+ee6948cf8"
},
"duration": 5.34541754698148,
"completed": 10,
"total_input_tokens": 2517,
"total_input_text_tokens": 2517,
"total_input_vision_tokens": 0,
"total_output_tokens": 1404,
"total_output_tokens_retokenized": 1404,
"request_throughput": 1.870761247762007,
"input_throughput": 470.8706060616972,
"output_throughput": 262.6548791857858,
"total_throughput": 733.525485247483,
"mean_e2e_latency_ms": 3372.781996097183,
"median_e2e_latency_ms": 3498.675397480838,
"std_e2e_latency_ms": 1426.9563106485741,
"p90_e2e_latency_ms": 5125.400093087228,
"p99_e2e_latency_ms": 5314.590020017349,
"mean_ttft_ms": 478.8548628857825,
"median_ttft_ms": 506.4399079710711,
"std_ttft_ms": 82.89120389713447,
"p99_ttft_ms": 507.2712521097855,
"mean_tpot_ms": 22.334078483797782,
"median_tpot_ms": 21.948128755653826,
"std_tpot_ms": 3.1100348959353536,
"p99_tpot_ms": 29.154062995221466,
"mean_itl_ms": 20.75981864563044,
"median_itl_ms": 18.560815330905218,
"std_itl_ms": 11.544034313142475,
"p95_itl_ms": 30.416983005125076,
"p99_itl_ms": 76.09861597185954,
"concurrency": 6.309669855447998,
"accept_length": 2.8933333333333335,
"max_output_tokens_per_s": 396.0,
"max_concurrent_requests": 10,
"input_lens": [
38,
236,
397,
73,
256,
394,
204,
134,
336,
449
],
"output_lens": [
145,
130,
205,
72,
238,
253,
135,
26,
179,
21
],
"ttfts": [
0.23018694896018133,
0.5072830729768611,
0.5071517300093547,
0.506980306992773,
0.5067555939895101,
0.5065123789827339,
0.5063674369594082,
0.5061119620222598,
0.5057157859555446,
0.5054834120091982
],
"itls": [
[
0.1265941219899105,
0.1265941219899105,
0.1265941219899105,
0.1265941219899105,
0.03003738750703633,
0.03003738750703633,
0.020075495665272076,
0.020075495665272076,
0.020075495665272076,
0.020034517995857943,
0.020034517995857943,
0.020034517995857943,
0.020025947674488027,
0.020025947674488027,
0.020025947674488027,
0.01982437400147319,
0.01982437400147319,
0.01982437400147319,
0.030580375489080325,
0.030580375489080325,
0.015474200758035295,
0.015474200758035295,
0.015474200758035295,
0.015474200758035295,
0.02975128698744811,
0.02975128698744811,
0.018891388666816056,
0.018891388666816056,
0.018891388666816056,
0.030985667331454653,
0.030985667331454653,
0.030985667331454653,
0.01450748200295493,
0.01450748200295493,
0.01450748200295493,
0.01450748200295493,
0.02843167950049974,
0.02843167950049974,
0.019097025661418836,
0.019097025661418836,
0.019097025661418836,
0.019033391009240102,
0.019033391009240102,
0.019033391009240102,
0.02840644898242317,
0.02840644898242317,
0.0287785355176311,
0.0287785355176311,
0.01427662099013105,
0.01427662099013105,
0.01427662099013105,
0.01427662099013105,
0.028972148516913876,
0.028972148516913876,
0.0198178769981799,
0.0198178769981799,
0.0198178769981799,
0.019058960994395118,
0.019058960994395118,
0.019058960994395118,
0.014342486509121954,
0.014342486509121954,
0.014342486509121954,
0.014342486509121954,
0.028646462480537593,
0.028646462480537593,
0.01456742275331635,
0.01456742275331635,
0.01456742275331635,
0.01456742275331635,
0.01943501700103904,
0.01943501700103904,
0.01943501700103904,
0.02791283550322987,
0.02791283550322987,
0.013792212252155878,
0.013792212252155878,
0.013792212252155878,
0.013792212252155878,
0.018324121988068025,
0.018324121988068025,
0.018324121988068025,
0.018392249670190115,
0.018392249670190115,
0.018392249670190115,
0.018568886327557266,
0.018568886327557266,
0.018568886327557266,
0.018351545673795044,
0.018351545673795044,
0.018351545673795044,
0.02752352401148528,
0.02752352401148528,
0.013863828251487575,
0.013863828251487575,
0.013863828251487575,
0.013863828251487575,
0.018301511649042368,
0.018301511649042368,
0.018301511649042368,
0.027680501516442746,
0.027680501516442746,
0.01841827033786103,
0.01841827033786103,
0.01841827033786103,
0.01374593649234157,
0.01374593649234157,
0.01374593649234157,
0.01374593649234157,
0.018055193999316543,
0.018055193999316543,
0.018055193999316543,
0.014001160496263765,
0.014001160496263765,
0.014001160496263765,
0.014001160496263765,
0.018516544679490227,
0.018516544679490227,
0.018516544679490227,
0.018641953337161492,
0.018641953337161492,
0.018641953337161492,
0.01858119231959184,
0.01858119231959184,
0.01858119231959184,
0.018560895676879834,
0.018560895676879834,
0.018560895676879834,
0.027825927507365122,
0.027825927507365122,
0.027690790477208793,
0.027690790477208793,
0.014033740997547284,
0.014033740997547284,
0.014033740997547284,
0.014033740997547284,
0.028556485020089895,
0.028556485020089895,
0.018292231000183772,
0.018292231000183772,
0.018292231000183772,
0.03276129499621069,
0.03276129499621069,
0.03276129499621069
],
[
0.114176959003089,
0.114176959003089,
0.02001935833444198,
0.02001935833444198,
0.02001935833444198,
0.020075112329019856,
0.020075112329019856,
0.020075112329019856,
0.02003466134192422,
0.02003466134192422,
0.02003466134192422,
0.020026450996131945,
0.020026450996131945,
0.020026450996131945,
0.02973513101460412,
0.02973513101460412,
0.030412352993153036,
0.030412352993153036,
0.015535941754933447,
0.015535941754933447,
0.015535941754933447,
0.015535941754933447,
0.02982658799737692,
0.02982658799737692,
0.018890631986626733,
0.018890631986626733,
0.018890631986626733,
0.03098562400555238,
0.03098562400555238,
0.03098562400555238,
0.019343592672764014,
0.019343592672764014,
0.019343592672764014,
0.018950106323851895,
0.018950106323851895,
0.018950106323851895,
0.019099269004072994,
0.019099269004072994,
0.019099269004072994,
0.019034251008027542,
0.019034251008027542,
0.019034251008027542,
0.01420391199644655,
0.01420391199644655,
0.01420391199644655,
0.01420391199644655,
0.01439423524425365,
0.01439423524425365,
0.01439423524425365,
0.01439423524425365,
0.028566046996274963,
0.028566046996274963,
0.01930476234216864,
0.01930476234216864,
0.01930476234216864,
0.019809113660206396,
0.019809113660206396,
0.019809113660206396,
0.014296345761977136,
0.014296345761977136,
0.014296345761977136,
0.014296345761977136,
0.02867897297255695,
0.02867897297255695,
0.019099251677592594,
0.019099251677592594,
0.019099251677592594,
0.019382666000941146,
0.019382666000941146,
0.019382666000941146,
0.02924422750947997,
0.02924422750947997,
0.027913359983358532,
0.027913359983358532,
0.013784522248897702,
0.013784522248897702,
0.013784522248897702,
0.013784522248897702,
0.02746082251542248,
0.02746082251542248,
0.01840783333561073,
0.01840783333561073,
0.01840783333561073,
0.01856821965581427,
0.01856821965581427,
0.01856821965581427,
0.01835258233283336,
0.01835258233283336,
0.01835258233283336,
0.018347706005442888,
0.018347706005442888,
0.018347706005442888,
0.018484461004845798,
0.018484461004845798,
0.018484461004845798,
0.018302411655895412,
0.018302411655895412,
0.018302411655895412,
0.027679786493536085,
0.027679786493536085,
0.027627180505078286,
0.027627180505078286,
0.01375247175747063,
0.01375247175747063,
0.01375247175747063,
0.01375247175747063,
0.0270681754918769,
0.0270681754918769,
0.01866779732517898,
0.01866779732517898,
0.01866779732517898,
0.018516544679490227,
0.018516544679490227,
0.018516544679490227,
0.013980599993374199,
0.013980599993374199,
0.013980599993374199,
0.013980599993374199,
0.027871913509443402,
0.027871913509443402,
0.018560129004375387,
0.018560129004375387,
0.018560129004375387,
0.018560815330905218,
0.018560815330905218,
0.018560815330905218,
0.027690300485119224,
0.027690300485119224,
0.05595995101612061
],
[
0.07610034267418087,
0.07610034267418087,
0.07610034267418087,
0.02001753800626223,
0.02001753800626223,
0.02001753800626223,
0.020074322334645938,
0.020074322334645938,
0.020074322334645938,
0.015025673492345959,
0.015025673492345959,
0.015025673492345959,
0.015025673492345959,
0.03003843649639748,
0.03003843649639748,
0.01982440067998444,
0.01982440067998444,
0.01982440067998444,
0.02034523299274345,
0.02034523299274345,
0.02034523299274345,
0.030977007001638412,
0.030977007001638412,
0.029805967991705984,
0.029805967991705984,
0.014167338755214587,
0.014167338755214587,
0.014167338755214587,
0.014167338755214587,
0.046478810982080176,
0.046478810982080176,
0.01934337633429095,
0.01934337633429095,
0.01934337633429095,
0.0189525830016161,
0.0189525830016161,
0.0189525830016161,
0.014323239010991529,
0.014323239010991529,
0.014323239010991529,
0.014323239010991529,
0.028550161485327408,
0.028550161485327408,
0.018938299675937742,
0.018938299675937742,
0.018938299675937742,
0.019176949999139953,
0.019176949999139953,
0.019176949999139953,
0.028566726978169754,
0.028566726978169754,
0.028956993512110785,
0.028956993512110785,
0.014857275251415558,
0.014857275251415558,
0.014857275251415558,
0.014857275251415558,
0.028592451504664496,
0.028592451504664496,
0.01912001532036811,
0.01912001532036811,
0.01912001532036811,
0.019098964675019186,
0.019098964675019186,
0.019098964675019186,
0.014537554743583314,
0.014537554743583314,
0.014537554743583314,
0.014537554743583314,
0.029245732497656718,
0.029245732497656718,
0.018609130017769832,
0.018609130017769832,
0.018609130017769832,
0.018378569666917127,
0.018378569666917127,
0.018378569666917127,
0.027461227495223284,
0.027461227495223284,
0.013806014991132542,
0.013806014991132542,
0.013806014991132542,
0.013806014991132542,
0.013926397252362221,
0.013926397252362221,
0.013926397252362221,
0.013926397252362221,
0.01835247567699601,
0.01835247567699601,
0.01835247567699601,
0.01834764933058371,
0.01834764933058371,
0.01834764933058371,
0.013863673244486563,
0.013863673244486563,
0.013863673244486563,
0.013863673244486563,
0.027453012502519414,
0.027453012502519414,
0.018453627669562895,
0.018453627669562895,
0.018453627669562895,
0.018418410327285528,
0.018418410327285528,
0.018418410327285528,
0.027505588514031842,
0.027505588514031842,
0.013533967750845477,
0.013533967750845477,
0.013533967750845477,
0.013533967750845477,
0.014000577997649089,
0.014000577997649089,
0.014000577997649089,
0.014000577997649089,
0.018516601334946852,
0.018516601334946852,
0.018516601334946852,
0.018641606671735644,
0.018641606671735644,
0.018641606671735644,
0.013935844239313155,
0.013935844239313155,
0.013935844239313155,
0.013935844239313155,
0.02784075849922374,
0.02784075849922374,
0.018561025344145794,
0.018561025344145794,
0.018561025344145794,
0.01846017032706489,
0.01846017032706489,
0.01846017032706489,
0.02801839599851519,
0.02801839599851519,
0.014315253254608251,
0.014315253254608251,
0.014315253254608251,
0.014315253254608251,
0.013719310751184821,
0.013719310751184821,
0.013719310751184821,
0.013719310751184821,
0.032736671001960836,
0.032736671001960836,
0.032736671001960836,
0.01810533132326479,
0.01810533132326479,
0.01810533132326479,
0.025817634508712217,
0.025817634508712217,
0.017274240672122687,
0.017274240672122687,
0.017274240672122687,
0.012808780491468497,
0.012808780491468497,
0.012808780491468497,
0.012808780491468497,
0.017103748007987935,
0.017103748007987935,
0.017103748007987935,
0.0260774539783597,
0.0260774539783597,
0.012770477012963966,
0.012770477012963966,
0.012770477012963966,
0.012770477012963966,
0.012633724749321118,
0.012633724749321118,
0.012633724749321118,
0.012633724749321118,
0.016969308664556593,
0.016969308664556593,
0.016969308664556593,
0.017546184652019292,
0.017546184652019292,
0.017546184652019292,
0.02627176101668738,
0.02627176101668738,
0.01677496867099156,
0.01677496867099156,
0.01677496867099156,
0.012476857242290862,
0.012476857242290862,
0.012476857242290862,
0.012476857242290862,
0.016706993997407455,
0.016706993997407455,
0.016706993997407455,
0.02511487199808471,
0.02511487199808471,
0.012620112000149675,
0.012620112000149675,
0.012620112000149675,
0.012620112000149675,
0.013204801754909568,
0.013204801754909568,
0.013204801754909568,
0.013204801754909568,
0.016931711327439796,
0.016931711327439796,
0.016931711327439796,
0.025843739014817402,
0.025843739014817402
],
[
0.07609820265012483,
0.07609820265012483,
0.07609820265012483,
0.020024441337833803,
0.020024441337833803,
0.020024441337833803,
0.015056459509651177,
0.015056459509651177,
0.015056459509651177,
0.015056459509651177,
0.03005173648125492,
0.03005173648125492,
0.015019418249721639,
0.015019418249721639,
0.015019418249721639,
0.015019418249721639,
0.019824344334968675,
0.019824344334968675,
0.019824344334968675,
0.020387480326462537,
0.020387480326462537,
0.020387480326462537,
0.020631657680496573,
0.020631657680496573,
0.020631657680496573,
0.01487621849810239,
0.01487621849810239,
0.01487621849810239,
0.01487621849810239,
0.028335707494989038,
0.028335707494989038,
0.023239640504471026,
0.023239640504471026,
0.023239640504471026,
0.023239640504471026,
0.0193434563310196,
0.0193434563310196,
0.0193434563310196,
0.028431184502551332,
0.028431184502551332,
0.014322734248708002,
0.014322734248708002,
0.014322734248708002,
0.014322734248708002,
0.01903337399320056,
0.01903337399320056,
0.01903337399320056,
0.01893743300267185,
0.01893743300267185,
0.01893743300267185,
0.01917593334413444,
0.01917593334413444,
0.01917593334413444,
0.028568201989401132,
0.028568201989401132,
0.01930442900629714,
0.01930442900629714,
0.01930442900629714,
0.014869755497784354,
0.014869755497784354,
0.014869755497784354,
0.014869755497784354,
0.028589651483343914,
0.028589651483343914,
0.019121012010145932,
0.019121012010145932,
0.019121012010145932,
0.01909933132507528,
0.01909933132507528,
0.01909933132507528,
0.0582002800074406
],
[
0.07609824265819043,
0.07609824265819043,
0.07609824265819043,
0.015011793249868788,
0.015011793249868788,
0.015011793249868788,
0.015011793249868788,
0.020074732679252822,
0.020074732679252822,
0.020074732679252822,
0.015025845990749076,
0.015025845990749076,
0.015025845990749076,
0.015025845990749076,
0.030038921511732042,
0.030038921511732042,
0.019824267326233286,
0.019824267326233286,
0.019824267326233286,
0.020395867332505684,
0.020395867332505684,
0.020395867332505684,
0.03094582151970826,
0.03094582151970826,
0.014877470748615451,
0.014877470748615451,
0.014877470748615451,
0.014877470748615451,
0.014167653993354179,
0.014167653993354179,
0.014167653993354179,
0.014167653993354179,
0.030986197331609826,
0.030986197331609826,
0.030986197331609826,
0.01934319967404008,
0.01934319967404008,
0.01934319967404008,
0.014215631992556155,
0.014215631992556155,
0.014215631992556155,
0.014215631992556155,
0.02864447850151919,
0.02864447850151919,
0.01903398767656957,
0.01903398767656957,
0.01903398767656957,
0.018937196330322575,
0.018937196330322575,
0.018937196330322575,
0.028763925016392022,
0.028763925016392022,
0.02856709697516635,
0.02856709697516635,
0.014478914003120735,
0.014478914003120735,
0.014478914003120735,
0.014478914003120735,
0.029714685515500605,
0.029714685515500605,
0.019061427660441648,
0.019061427660441648,
0.019061427660441648,
0.019120605332621683,
0.019120605332621683,
0.019120605332621683,
0.014324548756121658,
0.014324548756121658,
0.014324548756121658,
0.014324548756121658,
0.0290751994762104,
0.0290751994762104,
0.019497701665386558,
0.019497701665386558,
0.019497701665386558,
0.01860897334214921,
0.01860897334214921,
0.01860897334214921,
0.02756879449589178,
0.02756879449589178,
0.02748415298992768,
0.02748415298992768,
0.01379515475127846,
0.01379515475127846,
0.01379515475127846,
0.01379515475127846,
0.02785251452587545,
0.02785251452587545,
0.018352062324993312,
0.018352062324993312,
0.018352062324993312,
0.01834822599388038,
0.01834822599388038,
0.01834822599388038,
0.01386378325696569,
0.01386378325696569,
0.01386378325696569,
0.01386378325696569,
0.02745306250290014,
0.02745306250290014,
0.018453344334072124,
0.018453344334072124,
0.018453344334072124,
0.018418603654329974,
0.018418603654329974,
0.018418603654329974,
0.02750637350254692,
0.02750637350254692,
0.027067835500929505,
0.027067835500929505,
0.014000943003338762,
0.014000943003338762,
0.014000943003338762,
0.014000943003338762,
0.027774896996561438,
0.027774896996561438,
0.01864148332970217,
0.01864148332970217,
0.01864148332970217,
0.018581362344169367,
0.018581362344169367,
0.018581362344169367,
0.013920626748586074,
0.013920626748586074,
0.013920626748586074,
0.013920626748586074,
0.027841037983307615,
0.027841037983307615,
0.018460453681958217,
0.018460453681958217,
0.018460453681958217,
0.018680113988618057,
0.018680113988618057,
0.018680113988618057,
0.028647751518292353,
0.028647751518292353,
0.02743947648559697,
0.02743947648559697,
0.024545218504499644,
0.024545218504499644,
0.024545218504499644,
0.024545218504499644,
0.027157786476891488,
0.027157786476891488,
0.017211266347051907,
0.017211266347051907,
0.017211266347051907,
0.017274483999547858,
0.017274483999547858,
0.017274483999547858,
0.012808570492779836,
0.012808570492779836,
0.012808570492779836,
0.012808570492779836,
0.02565537701593712,
0.02565537701593712,
0.017431950002598267,
0.017431950002598267,
0.017431950002598267,
0.016987925317759316,
0.016987925317759316,
0.016987925317759316,
0.025259989517508075,
0.025259989517508075,
0.012736934004351497,
0.012736934004351497,
0.012736934004351497,
0.012736934004351497,
0.013151180988643318,
0.013151180988643318,
0.013151180988643318,
0.013151180988643318,
0.01752863434376195,
0.01752863434376195,
0.01752863434376195,
0.01677370866915832,
0.01677370866915832,
0.01677370866915832,
0.01247714224155061,
0.01247714224155061,
0.01247714224155061,
0.01247714224155061,
0.025057951017515734,
0.025057951017515734,
0.016742911325612415,
0.016742911325612415,
0.016742911325612415,
0.016827172657940537,
0.016827172657940537,
0.016827172657940537,
0.0176063990026402,
0.0176063990026402,
0.0176063990026402,
0.01693138466604675,
0.01693138466604675,
0.01693138466604675,
0.012955810001585633,
0.012955810001585633,
0.012955810001585633,
0.012955810001585633,
0.016913124340741586,
0.016913124340741586,
0.016913124340741586,
0.015995322338615853,
0.015995322338615853,
0.015995322338615853,
0.012007721743430011,
0.012007721743430011,
0.012007721743430011,
0.012007721743430011,
0.024051369488006458,
0.024051369488006458,
0.015974845349167783,
0.015974845349167783,
0.015974845349167783,
0.015958434999144327,
0.015958434999144327,
0.015958434999144327,
0.015792935659798484,
0.015792935659798484,
0.015792935659798484,
0.01592183733979861,
0.01592183733979861,
0.01592183733979861,
0.011986501252977178,
0.011986501252977178,
0.011986501252977178,
0.011986501252977178,
0.01585641598406558,
0.01585641598406558,
0.01585641598406558,
0.015924710663966835,
0.015924710663966835,
0.015924710663966835,
0.01634039101190865,
0.01634039101190865,
0.01634039101190865
],
[
0.057074277006904595,
0.057074277006904595,
0.057074277006904595,
0.057074277006904595,
0.030024631996639073,
0.030024631996639073,
0.020074642340963084,
0.020074642340963084,
0.020074642340963084,
0.020034064655192196,
0.020034064655192196,
0.020034064655192196,
0.015019408252555877,
0.015019408252555877,
0.015019408252555877,
0.015019408252555877,
0.02973640151321888,
0.02973640151321888,
0.020345932998073597,
0.020345932998073597,
0.020345932998073597,
0.020652821326317888,
0.020652821326317888,
0.020652821326317888,
0.014902361246640794,
0.014902361246640794,
0.014902361246640794,
0.014902361246640794,
0.014167511501000263,
0.014167511501000263,
0.014167511501000263,
0.014167511501000263,
0.04647939599817619,
0.04647939599817619,
0.019343009334988892,
0.019343009334988892,
0.019343009334988892,
0.01895378966582939,
0.01895378966582939,
0.01895378966582939,
0.014322384260594845,
0.014322384260594845,
0.014322384260594845,
0.014322384260594845,
0.02855006148456596,
0.02855006148456596,
0.018938185996375978,
0.018938185996375978,
0.018938185996375978,
0.019176096674830962,
0.019176096674830962,
0.019176096674830962,
0.019044518005102873,
0.019044518005102873,
0.019044518005102873,
0.019305008987430483,
0.019305008987430483,
0.019305008987430483,
0.014857510250294581,
0.014857510250294581,
0.014857510250294581,
0.014857510250294581,
0.0190613410086371,
0.0190613410086371,
0.0190613410086371,
0.019120295338022213,
0.019120295338022213,
0.019120295338022213,
0.0143242962367367,
0.0143242962367367,
0.0143242962367367,
0.0143242962367367,
0.029075569502310827,
0.029075569502310827,
0.019497531349770725,
0.019497531349770725,
0.019497531349770725,
0.018608823651447892,
0.018608823651447892,
0.018608823651447892,
0.018379226346344996,
0.018379226346344996,
0.018379226346344996,
0.018320148325680446,
0.018320148325680446,
0.018320148325680446,
0.01379665250715334,
0.01379665250715334,
0.01379665250715334,
0.01379665250715334,
0.01856846265339603,
0.01856846265339603,
0.01856846265339603,
0.018352199345827103,
0.018352199345827103,
0.018352199345827103,
0.018347878998611122,
0.018347878998611122,
0.018347878998611122,
0.02772726200055331,
0.02772726200055331,
0.013726471239351667,
0.013726471239351667,
0.013726471239351667,
0.013726471239351667,
0.018453674002860982,
0.018453674002860982,
0.018453674002860982,
0.027627600502455607,
0.027627600502455607,
0.013753056759014726,
0.013753056759014726,
0.013753056759014726,
0.013753056759014726,
0.013533932986319996,
0.013533932986319996,
0.013533932986319996,
0.013533932986319996,
0.018667823673846822,
0.018667823673846822,
0.018667823673846822,
0.01851649465970695,
0.01851649465970695,
0.01851649465970695,
0.027962180523900315,
0.027962180523900315,
0.01858143232918034,
0.01858143232918034,
0.01858143232918034,
0.01392033399315551,
0.01392033399315551,
0.01392033399315551,
0.01392033399315551,
0.01856105534049372,
0.01856105534049372,
0.01856105534049372,
0.027690295508364215,
0.027690295508364215,
0.014009808248374611,
0.014009808248374611,
0.014009808248374611,
0.014009808248374611,
0.014324050745926797,
0.014324050745926797,
0.014324050745926797,
0.014324050745926797,
0.018292734341230243,
0.018292734341230243,
0.018292734341230243,
0.03272628099269544,
0.03272628099269544,
0.03272628099269544,
0.027157796488609165,
0.027157796488609165,
0.017211413010954857,
0.017211413010954857,
0.017211413010954857,
0.012955822996445931,
0.012955822996445931,
0.012955822996445931,
0.012955822996445931,
0.017078191003141303,
0.017078191003141303,
0.017078191003141303,
0.02565545149263926,
0.02565545149263926,
0.026133759994991124,
0.026133759994991124,
0.012745351748890243,
0.012745351748890243,
0.012745351748890243,
0.012745351748890243,
0.02526378951733932,
0.02526378951733932,
0.016981848670790594,
0.016981848670790594,
0.016981848670790594,
0.01753506433063497,
0.01753506433063497,
0.01753506433063497,
0.013146101002348587,
0.013146101002348587,
0.013146101002348587,
0.013146101002348587,
0.025147862499579787,
0.025147862499579787,
0.016644779650960118,
0.016644779650960118,
0.016644779650960118,
0.01670563401421532,
0.01670563401421532,
0.01670563401421532,
0.025114176998613402,
0.025114176998613402,
0.025240779505111277,
0.025240779505111277,
0.01760628231568262,
0.01760628231568262,
0.01760628231568262,
0.016931691012966137,
0.016931691012966137,
0.016931691012966137,
0.01727279700571671,
0.01727279700571671,
0.01727279700571671,
0.016912257318229724,
0.016912257318229724,
0.016912257318229724,
0.01598840901472916,
0.01598840901472916,
0.01598840901472916,
0.016010235665210832,
0.016010235665210832,
0.016010235665210832,
0.016034339651620638,
0.016034339651620638,
0.016034339651620638,
0.015974988675831508,
0.015974988675831508,
0.015974988675831508,
0.023938142490806058,
0.023938142490806058,
0.023689378518611193,
0.023689378518611193,
0.01592199399601668,
0.01592199399601668,
0.01592199399601668,
0.01598192499174426,
0.01598192499174426,
0.01598192499174426,
0.01586189299511413,
0.01586189299511413,
0.01586189299511413,
0.015926023673576612,
0.015926023673576612,
0.015926023673576612,
0.016315380674010765,
0.016315380674010765,
0.016315380674010765,
0.017069103002237778,
0.017069103002237778,
0.017069103002237778,
0.015235118994799754,
0.015235118994799754,
0.015235118994799754,
0.015159054659307003,
0.015159054659307003,
0.015159054659307003,
0.022672360995784402,
0.022672360995784402,
0.023076677520293742,
0.023076677520293742
],
[
0.07599885431894411,
0.07599885431894411,
0.07599885431894411,
0.020025841678337503,
0.020025841678337503,
0.020025841678337503,
0.020075395664510626,
0.020075395664510626,
0.020075395664510626,
0.020034617996619392,
0.020034617996619392,
0.020034617996619392,
0.020026374336642522,
0.020026374336642522,
0.020026374336642522,
0.019823340660271544,
0.019823340660271544,
0.019823340660271544,
0.030416983005125076,
0.030416983005125076,
0.03106233850121498,
0.03106233850121498,
0.019885292005104322,
0.019885292005104322,
0.019885292005104322,
0.018892501985343795,
0.018892501985343795,
0.018892501985343795,
0.030985270665648084,
0.030985270665648084,
0.030985270665648084,
0.019343586017688114,
0.019343586017688114,
0.019343586017688114,
0.018950182983341318,
0.018950182983341318,
0.018950182983341318,
0.019099209011377145,
0.019099209011377145,
0.019099209011377145,
0.01903421765503784,
0.01903421765503784,
0.01903421765503784,
0.018937779338254284,
0.018937779338254284,
0.018937779338254284,
0.02875587999005802,
0.02875587999005802,
0.02856260701082647,
0.02856260701082647,
0.019304305334420253,
0.019304305334420253,
0.019304305334420253,
0.019817157338062923,
0.019817157338062923,
0.019817157338062923,
0.019061451001713674,
0.019061451001713674,
0.019061451001713674,
0.019113485332733642,
0.019113485332733642,
0.019113485332733642,
0.019104608334600925,
0.019104608334600925,
0.019104608334600925,
0.02907446399331093,
0.02907446399331093,
0.014618368746596389,
0.014618368746596389,
0.014618368746596389,
0.014618368746596389,
0.018610139998296898,
0.018610139998296898,
0.018610139998296898,
0.02757459500571713,
0.02757459500571713,
0.027460306999273598,
0.027460306999273598,
0.01380542025435716,
0.01380542025435716,
0.01380542025435716,
0.01380542025435716,
0.027853768988279626,
0.027853768988279626,
0.0183525126776658,
0.0183525126776658,
0.0183525126776658,
0.018347032329378028,
0.018347032329378028,
0.018347032329378028,
0.018484470997160923,
0.018484470997160923,
0.018484470997160923,
0.01830341199335332,
0.01830341199335332,
0.01830341199335332,
0.01845283400810634,
0.01845283400810634,
0.01845283400810634,
0.018417870334815234,
0.018417870334815234,
0.018417870334815234,
0.027498273499077186,
0.027498273499077186,
0.027074395504314452,
0.027074395504314452,
0.018667700661656756,
0.018667700661656756,
0.018667700661656756,
0.01851662133897965,
0.01851662133897965,
0.01851662133897965,
0.018641199994211394,
0.018641199994211394,
0.018641199994211394,
0.018580482341349125,
0.018580482341349125,
0.018580482341349125,
0.018559868990754087,
0.018559868990754087,
0.018559868990754087,
0.018557828676421195,
0.018557828676421195,
0.018557828676421195,
0.018460286994619917,
0.018460286994619917,
0.018460286994619917,
0.018637193328080077,
0.018637193328080077,
0.018637193328080077,
0.028849135007476434,
0.028849135007476434,
0.027445936488220468,
0.027445936488220468
],
[
0.07609848600501816,
0.07609848600501816,
0.07609848600501816,
0.02001590799773112,
0.02001590799773112,
0.02001590799773112,
0.020074725985371817,
0.020074725985371817,
0.020074725985371817,
0.015025570755824447,
0.015025570755824447,
0.015025570755824447,
0.015025570755824447,
0.0200260576675646,
0.0200260576675646,
0.0200260576675646,
0.014868118247250095,
0.014868118247250095,
0.014868118247250095,
0.014868118247250095,
0.01529539775219746,
0.01529539775219746,
0.01529539775219746,
0.01529539775219746,
0.0618297120090574
],
[
0.11415219900663942,
0.11415219900663942,
0.020037391669272136,
0.020037391669272136,
0.020037391669272136,
0.015055509502417408,
0.015055509502417408,
0.015055509502417408,
0.015055509502417408,
0.020034414328013856,
0.020034414328013856,
0.020034414328013856,
0.020025807665660977,
0.020025807665660977,
0.020025807665660977,
0.01982415100792423,
0.01982415100792423,
0.01982415100792423,
0.030516334489220753,
0.030516334489220753,
0.015466665499843657,
0.015466665499843657,
0.015466665499843657,
0.015466665499843657,
0.019884118674478184,
0.019884118674478184,
0.019884118674478184,
0.028335032984614372,
0.028335032984614372,
0.023239655507495627,
0.023239655507495627,
0.023239655507495627,
0.023239655507495627,
0.014507339743431658,
0.014507339743431658,
0.014507339743431658,
0.014507339743431658,
0.01895076932851225,
0.01895076932851225,
0.01895076932851225,
0.019099789011913042,
0.019099789011913042,
0.019099789011913042,
0.028550306480610743,
0.028550306480610743,
0.018938216341969866,
0.018938216341969866,
0.018938216341969866,
0.019193043660682935,
0.019193043660682935,
0.019193043660682935,
0.019044294662307948,
0.019044294662307948,
0.019044294662307948,
0.019304565348041553,
0.019304565348041553,
0.019304565348041553,
0.014856965237413533,
0.014856965237413533,
0.014856965237413533,
0.014856965237413533,
0.014296558263595216,
0.014296558263595216,
0.014296558263595216,
0.014296558263595216,
0.01911949533193062,
0.01911949533193062,
0.01911949533193062,
0.019099241665874917,
0.019099241665874917,
0.019099241665874917,
0.029074459482217208,
0.029074459482217208,
0.01949675833263124,
0.01949675833263124,
0.01949675833263124,
0.013956615002825856,
0.013956615002825856,
0.013956615002825856,
0.013956615002825856,
0.018378923006821424,
0.018378923006821424,
0.018378923006821424,
0.027460777491796762,
0.027460777491796762,
0.013805887501803227,
0.013805887501803227,
0.013805887501803227,
0.013805887501803227,
0.013926447005360387,
0.013926447005360387,
0.013926447005360387,
0.013926447005360387,
0.01835258931775267,
0.01835258931775267,
0.01835258931775267,
0.018347795664643247,
0.018347795664643247,
0.018347795664643247,
0.013863318512449041,
0.013863318512449041,
0.013863318512449041,
0.013863318512449041,
0.027453502494608983,
0.027453502494608983,
0.01845339099721362,
0.01845339099721362,
0.01845339099721362,
0.018418139991505694,
0.018418139991505694,
0.018418139991505694,
0.027505743521032855,
0.027505743521032855,
0.027068145980592817,
0.027068145980592817,
0.0186672803441373,
0.0186672803441373,
0.0186672803441373,
0.018516397996184725,
0.018516397996184725,
0.018516397996184725,
0.018641243669359635,
0.018641243669359635,
0.018641243669359635,
0.018581482329561066,
0.018581482329561066,
0.018581482329561066,
0.01392028150439728,
0.01392028150439728,
0.01392028150439728,
0.01392028150439728,
0.027841487986734137,
0.027841487986734137,
0.018460020675168682,
0.018460020675168682,
0.018460020675168682,
0.01867730700178072,
0.01867730700178072,
0.01867730700178072,
0.028588780987774953,
0.028588780987774953,
0.018283454002812505,
0.018283454002812505,
0.018283454002812505,
0.02455506350088399,
0.02455506350088399,
0.02455506350088399,
0.02455506350088399,
0.018118704669177532,
0.018118704669177532,
0.018118704669177532,
0.012909814744489267,
0.012909814744489267,
0.012909814744489267,
0.012909814744489267,
0.012955870508449152,
0.012955870508449152,
0.012955870508449152,
0.012955870508449152,
0.025617786479415372,
0.025617786479415372,
0.01710334767509873,
0.01710334767509873,
0.01710334767509873,
0.01738610933534801,
0.01738610933534801,
0.01738610933534801,
0.025520398980006576,
0.025520398980006576,
0.016854186678150047,
0.016854186678150047,
0.016854186678150047,
0.01274280650250148,
0.01274280650250148,
0.01274280650250148,
0.01274280650250148,
0.02618332498241216,
0.02618332498241216
],
[
0.07610312267206609,
0.07610312267206609,
0.07610312267206609,
0.01501510600792244,
0.01501510600792244,
0.01501510600792244,
0.01501510600792244,
0.020073268989411492,
0.020073268989411492,
0.020073268989411492,
0.020034901332110167,
0.020034901332110167,
0.020034901332110167,
0.01501977325824555,
0.01501977325824555,
0.01501977325824555,
0.01501977325824555,
0.02973549149464816,
0.02973549149464816,
0.06096841796534136
]
],
"generated_texts": [
" the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed",
" dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit volupt",
" quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog.",
"piciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The",
" The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspic",
"atem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem",
" jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem",
"nis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis",
" error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over",
" voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde om"
],
"errors": [
"",
"",
"",
"",
"",
"",
"",
"",
"",
""
]
}
]
}
]
}