{ "metadata": { "experiment": "dsv4_p800_sglang", "run_id": "20260708-051316", "timestamp": "2026-07-08T05:13:16+00:00", "model": "/data1/models/DeepSeek-V4-Flash-INT8", "backend": "sglang", "engine": "sglang-xpu", "hardware": "8x Kunlun P800 XPU", "accelerator": "Kunlun P800 XPU", "chip": "kunlun_p800", "script": "experiments/dsv4_p800_sglang/run_bench.sh", "env": "", "git_commit": "2c332ff", "git_dirty": "dirty", "description": "Kunlun P800 SGLang benchmark for DeepSeek-V4-Flash-INT8" }, "config": {}, "scenarios": [ { "name": "c32_i512_o256", "concurrency": 32, "input_len": 512, "output_len": 256, "success": 10, "failed": 0, "duration_s": 5.35, "request_throughput": 1.87, "input_token_throughput": 470.87, "output_token_throughput": 262.65, "total_token_throughput": 733.53, "accept_length": null, "latencies": { "e2e_ms": { "mean": 3372.781996097183, "p50": 3498.675397480838, "p90": 5125.400093087228, "p95": null, "p99": 5314.590020017349 }, "ttft_ms": { "mean": 478.8548628857825, "p50": 506.4399079710711, "p90": null, "p95": null, "p99": 507.2712521097855 }, "tpot_ms": { "mean": 22.334078483797782, "p50": 21.948128755653826, "p90": null, "p95": null, "p99": 29.154062995221466 }, "itl_ms": { "mean": 20.75981864563044, "p50": 18.560815330905218, "p90": null, "p95": 30.416983005125076, "p99": 76.09861597185954 } }, "raw_requests": [ { "tag": null, "backend": "sglang", "dataset_name": "random", "request_rate": Infinity, "max_concurrency": 32, "sharegpt_output_len": null, "random_input_len": 512, "random_output_len": 256, "random_range_ratio": 0.0, "server_info": { "model_path": "/data1/models/DeepSeek-V4-Flash-INT8", "tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8", "tokenizer_mode": "auto", "tokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "grpc_mode": false, "skip_server_warmup": false, "warmups": null, "nccl_port": null, "checkpoint_engine_wait_weights_before_ready": false, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "dtype": "auto", "quantization": "w8a8_int8", "quantization_param_path": null, "kv_cache_dtype": "float16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "mem_fraction_static": 0.8, "max_running_requests": 64, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "custom_max_context_len": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "schedule_conservativeness": 1.0, "page_size": 256, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "device": "cuda", "tp_size": 8, "pp_size": 1, "pp_max_micro_batch_size": null, "pp_async_batch_depth": 0, "stream_interval": 1, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "random_seed": 228424360, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": true, "watchdog_timeout": 3000000.0, "soft_watchdog_timeout": null, "dist_timeout": null, "download_dir": null, "model_checksum": null, "base_gpu_id": 0, "gpu_id_step": 1, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "collect_tokens_histogram": false, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "enable_trace": false, "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "api_key": null, "admin_api_key": null, "served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "json_model_override_args": "{}", "preferred_sampling_params": null, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "attention_backend": "compressed", "decode_attention_backend": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "nsa_prefill_backend": "klxdsa", "nsa_decode_backend": "klxdsa", "disable_flashinfer_autotune": false, "mamba_backend": "triton", "speculative_algorithm": "EAGLE", "speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8", "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": 3, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 4, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": "w8a8_int8", "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "enable_multi_layer_eagle": false, "ep_size": 8, "moe_a2a_backend": "none", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "enable_hisparse": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "enable_aiter_allreduce_fusion": false, "deepep_mode": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "enable_expert_distribution_metrics": false, "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_full_memory_ratio": 0.9, "mamba_scheduler_strategy": "no_buffer", "mamba_track_interval": 256, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "best_effort", "hicache_storage_backend_extra_config": null, "hisparse_config": null, "enable_lmcache": false, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "enable_double_sparsity": false, "ds_channel_config_path": null, "ds_heavy_channel_num": 32, "ds_heavy_token_num": 256, "ds_heavy_channel_type": "qk", "ds_sparse_decode_threshold": 4096, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "multi_item_scoring_delimiter": null, "disable_radix_cache": false, "cuda_graph_max_bs": 32, "cuda_graph_bs": [ 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32 ], "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "disable_outlines_disk_cache": false, "disable_custom_all_reduce": true, "enable_mscclpp": false, "enable_torch_symm_mem": false, "pre_warm_nccl": false, "disable_overlap_schedule": false, "enable_mixed_chunk": false, "enable_dp_attention": false, "enable_dp_lm_head": false, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "enable_torch_compile": false, "disable_piecewise_cuda_graph": true, "enforce_piecewise_cuda_graph": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "piecewise_cuda_graph_max_tokens": 8192, "piecewise_cuda_graph_tokens": [ 4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192 ], "piecewise_cuda_graph_compiler": "eager", "torchao_config": "", "enable_nan_detection": false, "enable_p2p_check": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "num_continuous_decode_steps": 1, "delete_ckpt_after_loading": false, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "allow_auto_truncate": false, "enable_custom_logit_processor": false, "flashinfer_mla_disable_ragged": false, "disable_shared_experts_fusion": true, "disable_chunked_prefix_cache": false, "disable_fast_image_processor": false, "keep_mm_feature_on_device": false, "enable_return_hidden_states": false, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "scheduler_recv_interval": 1, "numa_node": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "enable_attn_tp_input_scattered": false, "gc_threshold": null, "enable_nsa_prefill_context_parallel": false, "nsa_prefill_cp_mode": "in-seq-split", "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "debug_tensor_dump_inject": false, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_offload_kvcache": false, "num_reserved_decode_tokens": 512, "disaggregation_decode_polling_interval": 1, "encoder_only": false, "language_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_over_encoding": false, "enable_kv_mirror": false, "custom_weight_loader": [], "weight_loader_disable_mmap": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "forward_hooks": null, "index_cache_topk_freq": 1, "index_cache_pattern": null, "status": "ready", "max_total_num_tokens": 1593344, "max_req_input_len": 1048570, "internal_states": [ { "model_path": "/data1/models/DeepSeek-V4-Flash-INT8", "tokenizer_path": "/data1/models/DeepSeek-V4-Flash-INT8", "tokenizer_mode": "auto", "tokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": 1048576, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "grpc_mode": false, "skip_server_warmup": false, "warmups": null, "nccl_port": null, "checkpoint_engine_wait_weights_before_ready": false, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "dtype": "auto", "quantization": "w8a8_int8", "quantization_param_path": null, "kv_cache_dtype": "float16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "mem_fraction_static": 0.8, "max_running_requests": 64, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "custom_max_context_len": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "schedule_conservativeness": 1.0, "page_size": 256, "swa_full_tokens_ratio": 0.1, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "device": "cuda", "tp_size": 8, "pp_size": 1, "pp_max_micro_batch_size": 64, "pp_async_batch_depth": 0, "stream_interval": 1, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "random_seed": 228424360, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": true, "watchdog_timeout": 3000000.0, "soft_watchdog_timeout": null, "dist_timeout": null, "download_dir": null, "model_checksum": null, "base_gpu_id": 0, "gpu_id_step": 1, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "collect_tokens_histogram": false, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "enable_trace": false, "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "api_key": null, "admin_api_key": null, "served_model_name": "/data1/models/DeepSeek-V4-Flash-INT8", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": "deepseek-v4", "tool_call_parser": "deepseekv4", "tool_server": null, "sampling_defaults": "model", "dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dist_init_addr": null, "nnodes": 1, "node_rank": 0, "json_model_override_args": "{}", "preferred_sampling_params": null, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "attention_backend": "compressed", "decode_attention_backend": "compressed", "prefill_attention_backend": "compressed", "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "nsa_prefill_backend": "klxdsa", "nsa_decode_backend": "klxdsa", "disable_flashinfer_autotune": false, "mamba_backend": "triton", "speculative_algorithm": "EAGLE", "speculative_draft_model_path": "/data1/models/DeepSeek-V4-Flash-INT8", "speculative_draft_model_revision": null, "speculative_draft_load_format": null, "speculative_num_steps": 3, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 4, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": null, "speculative_moe_runner_backend": "auto", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": "w8a8_int8", "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "enable_multi_layer_eagle": false, "ep_size": 8, "moe_a2a_backend": "none", "moe_runner_backend": "auto", "flashinfer_mxfp4_moe_precision": "default", "enable_hisparse": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "enable_aiter_allreduce_fusion": false, "deepep_mode": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "enable_expert_distribution_metrics": false, "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_full_memory_ratio": 0.9, "mamba_scheduler_strategy": "no_buffer", "mamba_track_interval": 256, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "enable_hierarchical_cache": false, "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "best_effort", "hicache_storage_backend_extra_config": null, "hisparse_config": null, "enable_lmcache": false, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "enable_double_sparsity": false, "ds_channel_config_path": null, "ds_heavy_channel_num": 32, "ds_heavy_token_num": 256, "ds_heavy_channel_type": "qk", "ds_sparse_decode_threshold": 4096, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "multi_item_scoring_delimiter": null, "disable_radix_cache": false, "cuda_graph_max_bs": 32, "cuda_graph_bs": [ 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30, 32 ], "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "disable_outlines_disk_cache": false, "disable_custom_all_reduce": true, "enable_mscclpp": false, "enable_torch_symm_mem": false, "pre_warm_nccl": false, "disable_overlap_schedule": false, "enable_mixed_chunk": false, "enable_dp_attention": false, "enable_dp_lm_head": false, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "enable_torch_compile": false, "disable_piecewise_cuda_graph": true, "enforce_piecewise_cuda_graph": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "piecewise_cuda_graph_max_tokens": 8192, "piecewise_cuda_graph_tokens": [ 4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048, 2304, 2560, 2816, 3072, 3328, 3584, 3840, 4096, 4608, 5120, 5632, 6144, 6656, 7168, 7680, 8192 ], "piecewise_cuda_graph_compiler": "eager", "torchao_config": "", "enable_nan_detection": false, "enable_p2p_check": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "num_continuous_decode_steps": 1, "delete_ckpt_after_loading": false, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "allow_auto_truncate": false, "enable_custom_logit_processor": false, "flashinfer_mla_disable_ragged": false, "disable_shared_experts_fusion": true, "disable_chunked_prefix_cache": true, "disable_fast_image_processor": false, "keep_mm_feature_on_device": false, "enable_return_hidden_states": false, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "scheduler_recv_interval": 1, "numa_node": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "enable_attn_tp_input_scattered": false, "gc_threshold": null, "enable_nsa_prefill_context_parallel": false, "nsa_prefill_cp_mode": "in-seq-split", "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "debug_tensor_dump_inject": false, "disaggregation_mode": "null", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 8998, "disaggregation_ib_device": null, "disaggregation_decode_enable_offload_kvcache": false, "num_reserved_decode_tokens": 512, "disaggregation_decode_polling_interval": 1, "encoder_only": false, "language_only": false, "encoder_transfer_backend": "zmq_to_scheduler", "encoder_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_over_encoding": false, "enable_kv_mirror": false, "custom_weight_loader": [], "weight_loader_disable_mmap": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "forward_hooks": null, "index_cache_topk_freq": 1, "index_cache_pattern": null, "use_mla_backend": false, "_mx_config_cache": {}, "_draft_pool_config": { "full_max_total_num_tokens": 1593344, "swa_max_total_num_tokens": 159232 }, "last_gen_throughput": 304.3209280340091, "memory_usage": { "weight": 37.94, "kvcache": 0, "token_capacity": 1593344, "graph": 1.2 }, "effective_max_running_requests_per_dp": 64, "avg_spec_accept_length": 2.8933333333333335 } ], "version": "0.5.10+ee6948cf8" }, "duration": 5.34541754698148, "completed": 10, "total_input_tokens": 2517, "total_input_text_tokens": 2517, "total_input_vision_tokens": 0, "total_output_tokens": 1404, "total_output_tokens_retokenized": 1404, "request_throughput": 1.870761247762007, "input_throughput": 470.8706060616972, "output_throughput": 262.6548791857858, "total_throughput": 733.525485247483, "mean_e2e_latency_ms": 3372.781996097183, "median_e2e_latency_ms": 3498.675397480838, "std_e2e_latency_ms": 1426.9563106485741, "p90_e2e_latency_ms": 5125.400093087228, "p99_e2e_latency_ms": 5314.590020017349, "mean_ttft_ms": 478.8548628857825, "median_ttft_ms": 506.4399079710711, "std_ttft_ms": 82.89120389713447, "p99_ttft_ms": 507.2712521097855, "mean_tpot_ms": 22.334078483797782, "median_tpot_ms": 21.948128755653826, "std_tpot_ms": 3.1100348959353536, "p99_tpot_ms": 29.154062995221466, "mean_itl_ms": 20.75981864563044, "median_itl_ms": 18.560815330905218, "std_itl_ms": 11.544034313142475, "p95_itl_ms": 30.416983005125076, "p99_itl_ms": 76.09861597185954, "concurrency": 6.309669855447998, "accept_length": 2.8933333333333335, "max_output_tokens_per_s": 396.0, "max_concurrent_requests": 10, "input_lens": [ 38, 236, 397, 73, 256, 394, 204, 134, 336, 449 ], "output_lens": [ 145, 130, 205, 72, 238, 253, 135, 26, 179, 21 ], "ttfts": [ 0.23018694896018133, 0.5072830729768611, 0.5071517300093547, 0.506980306992773, 0.5067555939895101, 0.5065123789827339, 0.5063674369594082, 0.5061119620222598, 0.5057157859555446, 0.5054834120091982 ], "itls": [ [ 0.1265941219899105, 0.1265941219899105, 0.1265941219899105, 0.1265941219899105, 0.03003738750703633, 0.03003738750703633, 0.020075495665272076, 0.020075495665272076, 0.020075495665272076, 0.020034517995857943, 0.020034517995857943, 0.020034517995857943, 0.020025947674488027, 0.020025947674488027, 0.020025947674488027, 0.01982437400147319, 0.01982437400147319, 0.01982437400147319, 0.030580375489080325, 0.030580375489080325, 0.015474200758035295, 0.015474200758035295, 0.015474200758035295, 0.015474200758035295, 0.02975128698744811, 0.02975128698744811, 0.018891388666816056, 0.018891388666816056, 0.018891388666816056, 0.030985667331454653, 0.030985667331454653, 0.030985667331454653, 0.01450748200295493, 0.01450748200295493, 0.01450748200295493, 0.01450748200295493, 0.02843167950049974, 0.02843167950049974, 0.019097025661418836, 0.019097025661418836, 0.019097025661418836, 0.019033391009240102, 0.019033391009240102, 0.019033391009240102, 0.02840644898242317, 0.02840644898242317, 0.0287785355176311, 0.0287785355176311, 0.01427662099013105, 0.01427662099013105, 0.01427662099013105, 0.01427662099013105, 0.028972148516913876, 0.028972148516913876, 0.0198178769981799, 0.0198178769981799, 0.0198178769981799, 0.019058960994395118, 0.019058960994395118, 0.019058960994395118, 0.014342486509121954, 0.014342486509121954, 0.014342486509121954, 0.014342486509121954, 0.028646462480537593, 0.028646462480537593, 0.01456742275331635, 0.01456742275331635, 0.01456742275331635, 0.01456742275331635, 0.01943501700103904, 0.01943501700103904, 0.01943501700103904, 0.02791283550322987, 0.02791283550322987, 0.013792212252155878, 0.013792212252155878, 0.013792212252155878, 0.013792212252155878, 0.018324121988068025, 0.018324121988068025, 0.018324121988068025, 0.018392249670190115, 0.018392249670190115, 0.018392249670190115, 0.018568886327557266, 0.018568886327557266, 0.018568886327557266, 0.018351545673795044, 0.018351545673795044, 0.018351545673795044, 0.02752352401148528, 0.02752352401148528, 0.013863828251487575, 0.013863828251487575, 0.013863828251487575, 0.013863828251487575, 0.018301511649042368, 0.018301511649042368, 0.018301511649042368, 0.027680501516442746, 0.027680501516442746, 0.01841827033786103, 0.01841827033786103, 0.01841827033786103, 0.01374593649234157, 0.01374593649234157, 0.01374593649234157, 0.01374593649234157, 0.018055193999316543, 0.018055193999316543, 0.018055193999316543, 0.014001160496263765, 0.014001160496263765, 0.014001160496263765, 0.014001160496263765, 0.018516544679490227, 0.018516544679490227, 0.018516544679490227, 0.018641953337161492, 0.018641953337161492, 0.018641953337161492, 0.01858119231959184, 0.01858119231959184, 0.01858119231959184, 0.018560895676879834, 0.018560895676879834, 0.018560895676879834, 0.027825927507365122, 0.027825927507365122, 0.027690790477208793, 0.027690790477208793, 0.014033740997547284, 0.014033740997547284, 0.014033740997547284, 0.014033740997547284, 0.028556485020089895, 0.028556485020089895, 0.018292231000183772, 0.018292231000183772, 0.018292231000183772, 0.03276129499621069, 0.03276129499621069, 0.03276129499621069 ], [ 0.114176959003089, 0.114176959003089, 0.02001935833444198, 0.02001935833444198, 0.02001935833444198, 0.020075112329019856, 0.020075112329019856, 0.020075112329019856, 0.02003466134192422, 0.02003466134192422, 0.02003466134192422, 0.020026450996131945, 0.020026450996131945, 0.020026450996131945, 0.02973513101460412, 0.02973513101460412, 0.030412352993153036, 0.030412352993153036, 0.015535941754933447, 0.015535941754933447, 0.015535941754933447, 0.015535941754933447, 0.02982658799737692, 0.02982658799737692, 0.018890631986626733, 0.018890631986626733, 0.018890631986626733, 0.03098562400555238, 0.03098562400555238, 0.03098562400555238, 0.019343592672764014, 0.019343592672764014, 0.019343592672764014, 0.018950106323851895, 0.018950106323851895, 0.018950106323851895, 0.019099269004072994, 0.019099269004072994, 0.019099269004072994, 0.019034251008027542, 0.019034251008027542, 0.019034251008027542, 0.01420391199644655, 0.01420391199644655, 0.01420391199644655, 0.01420391199644655, 0.01439423524425365, 0.01439423524425365, 0.01439423524425365, 0.01439423524425365, 0.028566046996274963, 0.028566046996274963, 0.01930476234216864, 0.01930476234216864, 0.01930476234216864, 0.019809113660206396, 0.019809113660206396, 0.019809113660206396, 0.014296345761977136, 0.014296345761977136, 0.014296345761977136, 0.014296345761977136, 0.02867897297255695, 0.02867897297255695, 0.019099251677592594, 0.019099251677592594, 0.019099251677592594, 0.019382666000941146, 0.019382666000941146, 0.019382666000941146, 0.02924422750947997, 0.02924422750947997, 0.027913359983358532, 0.027913359983358532, 0.013784522248897702, 0.013784522248897702, 0.013784522248897702, 0.013784522248897702, 0.02746082251542248, 0.02746082251542248, 0.01840783333561073, 0.01840783333561073, 0.01840783333561073, 0.01856821965581427, 0.01856821965581427, 0.01856821965581427, 0.01835258233283336, 0.01835258233283336, 0.01835258233283336, 0.018347706005442888, 0.018347706005442888, 0.018347706005442888, 0.018484461004845798, 0.018484461004845798, 0.018484461004845798, 0.018302411655895412, 0.018302411655895412, 0.018302411655895412, 0.027679786493536085, 0.027679786493536085, 0.027627180505078286, 0.027627180505078286, 0.01375247175747063, 0.01375247175747063, 0.01375247175747063, 0.01375247175747063, 0.0270681754918769, 0.0270681754918769, 0.01866779732517898, 0.01866779732517898, 0.01866779732517898, 0.018516544679490227, 0.018516544679490227, 0.018516544679490227, 0.013980599993374199, 0.013980599993374199, 0.013980599993374199, 0.013980599993374199, 0.027871913509443402, 0.027871913509443402, 0.018560129004375387, 0.018560129004375387, 0.018560129004375387, 0.018560815330905218, 0.018560815330905218, 0.018560815330905218, 0.027690300485119224, 0.027690300485119224, 0.05595995101612061 ], [ 0.07610034267418087, 0.07610034267418087, 0.07610034267418087, 0.02001753800626223, 0.02001753800626223, 0.02001753800626223, 0.020074322334645938, 0.020074322334645938, 0.020074322334645938, 0.015025673492345959, 0.015025673492345959, 0.015025673492345959, 0.015025673492345959, 0.03003843649639748, 0.03003843649639748, 0.01982440067998444, 0.01982440067998444, 0.01982440067998444, 0.02034523299274345, 0.02034523299274345, 0.02034523299274345, 0.030977007001638412, 0.030977007001638412, 0.029805967991705984, 0.029805967991705984, 0.014167338755214587, 0.014167338755214587, 0.014167338755214587, 0.014167338755214587, 0.046478810982080176, 0.046478810982080176, 0.01934337633429095, 0.01934337633429095, 0.01934337633429095, 0.0189525830016161, 0.0189525830016161, 0.0189525830016161, 0.014323239010991529, 0.014323239010991529, 0.014323239010991529, 0.014323239010991529, 0.028550161485327408, 0.028550161485327408, 0.018938299675937742, 0.018938299675937742, 0.018938299675937742, 0.019176949999139953, 0.019176949999139953, 0.019176949999139953, 0.028566726978169754, 0.028566726978169754, 0.028956993512110785, 0.028956993512110785, 0.014857275251415558, 0.014857275251415558, 0.014857275251415558, 0.014857275251415558, 0.028592451504664496, 0.028592451504664496, 0.01912001532036811, 0.01912001532036811, 0.01912001532036811, 0.019098964675019186, 0.019098964675019186, 0.019098964675019186, 0.014537554743583314, 0.014537554743583314, 0.014537554743583314, 0.014537554743583314, 0.029245732497656718, 0.029245732497656718, 0.018609130017769832, 0.018609130017769832, 0.018609130017769832, 0.018378569666917127, 0.018378569666917127, 0.018378569666917127, 0.027461227495223284, 0.027461227495223284, 0.013806014991132542, 0.013806014991132542, 0.013806014991132542, 0.013806014991132542, 0.013926397252362221, 0.013926397252362221, 0.013926397252362221, 0.013926397252362221, 0.01835247567699601, 0.01835247567699601, 0.01835247567699601, 0.01834764933058371, 0.01834764933058371, 0.01834764933058371, 0.013863673244486563, 0.013863673244486563, 0.013863673244486563, 0.013863673244486563, 0.027453012502519414, 0.027453012502519414, 0.018453627669562895, 0.018453627669562895, 0.018453627669562895, 0.018418410327285528, 0.018418410327285528, 0.018418410327285528, 0.027505588514031842, 0.027505588514031842, 0.013533967750845477, 0.013533967750845477, 0.013533967750845477, 0.013533967750845477, 0.014000577997649089, 0.014000577997649089, 0.014000577997649089, 0.014000577997649089, 0.018516601334946852, 0.018516601334946852, 0.018516601334946852, 0.018641606671735644, 0.018641606671735644, 0.018641606671735644, 0.013935844239313155, 0.013935844239313155, 0.013935844239313155, 0.013935844239313155, 0.02784075849922374, 0.02784075849922374, 0.018561025344145794, 0.018561025344145794, 0.018561025344145794, 0.01846017032706489, 0.01846017032706489, 0.01846017032706489, 0.02801839599851519, 0.02801839599851519, 0.014315253254608251, 0.014315253254608251, 0.014315253254608251, 0.014315253254608251, 0.013719310751184821, 0.013719310751184821, 0.013719310751184821, 0.013719310751184821, 0.032736671001960836, 0.032736671001960836, 0.032736671001960836, 0.01810533132326479, 0.01810533132326479, 0.01810533132326479, 0.025817634508712217, 0.025817634508712217, 0.017274240672122687, 0.017274240672122687, 0.017274240672122687, 0.012808780491468497, 0.012808780491468497, 0.012808780491468497, 0.012808780491468497, 0.017103748007987935, 0.017103748007987935, 0.017103748007987935, 0.0260774539783597, 0.0260774539783597, 0.012770477012963966, 0.012770477012963966, 0.012770477012963966, 0.012770477012963966, 0.012633724749321118, 0.012633724749321118, 0.012633724749321118, 0.012633724749321118, 0.016969308664556593, 0.016969308664556593, 0.016969308664556593, 0.017546184652019292, 0.017546184652019292, 0.017546184652019292, 0.02627176101668738, 0.02627176101668738, 0.01677496867099156, 0.01677496867099156, 0.01677496867099156, 0.012476857242290862, 0.012476857242290862, 0.012476857242290862, 0.012476857242290862, 0.016706993997407455, 0.016706993997407455, 0.016706993997407455, 0.02511487199808471, 0.02511487199808471, 0.012620112000149675, 0.012620112000149675, 0.012620112000149675, 0.012620112000149675, 0.013204801754909568, 0.013204801754909568, 0.013204801754909568, 0.013204801754909568, 0.016931711327439796, 0.016931711327439796, 0.016931711327439796, 0.025843739014817402, 0.025843739014817402 ], [ 0.07609820265012483, 0.07609820265012483, 0.07609820265012483, 0.020024441337833803, 0.020024441337833803, 0.020024441337833803, 0.015056459509651177, 0.015056459509651177, 0.015056459509651177, 0.015056459509651177, 0.03005173648125492, 0.03005173648125492, 0.015019418249721639, 0.015019418249721639, 0.015019418249721639, 0.015019418249721639, 0.019824344334968675, 0.019824344334968675, 0.019824344334968675, 0.020387480326462537, 0.020387480326462537, 0.020387480326462537, 0.020631657680496573, 0.020631657680496573, 0.020631657680496573, 0.01487621849810239, 0.01487621849810239, 0.01487621849810239, 0.01487621849810239, 0.028335707494989038, 0.028335707494989038, 0.023239640504471026, 0.023239640504471026, 0.023239640504471026, 0.023239640504471026, 0.0193434563310196, 0.0193434563310196, 0.0193434563310196, 0.028431184502551332, 0.028431184502551332, 0.014322734248708002, 0.014322734248708002, 0.014322734248708002, 0.014322734248708002, 0.01903337399320056, 0.01903337399320056, 0.01903337399320056, 0.01893743300267185, 0.01893743300267185, 0.01893743300267185, 0.01917593334413444, 0.01917593334413444, 0.01917593334413444, 0.028568201989401132, 0.028568201989401132, 0.01930442900629714, 0.01930442900629714, 0.01930442900629714, 0.014869755497784354, 0.014869755497784354, 0.014869755497784354, 0.014869755497784354, 0.028589651483343914, 0.028589651483343914, 0.019121012010145932, 0.019121012010145932, 0.019121012010145932, 0.01909933132507528, 0.01909933132507528, 0.01909933132507528, 0.0582002800074406 ], [ 0.07609824265819043, 0.07609824265819043, 0.07609824265819043, 0.015011793249868788, 0.015011793249868788, 0.015011793249868788, 0.015011793249868788, 0.020074732679252822, 0.020074732679252822, 0.020074732679252822, 0.015025845990749076, 0.015025845990749076, 0.015025845990749076, 0.015025845990749076, 0.030038921511732042, 0.030038921511732042, 0.019824267326233286, 0.019824267326233286, 0.019824267326233286, 0.020395867332505684, 0.020395867332505684, 0.020395867332505684, 0.03094582151970826, 0.03094582151970826, 0.014877470748615451, 0.014877470748615451, 0.014877470748615451, 0.014877470748615451, 0.014167653993354179, 0.014167653993354179, 0.014167653993354179, 0.014167653993354179, 0.030986197331609826, 0.030986197331609826, 0.030986197331609826, 0.01934319967404008, 0.01934319967404008, 0.01934319967404008, 0.014215631992556155, 0.014215631992556155, 0.014215631992556155, 0.014215631992556155, 0.02864447850151919, 0.02864447850151919, 0.01903398767656957, 0.01903398767656957, 0.01903398767656957, 0.018937196330322575, 0.018937196330322575, 0.018937196330322575, 0.028763925016392022, 0.028763925016392022, 0.02856709697516635, 0.02856709697516635, 0.014478914003120735, 0.014478914003120735, 0.014478914003120735, 0.014478914003120735, 0.029714685515500605, 0.029714685515500605, 0.019061427660441648, 0.019061427660441648, 0.019061427660441648, 0.019120605332621683, 0.019120605332621683, 0.019120605332621683, 0.014324548756121658, 0.014324548756121658, 0.014324548756121658, 0.014324548756121658, 0.0290751994762104, 0.0290751994762104, 0.019497701665386558, 0.019497701665386558, 0.019497701665386558, 0.01860897334214921, 0.01860897334214921, 0.01860897334214921, 0.02756879449589178, 0.02756879449589178, 0.02748415298992768, 0.02748415298992768, 0.01379515475127846, 0.01379515475127846, 0.01379515475127846, 0.01379515475127846, 0.02785251452587545, 0.02785251452587545, 0.018352062324993312, 0.018352062324993312, 0.018352062324993312, 0.01834822599388038, 0.01834822599388038, 0.01834822599388038, 0.01386378325696569, 0.01386378325696569, 0.01386378325696569, 0.01386378325696569, 0.02745306250290014, 0.02745306250290014, 0.018453344334072124, 0.018453344334072124, 0.018453344334072124, 0.018418603654329974, 0.018418603654329974, 0.018418603654329974, 0.02750637350254692, 0.02750637350254692, 0.027067835500929505, 0.027067835500929505, 0.014000943003338762, 0.014000943003338762, 0.014000943003338762, 0.014000943003338762, 0.027774896996561438, 0.027774896996561438, 0.01864148332970217, 0.01864148332970217, 0.01864148332970217, 0.018581362344169367, 0.018581362344169367, 0.018581362344169367, 0.013920626748586074, 0.013920626748586074, 0.013920626748586074, 0.013920626748586074, 0.027841037983307615, 0.027841037983307615, 0.018460453681958217, 0.018460453681958217, 0.018460453681958217, 0.018680113988618057, 0.018680113988618057, 0.018680113988618057, 0.028647751518292353, 0.028647751518292353, 0.02743947648559697, 0.02743947648559697, 0.024545218504499644, 0.024545218504499644, 0.024545218504499644, 0.024545218504499644, 0.027157786476891488, 0.027157786476891488, 0.017211266347051907, 0.017211266347051907, 0.017211266347051907, 0.017274483999547858, 0.017274483999547858, 0.017274483999547858, 0.012808570492779836, 0.012808570492779836, 0.012808570492779836, 0.012808570492779836, 0.02565537701593712, 0.02565537701593712, 0.017431950002598267, 0.017431950002598267, 0.017431950002598267, 0.016987925317759316, 0.016987925317759316, 0.016987925317759316, 0.025259989517508075, 0.025259989517508075, 0.012736934004351497, 0.012736934004351497, 0.012736934004351497, 0.012736934004351497, 0.013151180988643318, 0.013151180988643318, 0.013151180988643318, 0.013151180988643318, 0.01752863434376195, 0.01752863434376195, 0.01752863434376195, 0.01677370866915832, 0.01677370866915832, 0.01677370866915832, 0.01247714224155061, 0.01247714224155061, 0.01247714224155061, 0.01247714224155061, 0.025057951017515734, 0.025057951017515734, 0.016742911325612415, 0.016742911325612415, 0.016742911325612415, 0.016827172657940537, 0.016827172657940537, 0.016827172657940537, 0.0176063990026402, 0.0176063990026402, 0.0176063990026402, 0.01693138466604675, 0.01693138466604675, 0.01693138466604675, 0.012955810001585633, 0.012955810001585633, 0.012955810001585633, 0.012955810001585633, 0.016913124340741586, 0.016913124340741586, 0.016913124340741586, 0.015995322338615853, 0.015995322338615853, 0.015995322338615853, 0.012007721743430011, 0.012007721743430011, 0.012007721743430011, 0.012007721743430011, 0.024051369488006458, 0.024051369488006458, 0.015974845349167783, 0.015974845349167783, 0.015974845349167783, 0.015958434999144327, 0.015958434999144327, 0.015958434999144327, 0.015792935659798484, 0.015792935659798484, 0.015792935659798484, 0.01592183733979861, 0.01592183733979861, 0.01592183733979861, 0.011986501252977178, 0.011986501252977178, 0.011986501252977178, 0.011986501252977178, 0.01585641598406558, 0.01585641598406558, 0.01585641598406558, 0.015924710663966835, 0.015924710663966835, 0.015924710663966835, 0.01634039101190865, 0.01634039101190865, 0.01634039101190865 ], [ 0.057074277006904595, 0.057074277006904595, 0.057074277006904595, 0.057074277006904595, 0.030024631996639073, 0.030024631996639073, 0.020074642340963084, 0.020074642340963084, 0.020074642340963084, 0.020034064655192196, 0.020034064655192196, 0.020034064655192196, 0.015019408252555877, 0.015019408252555877, 0.015019408252555877, 0.015019408252555877, 0.02973640151321888, 0.02973640151321888, 0.020345932998073597, 0.020345932998073597, 0.020345932998073597, 0.020652821326317888, 0.020652821326317888, 0.020652821326317888, 0.014902361246640794, 0.014902361246640794, 0.014902361246640794, 0.014902361246640794, 0.014167511501000263, 0.014167511501000263, 0.014167511501000263, 0.014167511501000263, 0.04647939599817619, 0.04647939599817619, 0.019343009334988892, 0.019343009334988892, 0.019343009334988892, 0.01895378966582939, 0.01895378966582939, 0.01895378966582939, 0.014322384260594845, 0.014322384260594845, 0.014322384260594845, 0.014322384260594845, 0.02855006148456596, 0.02855006148456596, 0.018938185996375978, 0.018938185996375978, 0.018938185996375978, 0.019176096674830962, 0.019176096674830962, 0.019176096674830962, 0.019044518005102873, 0.019044518005102873, 0.019044518005102873, 0.019305008987430483, 0.019305008987430483, 0.019305008987430483, 0.014857510250294581, 0.014857510250294581, 0.014857510250294581, 0.014857510250294581, 0.0190613410086371, 0.0190613410086371, 0.0190613410086371, 0.019120295338022213, 0.019120295338022213, 0.019120295338022213, 0.0143242962367367, 0.0143242962367367, 0.0143242962367367, 0.0143242962367367, 0.029075569502310827, 0.029075569502310827, 0.019497531349770725, 0.019497531349770725, 0.019497531349770725, 0.018608823651447892, 0.018608823651447892, 0.018608823651447892, 0.018379226346344996, 0.018379226346344996, 0.018379226346344996, 0.018320148325680446, 0.018320148325680446, 0.018320148325680446, 0.01379665250715334, 0.01379665250715334, 0.01379665250715334, 0.01379665250715334, 0.01856846265339603, 0.01856846265339603, 0.01856846265339603, 0.018352199345827103, 0.018352199345827103, 0.018352199345827103, 0.018347878998611122, 0.018347878998611122, 0.018347878998611122, 0.02772726200055331, 0.02772726200055331, 0.013726471239351667, 0.013726471239351667, 0.013726471239351667, 0.013726471239351667, 0.018453674002860982, 0.018453674002860982, 0.018453674002860982, 0.027627600502455607, 0.027627600502455607, 0.013753056759014726, 0.013753056759014726, 0.013753056759014726, 0.013753056759014726, 0.013533932986319996, 0.013533932986319996, 0.013533932986319996, 0.013533932986319996, 0.018667823673846822, 0.018667823673846822, 0.018667823673846822, 0.01851649465970695, 0.01851649465970695, 0.01851649465970695, 0.027962180523900315, 0.027962180523900315, 0.01858143232918034, 0.01858143232918034, 0.01858143232918034, 0.01392033399315551, 0.01392033399315551, 0.01392033399315551, 0.01392033399315551, 0.01856105534049372, 0.01856105534049372, 0.01856105534049372, 0.027690295508364215, 0.027690295508364215, 0.014009808248374611, 0.014009808248374611, 0.014009808248374611, 0.014009808248374611, 0.014324050745926797, 0.014324050745926797, 0.014324050745926797, 0.014324050745926797, 0.018292734341230243, 0.018292734341230243, 0.018292734341230243, 0.03272628099269544, 0.03272628099269544, 0.03272628099269544, 0.027157796488609165, 0.027157796488609165, 0.017211413010954857, 0.017211413010954857, 0.017211413010954857, 0.012955822996445931, 0.012955822996445931, 0.012955822996445931, 0.012955822996445931, 0.017078191003141303, 0.017078191003141303, 0.017078191003141303, 0.02565545149263926, 0.02565545149263926, 0.026133759994991124, 0.026133759994991124, 0.012745351748890243, 0.012745351748890243, 0.012745351748890243, 0.012745351748890243, 0.02526378951733932, 0.02526378951733932, 0.016981848670790594, 0.016981848670790594, 0.016981848670790594, 0.01753506433063497, 0.01753506433063497, 0.01753506433063497, 0.013146101002348587, 0.013146101002348587, 0.013146101002348587, 0.013146101002348587, 0.025147862499579787, 0.025147862499579787, 0.016644779650960118, 0.016644779650960118, 0.016644779650960118, 0.01670563401421532, 0.01670563401421532, 0.01670563401421532, 0.025114176998613402, 0.025114176998613402, 0.025240779505111277, 0.025240779505111277, 0.01760628231568262, 0.01760628231568262, 0.01760628231568262, 0.016931691012966137, 0.016931691012966137, 0.016931691012966137, 0.01727279700571671, 0.01727279700571671, 0.01727279700571671, 0.016912257318229724, 0.016912257318229724, 0.016912257318229724, 0.01598840901472916, 0.01598840901472916, 0.01598840901472916, 0.016010235665210832, 0.016010235665210832, 0.016010235665210832, 0.016034339651620638, 0.016034339651620638, 0.016034339651620638, 0.015974988675831508, 0.015974988675831508, 0.015974988675831508, 0.023938142490806058, 0.023938142490806058, 0.023689378518611193, 0.023689378518611193, 0.01592199399601668, 0.01592199399601668, 0.01592199399601668, 0.01598192499174426, 0.01598192499174426, 0.01598192499174426, 0.01586189299511413, 0.01586189299511413, 0.01586189299511413, 0.015926023673576612, 0.015926023673576612, 0.015926023673576612, 0.016315380674010765, 0.016315380674010765, 0.016315380674010765, 0.017069103002237778, 0.017069103002237778, 0.017069103002237778, 0.015235118994799754, 0.015235118994799754, 0.015235118994799754, 0.015159054659307003, 0.015159054659307003, 0.015159054659307003, 0.022672360995784402, 0.022672360995784402, 0.023076677520293742, 0.023076677520293742 ], [ 0.07599885431894411, 0.07599885431894411, 0.07599885431894411, 0.020025841678337503, 0.020025841678337503, 0.020025841678337503, 0.020075395664510626, 0.020075395664510626, 0.020075395664510626, 0.020034617996619392, 0.020034617996619392, 0.020034617996619392, 0.020026374336642522, 0.020026374336642522, 0.020026374336642522, 0.019823340660271544, 0.019823340660271544, 0.019823340660271544, 0.030416983005125076, 0.030416983005125076, 0.03106233850121498, 0.03106233850121498, 0.019885292005104322, 0.019885292005104322, 0.019885292005104322, 0.018892501985343795, 0.018892501985343795, 0.018892501985343795, 0.030985270665648084, 0.030985270665648084, 0.030985270665648084, 0.019343586017688114, 0.019343586017688114, 0.019343586017688114, 0.018950182983341318, 0.018950182983341318, 0.018950182983341318, 0.019099209011377145, 0.019099209011377145, 0.019099209011377145, 0.01903421765503784, 0.01903421765503784, 0.01903421765503784, 0.018937779338254284, 0.018937779338254284, 0.018937779338254284, 0.02875587999005802, 0.02875587999005802, 0.02856260701082647, 0.02856260701082647, 0.019304305334420253, 0.019304305334420253, 0.019304305334420253, 0.019817157338062923, 0.019817157338062923, 0.019817157338062923, 0.019061451001713674, 0.019061451001713674, 0.019061451001713674, 0.019113485332733642, 0.019113485332733642, 0.019113485332733642, 0.019104608334600925, 0.019104608334600925, 0.019104608334600925, 0.02907446399331093, 0.02907446399331093, 0.014618368746596389, 0.014618368746596389, 0.014618368746596389, 0.014618368746596389, 0.018610139998296898, 0.018610139998296898, 0.018610139998296898, 0.02757459500571713, 0.02757459500571713, 0.027460306999273598, 0.027460306999273598, 0.01380542025435716, 0.01380542025435716, 0.01380542025435716, 0.01380542025435716, 0.027853768988279626, 0.027853768988279626, 0.0183525126776658, 0.0183525126776658, 0.0183525126776658, 0.018347032329378028, 0.018347032329378028, 0.018347032329378028, 0.018484470997160923, 0.018484470997160923, 0.018484470997160923, 0.01830341199335332, 0.01830341199335332, 0.01830341199335332, 0.01845283400810634, 0.01845283400810634, 0.01845283400810634, 0.018417870334815234, 0.018417870334815234, 0.018417870334815234, 0.027498273499077186, 0.027498273499077186, 0.027074395504314452, 0.027074395504314452, 0.018667700661656756, 0.018667700661656756, 0.018667700661656756, 0.01851662133897965, 0.01851662133897965, 0.01851662133897965, 0.018641199994211394, 0.018641199994211394, 0.018641199994211394, 0.018580482341349125, 0.018580482341349125, 0.018580482341349125, 0.018559868990754087, 0.018559868990754087, 0.018559868990754087, 0.018557828676421195, 0.018557828676421195, 0.018557828676421195, 0.018460286994619917, 0.018460286994619917, 0.018460286994619917, 0.018637193328080077, 0.018637193328080077, 0.018637193328080077, 0.028849135007476434, 0.028849135007476434, 0.027445936488220468, 0.027445936488220468 ], [ 0.07609848600501816, 0.07609848600501816, 0.07609848600501816, 0.02001590799773112, 0.02001590799773112, 0.02001590799773112, 0.020074725985371817, 0.020074725985371817, 0.020074725985371817, 0.015025570755824447, 0.015025570755824447, 0.015025570755824447, 0.015025570755824447, 0.0200260576675646, 0.0200260576675646, 0.0200260576675646, 0.014868118247250095, 0.014868118247250095, 0.014868118247250095, 0.014868118247250095, 0.01529539775219746, 0.01529539775219746, 0.01529539775219746, 0.01529539775219746, 0.0618297120090574 ], [ 0.11415219900663942, 0.11415219900663942, 0.020037391669272136, 0.020037391669272136, 0.020037391669272136, 0.015055509502417408, 0.015055509502417408, 0.015055509502417408, 0.015055509502417408, 0.020034414328013856, 0.020034414328013856, 0.020034414328013856, 0.020025807665660977, 0.020025807665660977, 0.020025807665660977, 0.01982415100792423, 0.01982415100792423, 0.01982415100792423, 0.030516334489220753, 0.030516334489220753, 0.015466665499843657, 0.015466665499843657, 0.015466665499843657, 0.015466665499843657, 0.019884118674478184, 0.019884118674478184, 0.019884118674478184, 0.028335032984614372, 0.028335032984614372, 0.023239655507495627, 0.023239655507495627, 0.023239655507495627, 0.023239655507495627, 0.014507339743431658, 0.014507339743431658, 0.014507339743431658, 0.014507339743431658, 0.01895076932851225, 0.01895076932851225, 0.01895076932851225, 0.019099789011913042, 0.019099789011913042, 0.019099789011913042, 0.028550306480610743, 0.028550306480610743, 0.018938216341969866, 0.018938216341969866, 0.018938216341969866, 0.019193043660682935, 0.019193043660682935, 0.019193043660682935, 0.019044294662307948, 0.019044294662307948, 0.019044294662307948, 0.019304565348041553, 0.019304565348041553, 0.019304565348041553, 0.014856965237413533, 0.014856965237413533, 0.014856965237413533, 0.014856965237413533, 0.014296558263595216, 0.014296558263595216, 0.014296558263595216, 0.014296558263595216, 0.01911949533193062, 0.01911949533193062, 0.01911949533193062, 0.019099241665874917, 0.019099241665874917, 0.019099241665874917, 0.029074459482217208, 0.029074459482217208, 0.01949675833263124, 0.01949675833263124, 0.01949675833263124, 0.013956615002825856, 0.013956615002825856, 0.013956615002825856, 0.013956615002825856, 0.018378923006821424, 0.018378923006821424, 0.018378923006821424, 0.027460777491796762, 0.027460777491796762, 0.013805887501803227, 0.013805887501803227, 0.013805887501803227, 0.013805887501803227, 0.013926447005360387, 0.013926447005360387, 0.013926447005360387, 0.013926447005360387, 0.01835258931775267, 0.01835258931775267, 0.01835258931775267, 0.018347795664643247, 0.018347795664643247, 0.018347795664643247, 0.013863318512449041, 0.013863318512449041, 0.013863318512449041, 0.013863318512449041, 0.027453502494608983, 0.027453502494608983, 0.01845339099721362, 0.01845339099721362, 0.01845339099721362, 0.018418139991505694, 0.018418139991505694, 0.018418139991505694, 0.027505743521032855, 0.027505743521032855, 0.027068145980592817, 0.027068145980592817, 0.0186672803441373, 0.0186672803441373, 0.0186672803441373, 0.018516397996184725, 0.018516397996184725, 0.018516397996184725, 0.018641243669359635, 0.018641243669359635, 0.018641243669359635, 0.018581482329561066, 0.018581482329561066, 0.018581482329561066, 0.01392028150439728, 0.01392028150439728, 0.01392028150439728, 0.01392028150439728, 0.027841487986734137, 0.027841487986734137, 0.018460020675168682, 0.018460020675168682, 0.018460020675168682, 0.01867730700178072, 0.01867730700178072, 0.01867730700178072, 0.028588780987774953, 0.028588780987774953, 0.018283454002812505, 0.018283454002812505, 0.018283454002812505, 0.02455506350088399, 0.02455506350088399, 0.02455506350088399, 0.02455506350088399, 0.018118704669177532, 0.018118704669177532, 0.018118704669177532, 0.012909814744489267, 0.012909814744489267, 0.012909814744489267, 0.012909814744489267, 0.012955870508449152, 0.012955870508449152, 0.012955870508449152, 0.012955870508449152, 0.025617786479415372, 0.025617786479415372, 0.01710334767509873, 0.01710334767509873, 0.01710334767509873, 0.01738610933534801, 0.01738610933534801, 0.01738610933534801, 0.025520398980006576, 0.025520398980006576, 0.016854186678150047, 0.016854186678150047, 0.016854186678150047, 0.01274280650250148, 0.01274280650250148, 0.01274280650250148, 0.01274280650250148, 0.02618332498241216, 0.02618332498241216 ], [ 0.07610312267206609, 0.07610312267206609, 0.07610312267206609, 0.01501510600792244, 0.01501510600792244, 0.01501510600792244, 0.01501510600792244, 0.020073268989411492, 0.020073268989411492, 0.020073268989411492, 0.020034901332110167, 0.020034901332110167, 0.020034901332110167, 0.01501977325824555, 0.01501977325824555, 0.01501977325824555, 0.01501977325824555, 0.02973549149464816, 0.02973549149464816, 0.06096841796534136 ] ], "generated_texts": [ " the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed", " dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit volupt", " quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog.", "piciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The", " The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspic", "atem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem", " jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem", "nis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis", " error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde omnis iste natus error sit voluptatem. The quick brown fox jumps over", " voluptatem. The quick brown fox jumps over the lazy dog. Sed ut perspiciatis unde om" ], "errors": [ "", "", "", "", "", "", "", "", "", "" ] } ] } ] }