{
  "metadata": {
    "experiment": "dsv4_h200_vllm",
    "run_id": "20260708-062348",
    "timestamp": "2026-07-08T06:23:48+00:00",
    "model": "/data/models/DeepSeek-V4-Flash",
    "backend": "vllm",
    "engine": "vllm",
    "hardware": "8x NVIDIA H200 143GB",
    "accelerator": "NVIDIA H200",
    "chip": "nvidia_h200",
    "script": "experiments/dsv4_h200_vllm/run_bench.sh",
    "env": "/data/user1/yy/envs/vllm",
    "git_commit": "acf2e3d",
    "git_dirty": "dirty",
    "description": "H200 native vLLM baseline benchmark for DeepSeek-V4-Flash"
  },
  "config": {
    "tp": 4,
    "kv_cache_dtype": "fp8",
    "block_size": 256,
    "max_num_seqs": 256,
    "port": 30005,
    "num_prompts": 128,
    "scenarios": [
      "32 512 256",
      "128 512 256",
      "32 4000 512"
    ]
  },
  "scenarios": [
    {
      "name": "c128_i512_o256",
      "config": {
        "concurrency": 128,
        "input_len": 512,
        "output_len": 256,
        "dataset": "random",
        "num_prompts": 128
      },
      "metrics": {
        "success": 128,
        "failed": 0,
        "duration_s": 5.98181911901338,
        "request_throughput": 21.39817293925662,
        "input_token_throughput": 5892.521873147793,
        "output_token_throughput": 2825.2275208862256,
        "total_token_throughput": 8717.749394034017,
        "total_input_tokens": 35248,
        "total_output_tokens": 16900,
        "e2e_ms": {
          "mean": 3786.2239771795885,
          "p50": 3966.506554497755,
          "p90": 5763.936419402307,
          "p95": 5857.902326301701,
          "p99": 5919.940241662553
        },
        "ttft_ms": {
          "mean": 634.9285282819892,
          "p50": 651.9191234983737,
          "p90": 872.5053475049208,
          "p95": 873.6197502956202,
          "p99": 879.9700354097877
        },
        "tpot_ms": {
          "mean": 28.086932902452045,
          "p50": 24.940565250290092,
          "p90": 33.6093050082046,
          "p95": 40.66724980024446,
          "p99": 91.16725345989823
        },
        "itl_ms": {
          "mean": 24.055493866406856,
          "p50": 22.871914501592983,
          "p90": 26.806968497112393,
          "p95": 27.607820050616283,
          "p99": 32.99604622545303
        }
      },
      "raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_128_512_256.jsonl"
    },
    {
      "name": "c32_i4000_o512",
      "config": {
        "concurrency": 32,
        "input_len": 4000,
        "output_len": 512,
        "dataset": "random",
        "num_prompts": 128
      },
      "metrics": {
        "success": 128,
        "failed": 0,
        "duration_s": 63.96937208699819,
        "request_throughput": 2.000957580542143,
        "input_token_throughput": 4229.977427822796,
        "output_token_throughput": 505.9765156984964,
        "total_token_throughput": 4735.953943521293,
        "total_input_tokens": 270589,
        "total_output_tokens": 32367,
        "e2e_ms": {
          "mean": 15397.358620969157,
          "p50": 10554.476282501128,
          "p90": 39473.02784240455,
          "p95": 42952.8573812,
          "p99": 46968.37465279941
        },
        "ttft_ms": {
          "mean": 917.0502477351192,
          "p50": 236.0078425044776,
          "p90": 2404.053003701846,
          "p95": 2853.309046203503,
          "p99": 6638.214706212312
        },
        "tpot_ms": {
          "mean": 62.674261138657926,
          "p50": 38.76617610605361,
          "p90": 141.09892115795085,
          "p95": 165.57312805125648,
          "p99": 219.71921567381835
        },
        "itl_ms": {
          "mean": 57.50244012726164,
          "p50": 14.738622994627804,
          "p90": 19.66081059363205,
          "p95": 142.28972980054095,
          "p99": 1669.9934365210356
        }
      },
      "raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_32_4000_512.jsonl"
    },
    {
      "name": "c32_i512_o256",
      "config": {
        "concurrency": 32,
        "input_len": 512,
        "output_len": 256,
        "dataset": "random",
        "num_prompts": 128
      },
      "metrics": {
        "success": 128,
        "failed": 0,
        "duration_s": 38.25599173500086,
        "request_throughput": 3.3458811076355204,
        "input_token_throughput": 921.3720100151314,
        "output_token_throughput": 441.7608649925023,
        "total_token_throughput": 1363.1328750076339,
        "total_input_tokens": 35248,
        "total_output_tokens": 16900,
        "e2e_ms": {
          "mean": 9197.956188436932,
          "p50": 6626.5614014992025,
          "p90": 20719.612499210045,
          "p95": 27486.685669697905,
          "p99": 28020.42960330684
        },
        "ttft_ms": {
          "mean": 1228.4664978981255,
          "p50": 132.46312249975745,
          "p90": 3753.0612517031777,
          "p95": 11441.480474699347,
          "p99": 11446.530493390019
        },
        "tpot_ms": {
          "mean": 79.49513794690715,
          "p50": 35.04621934244705,
          "p90": 158.92302877744768,
          "p95": 246.69572046122963,
          "p99": 674.493606643706
        },
        "itl_ms": {
          "mean": 60.85736878373586,
          "p50": 15.390933993330691,
          "p90": 47.24743370315991,
          "p95": 57.582202005869476,
          "p99": 237.64244167061403
        }
      },
      "raw_file": "experiments/dsv4_h200_vllm/results/20260708-062348/raw_outputs/vllm_0708_32_512_256.jsonl"
    }
  ]
}