Upload 3 files

Dockerfile

@@ -0,0 +1,13 @@
+FROM python:3-bookworm
+RUN apt-get update && apt-get install -y \
+    git \
+    && rm -rf /var/lib/apt/lists/*
+RUN curl -sSL https://install.python-poetry.org | python3 -
+ENV PATH="/root/.local/bin:$PATH"
+WORKDIR /app
+RUN git clone https://github.com/huggingface/inference-benchmarker.git \
+    && cd inference-benchmarker \
+    && poetry install --no-root
+COPY results results
+WORKDIR /app/inference-benchmarker
+CMD ["poetry", "run", "python", "extra/dashboard/app.py", "--from-results-dir", "/app/results"]

results/meta-llama_Llama-3_1-8B-Instruct_2025-04-02-07-50-23.json

@@ -0,0 +1,951 @@
+{
+  "config": {
+    "max_vus": 128,
+    "duration_secs": 120,
+    "benchmark_kind": "Rate",
+    "warmup_duration_secs": 30,
+    "rates": [
+      0.8,
+      2.4,
+      4.0,
+      5.6,
+      7.2,
+      8.8,
+      10.4,
+      12.0,
+      13.6,
+      15.2,
+      16.8,
+      18.4,
+      20.0,
+      21.6,
+      23.2,
+      24.0
+    ],
+    "num_rates": 10,
+    "prompt_options": null,
+    "decode_options": {
+      "num_tokens": 200,
+      "min_tokens": 200,
+      "max_tokens": 200,
+      "variance": 100
+    },
+    "tokenizer": "meta-llama/Llama-3.1-8B-Instruct",
+    "model_name": "meta-llama/Llama-3.1-8B-Instruct",
+    "profile": null,
+    "meta": {
+      "version": "4d28897b4e345f4dfdd93d3434e50ac8afcdf9e1",
+      "engine": "TGI",
+      "tp": "1"
+    },
+    "run_id": "TGI-3.2.1"
+  },
+  "results": [
+    {
+      "id": "warmup",
+      "executor_type": "ConstantVUs",
+      "config": {
+        "max_vus": 1,
+        "duration_secs": 30,
+        "rate": null
+      },
+      "total_requests": 22,
+      "total_tokens": 3726,
+      "token_throughput_secs": 121.61045252166194,
+      "duration_ms": 30638,
+      "time_to_first_token_ms": {
+        "p50": 46.607,
+        "p60": 46.786,
+        "p70": 46.957,
+        "p80": 47.13,
+        "p90": 48.231,
+        "p95": 78.293,
+        "p99": 80.85,
+        "avg": 48.554
+      },
+      "inter_token_latency_ms": {
+        "p50": 7.973,
+        "p60": 7.982,
+        "p70": 7.99,
+        "p80": 8.0,
+        "p90": 8.043,
+        "p95": 8.083,
+        "p99": 8.125,
+        "avg": 7.927
+      },
+      "failed_requests": 0,
+      "successful_requests": 22,
+      "request_rate": 0.7180434663114769,
+      "total_tokens_sent": 1613,
+      "e2e_latency_ms": {
+        "p50": 1632.456,
+        "p60": 1633.79,
+        "p70": 1636.485,
+        "p80": 1638.246,
+        "p90": 1647.577,
+        "p95": 1688.143,
+        "p99": 1697.2,
+        "avg": 1392.459
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 0.8
+      },
+      "total_requests": 94,
+      "total_tokens": 14823,
+      "token_throughput_secs": 125.70685354915,
+      "duration_ms": 117917,
+      "time_to_first_token_ms": {
+        "p50": 26.689,
+        "p60": 28.276,
+        "p70": 29.925,
+        "p80": 31.171,
+        "p90": 31.927,
+        "p95": 32.252,
+        "p99": 33.737,
+        "avg": 27.559
+      },
+      "inter_token_latency_ms": {
+        "p50": 8.297,
+        "p60": 8.308,
+        "p70": 8.327,
+        "p80": 8.348,
+        "p90": 8.372,
+        "p95": 8.387,
+        "p99": 8.48,
+        "avg": 8.274
+      },
+      "failed_requests": 0,
+      "successful_requests": 94,
+      "request_rate": 0.7971695495932065,
+      "total_tokens_sent": 13915,
+      "e2e_latency_ms": {
+        "p50": 1675.47,
+        "p60": 1680.097,
+        "p70": 1683.628,
+        "p80": 1686.037,
+        "p90": 1689.112,
+        "p95": 1692.598,
+        "p99": 1711.849,
+        "avg": 1329.915
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 2.4
+      },
+      "total_requests": 283,
+      "total_tokens": 49472,
+      "token_throughput_secs": 417.57128544344323,
+      "duration_ms": 118475,
+      "time_to_first_token_ms": {
+        "p50": 29.056,
+        "p60": 29.943,
+        "p70": 30.586,
+        "p80": 31.684,
+        "p90": 32.58,
+        "p95": 33.297,
+        "p99": 42.848,
+        "avg": 29.12
+      },
+      "inter_token_latency_ms": {
+        "p50": 8.925,
+        "p60": 8.931,
+        "p70": 8.939,
+        "p80": 8.947,
+        "p90": 8.954,
+        "p95": 8.965,
+        "p99": 8.982,
+        "avg": 8.669
+      },
+      "failed_requests": 0,
+      "successful_requests": 283,
+      "request_rate": 2.3886779143858026,
+      "total_tokens_sent": 44403,
+      "e2e_latency_ms": {
+        "p50": 1804.541,
+        "p60": 1805.909,
+        "p70": 1807.748,
+        "p80": 1809.553,
+        "p90": 1812.308,
+        "p95": 1814.507,
+        "p99": 1817.862,
+        "avg": 1579.6
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 4.0
+      },
+      "total_requests": 474,
+      "total_tokens": 85697,
+      "token_throughput_secs": 720.8187174458998,
+      "duration_ms": 118888,
+      "time_to_first_token_ms": {
+        "p50": 28.261,
+        "p60": 29.236,
+        "p70": 30.047,
+        "p80": 31.071,
+        "p90": 32.258,
+        "p95": 32.956,
+        "p99": 44.447,
+        "avg": 28.686
+      },
+      "inter_token_latency_ms": {
+        "p50": 9.365,
+        "p60": 9.383,
+        "p70": 9.396,
+        "p80": 9.415,
+        "p90": 9.433,
+        "p95": 9.446,
+        "p99": 9.457,
+        "avg": 9.313
+      },
+      "failed_requests": 0,
+      "successful_requests": 474,
+      "request_rate": 3.9869315386694577,
+      "total_tokens_sent": 56591,
+      "e2e_latency_ms": {
+        "p50": 1890.864,
+        "p60": 1894.873,
+        "p70": 1898.381,
+        "p80": 1902.406,
+        "p90": 1906.151,
+        "p95": 1909.095,
+        "p99": 1912.544,
+        "avg": 1712.674
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 5.6
+      },
+      "total_requests": 660,
+      "total_tokens": 115159,
+      "token_throughput_secs": 966.2563530237055,
+      "duration_ms": 119180,
+      "time_to_first_token_ms": {
+        "p50": 29.312,
+        "p60": 30.64,
+        "p70": 31.75,
+        "p80": 32.952,
+        "p90": 44.096,
+        "p95": 46.404,
+        "p99": 48.15,
+        "avg": 30.915
+      },
+      "inter_token_latency_ms": {
+        "p50": 9.98,
+        "p60": 9.996,
+        "p70": 10.004,
+        "p80": 10.013,
+        "p90": 10.022,
+        "p95": 10.028,
+        "p99": 10.062,
+        "avg": 9.803
+      },
+      "failed_requests": 0,
+      "successful_requests": 660,
+      "request_rate": 5.537814612801827,
+      "total_tokens_sent": 113449,
+      "e2e_latency_ms": {
+        "p50": 2013.741,
+        "p60": 2017.376,
+        "p70": 2019.935,
+        "p80": 2021.638,
+        "p90": 2024.968,
+        "p95": 2026.227,
+        "p99": 2032.895,
+        "avg": 1754.082
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 7.2
+      },
+      "total_requests": 849,
+      "total_tokens": 151779,
+      "token_throughput_secs": 1267.369912316427,
+      "duration_ms": 119759,
+      "time_to_first_token_ms": {
+        "p50": 30.493,
+        "p60": 31.719,
+        "p70": 33.123,
+        "p80": 36.078,
+        "p90": 46.155,
+        "p95": 46.769,
+        "p99": 48.162,
+        "avg": 32.604
+      },
+      "inter_token_latency_ms": {
+        "p50": 10.423,
+        "p60": 10.475,
+        "p70": 10.5,
+        "p80": 10.524,
+        "p90": 10.555,
+        "p95": 10.575,
+        "p99": 10.597,
+        "avg": 10.321
+      },
+      "failed_requests": 0,
+      "successful_requests": 849,
+      "request_rate": 7.089235372196724,
+      "total_tokens_sent": 145128,
+      "e2e_latency_ms": {
+        "p50": 2100.596,
+        "p60": 2114.214,
+        "p70": 2119.354,
+        "p80": 2124.055,
+        "p90": 2130.161,
+        "p95": 2134.421,
+        "p99": 2140.161,
+        "avg": 1884.625
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 8.8
+      },
+      "total_requests": 1041,
+      "total_tokens": 181439,
+      "token_throughput_secs": 1528.9336762174746,
+      "duration_ms": 118670,
+      "time_to_first_token_ms": {
+        "p50": 33.409,
+        "p60": 41.426,
+        "p70": 45.116,
+        "p80": 45.65,
+        "p90": 47.216,
+        "p95": 47.524,
+        "p99": 47.814,
+        "avg": 36.264
+      },
+      "inter_token_latency_ms": {
+        "p50": 11.019,
+        "p60": 11.044,
+        "p70": 11.072,
+        "p80": 11.102,
+        "p90": 11.136,
+        "p95": 11.159,
+        "p99": 11.222,
+        "avg": 10.862
+      },
+      "failed_requests": 0,
+      "successful_requests": 1041,
+      "request_rate": 8.77220419503189,
+      "total_tokens_sent": 191937,
+      "e2e_latency_ms": {
+        "p50": 2223.571,
+        "p60": 2232.271,
+        "p70": 2241.046,
+        "p80": 2246.104,
+        "p90": 2253.422,
+        "p95": 2259.594,
+        "p99": 2267.867,
+        "avg": 1944.18
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 10.4
+      },
+      "total_requests": 1225,
+      "total_tokens": 208853,
+      "token_throughput_secs": 1750.44303623107,
+      "duration_ms": 119314,
+      "time_to_first_token_ms": {
+        "p50": 34.056,
+        "p60": 40.904,
+        "p70": 44.889,
+        "p80": 45.567,
+        "p90": 47.156,
+        "p95": 47.537,
+        "p99": 47.805,
+        "avg": 36.461
+      },
+      "inter_token_latency_ms": {
+        "p50": 11.655,
+        "p60": 11.693,
+        "p70": 11.73,
+        "p80": 11.772,
+        "p90": 11.827,
+        "p95": 11.861,
+        "p99": 11.909,
+        "avg": 11.475
+      },
+      "failed_requests": 0,
+      "successful_requests": 1225,
+      "request_rate": 10.266995060559632,
+      "total_tokens_sent": 219623,
+      "e2e_latency_ms": {
+        "p50": 2351.289,
+        "p60": 2360.059,
+        "p70": 2368.244,
+        "p80": 2378.908,
+        "p90": 2388.768,
+        "p95": 2395.775,
+        "p99": 2405.872,
+        "avg": 2012.23
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 12.0
+      },
+      "total_requests": 1411,
+      "total_tokens": 255188,
+      "token_throughput_secs": 2159.7525641398156,
+      "duration_ms": 118156,
+      "time_to_first_token_ms": {
+        "p50": 40.727,
+        "p60": 44.09,
+        "p70": 44.798,
+        "p80": 46.295,
+        "p90": 46.858,
+        "p95": 48.094,
+        "p99": 48.34,
+        "avg": 38.454
+      },
+      "inter_token_latency_ms": {
+        "p50": 12.566,
+        "p60": 12.629,
+        "p70": 12.67,
+        "p80": 12.718,
+        "p90": 12.802,
+        "p95": 12.951,
+        "p99": 13.08,
+        "avg": 12.404
+      },
+      "failed_requests": 0,
+      "successful_requests": 1411,
+      "request_rate": 11.941826684645358,
+      "total_tokens_sent": 272561,
+      "e2e_latency_ms": {
+        "p50": 2532.282,
+        "p60": 2547.776,
+        "p70": 2560.142,
+        "p80": 2570.947,
+        "p90": 2585.7,
+        "p95": 2612.572,
+        "p99": 2635.79,
+        "avg": 2297.575
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 13.6
+      },
+      "total_requests": 1564,
+      "total_tokens": 272803,
+      "token_throughput_secs": 2293.5215462094866,
+      "duration_ms": 118945,
+      "time_to_first_token_ms": {
+        "p50": 44.852,
+        "p60": 46.319,
+        "p70": 46.872,
+        "p80": 48.169,
+        "p90": 49.674,
+        "p95": 51.154,
+        "p99": 53.485,
+        "avg": 42.864
+      },
+      "inter_token_latency_ms": {
+        "p50": 25.88,
+        "p60": 26.011,
+        "p70": 26.149,
+        "p80": 26.243,
+        "p90": 26.308,
+        "p95": 26.354,
+        "p99": 26.436,
+        "avg": 24.547
+      },
+      "failed_requests": 0,
+      "successful_requests": 1564,
+      "request_rate": 13.148930540615892,
+      "total_tokens_sent": 255933,
+      "e2e_latency_ms": {
+        "p50": 5177.262,
+        "p60": 5201.64,
+        "p70": 5238.214,
+        "p80": 5261.34,
+        "p90": 5275.17,
+        "p95": 5286.208,
+        "p99": 5302.579,
+        "avg": 4359.271
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 15.2
+      },
+      "total_requests": 1759,
+      "total_tokens": 309348,
+      "token_throughput_secs": 2580.5086165701296,
+      "duration_ms": 119878,
+      "time_to_first_token_ms": {
+        "p50": 45.707,
+        "p60": 47.05,
+        "p70": 47.656,
+        "p80": 48.721,
+        "p90": 51.476,
+        "p95": 53.138,
+        "p99": 55.565,
+        "avg": 44.101
+      },
+      "inter_token_latency_ms": {
+        "p50": 26.733,
+        "p60": 26.968,
+        "p70": 27.102,
+        "p80": 27.19,
+        "p90": 27.265,
+        "p95": 27.368,
+        "p99": 27.441,
+        "avg": 26.359
+      },
+      "failed_requests": 0,
+      "successful_requests": 1759,
+      "request_rate": 14.673166325778274,
+      "total_tokens_sent": 270134,
+      "e2e_latency_ms": {
+        "p50": 5326.968,
+        "p60": 5382.346,
+        "p70": 5428.823,
+        "p80": 5452.294,
+        "p90": 5469.218,
+        "p95": 5483.97,
+        "p99": 5502.975,
+        "avg": 4685.437
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 16.8
+      },
+      "total_requests": 1928,
+      "total_tokens": 330549,
+      "token_throughput_secs": 2759.3148813615194,
+      "duration_ms": 119793,
+      "time_to_first_token_ms": {
+        "p50": 46.472,
+        "p60": 46.857,
+        "p70": 48.168,
+        "p80": 50.368,
+        "p90": 53.407,
+        "p95": 55.802,
+        "p99": 72.801,
+        "avg": 45.931
+      },
+      "inter_token_latency_ms": {
+        "p50": 27.881,
+        "p60": 28.021,
+        "p70": 28.093,
+        "p80": 28.16,
+        "p90": 28.336,
+        "p95": 28.602,
+        "p99": 29.638,
+        "avg": 27.078
+      },
+      "failed_requests": 0,
+      "successful_requests": 1928,
+      "request_rate": 16.094313070876055,
+      "total_tokens_sent": 320581,
+      "e2e_latency_ms": {
+        "p50": 5553.009,
+        "p60": 5600.423,
+        "p70": 5629.869,
+        "p80": 5644.616,
+        "p90": 5671.609,
+        "p95": 5710.139,
+        "p99": 5944.138,
+        "avg": 4721.266
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 18.4
+      },
+      "total_requests": 2118,
+      "total_tokens": 357373,
+      "token_throughput_secs": 2982.4464814578682,
+      "duration_ms": 119825,
+      "time_to_first_token_ms": {
+        "p50": 59.209,
+        "p60": 64.399,
+        "p70": 69.696,
+        "p80": 75.067,
+        "p90": 80.705,
+        "p95": 84.049,
+        "p99": 88.502,
+        "avg": 59.862
+      },
+      "inter_token_latency_ms": {
+        "p50": 28.563,
+        "p60": 28.749,
+        "p70": 28.916,
+        "p80": 29.057,
+        "p90": 29.191,
+        "p95": 29.317,
+        "p99": 30.202,
+        "avg": 28.34
+      },
+      "failed_requests": 0,
+      "successful_requests": 2118,
+      "request_rate": 17.67571038586509,
+      "total_tokens_sent": 398184,
+      "e2e_latency_ms": {
+        "p50": 5708.277,
+        "p60": 5743.555,
+        "p70": 5787.666,
+        "p80": 5828.147,
+        "p90": 5862.892,
+        "p95": 5894.271,
+        "p99": 6066.881,
+        "avg": 4844.433
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 20.0
+      },
+      "total_requests": 2290,
+      "total_tokens": 386940,
+      "token_throughput_secs": 3227.511698507537,
+      "duration_ms": 119888,
+      "time_to_first_token_ms": {
+        "p50": 58.122,
+        "p60": 63.978,
+        "p70": 70.049,
+        "p80": 76.219,
+        "p90": 82.824,
+        "p95": 86.111,
+        "p99": 90.923,
+        "avg": 59.703
+      },
+      "inter_token_latency_ms": {
+        "p50": 29.32,
+        "p60": 29.492,
+        "p70": 29.895,
+        "p80": 30.4,
+        "p90": 30.546,
+        "p95": 31.209,
+        "p99": 31.92,
+        "avg": 28.595
+      },
+      "failed_requests": 0,
+      "successful_requests": 2290,
+      "request_rate": 19.10115725844384,
+      "total_tokens_sent": 346259,
+      "e2e_latency_ms": {
+        "p50": 5821.093,
+        "p60": 5905.813,
+        "p70": 5981.542,
+        "p80": 6088.576,
+        "p90": 6138.484,
+        "p95": 6172.066,
+        "p99": 6421.6,
+        "avg": 4983.079
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 21.6
+      },
+      "total_requests": 2445,
+      "total_tokens": 438323,
+      "token_throughput_secs": 3667.2404231554137,
+      "duration_ms": 119523,
+      "time_to_first_token_ms": {
+        "p50": 60.891,
+        "p60": 67.221,
+        "p70": 73.352,
+        "p80": 80.009,
+        "p90": 86.246,
+        "p95": 89.966,
+        "p99": 95.937,
+        "avg": 62.545
+      },
+      "inter_token_latency_ms": {
+        "p50": 31.059,
+        "p60": 31.174,
+        "p70": 31.267,
+        "p80": 31.363,
+        "p90": 31.564,
+        "p95": 31.98,
+        "p99": 32.368,
+        "avg": 30.585
+      },
+      "failed_requests": 0,
+      "successful_requests": 2445,
+      "request_rate": 20.456154102374246,
+      "total_tokens_sent": 375764,
+      "e2e_latency_ms": {
+        "p50": 6210.518,
+        "p60": 6251.837,
+        "p70": 6276.884,
+        "p80": 6301.657,
+        "p90": 6339.383,
+        "p95": 6390.942,
+        "p99": 6504.767,
+        "avg": 5545.311
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 23.2
+      },
+      "total_requests": 2557,
+      "total_tokens": 451146,
+      "token_throughput_secs": 3774.5098883710634,
+      "duration_ms": 119524,
+      "time_to_first_token_ms": {
+        "p50": 61.047,
+        "p60": 67.493,
+        "p70": 74.009,
+        "p80": 80.579,
+        "p90": 87.143,
+        "p95": 90.788,
+        "p99": 95.729,
+        "avg": 63.122
+      },
+      "inter_token_latency_ms": {
+        "p50": 31.013,
+        "p60": 31.094,
+        "p70": 31.187,
+        "p80": 31.29,
+        "p90": 31.621,
+        "p95": 32.321,
+        "p99": 32.547,
+        "avg": 30.814
+      },
+      "failed_requests": 0,
+      "successful_requests": 2557,
+      "request_rate": 21.393122812935964,
+      "total_tokens_sent": 397220,
+      "e2e_latency_ms": {
+        "p50": 6207.811,
+        "p60": 6232.278,
+        "p70": 6255.491,
+        "p80": 6280.109,
+        "p90": 6327.14,
+        "p95": 6467.71,
+        "p99": 6530.251,
+        "avg": 5488.01
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 24.0
+      },
+      "total_requests": 2540,
+      "total_tokens": 454462,
+      "token_throughput_secs": 3793.8392202012997,
+      "duration_ms": 119789,
+      "time_to_first_token_ms": {
+        "p50": 61.504,
+        "p60": 67.687,
+        "p70": 74.328,
+        "p80": 81.247,
+        "p90": 87.806,
+        "p95": 91.29,
+        "p99": 96.912,
+        "avg": 63.516
+      },
+      "inter_token_latency_ms": {
+        "p50": 31.192,
+        "p60": 31.251,
+        "p70": 31.333,
+        "p80": 31.475,
+        "p90": 31.907,
+        "p95": 32.12,
+        "p99": 32.242,
+        "avg": 30.921
+      },
+      "failed_requests": 0,
+      "successful_requests": 2540,
+      "request_rate": 21.203866592391226,
+      "total_tokens_sent": 455432,
+      "e2e_latency_ms": {
+        "p50": 6257.817,
+        "p60": 6275.79,
+        "p70": 6293.476,
+        "p80": 6317.243,
+        "p90": 6372.71,
+        "p95": 6449.819,
+        "p99": 6483.148,
+        "avg": 5596.165
+      }
+    }
+  ],
+  "start_time": "2025-04-02T07:17:06.471305459+00:00",
+  "end_time": "2025-04-02T07:50:23.157564514+00:00",
+  "system": {
+    "cpu": [
+      "AMD EPYC 7R13 Processor cpu0@2649MHz",
+      "AMD EPYC 7R13 Processor cpu1@2516MHz",
+      "AMD EPYC 7R13 Processor cpu2@2516MHz",
+      "AMD EPYC 7R13 Processor cpu3@2649MHz",
+      "AMD EPYC 7R13 Processor cpu4@2516MHz",
+      "AMD EPYC 7R13 Processor cpu5@2516MHz",
+      "AMD EPYC 7R13 Processor cpu6@2516MHz",
+      "AMD EPYC 7R13 Processor cpu7@2649MHz",
+      "AMD EPYC 7R13 Processor cpu8@2649MHz",
+      "AMD EPYC 7R13 Processor cpu9@2649MHz",
+      "AMD EPYC 7R13 Processor cpu10@2649MHz",
+      "AMD EPYC 7R13 Processor cpu11@2649MHz",
+      "AMD EPYC 7R13 Processor cpu12@2516MHz",
+      "AMD EPYC 7R13 Processor cpu13@2881MHz",
+      "AMD EPYC 7R13 Processor cpu14@2516MHz",
+      "AMD EPYC 7R13 Processor cpu15@2649MHz",
+      "AMD EPYC 7R13 Processor cpu16@2649MHz",
+      "AMD EPYC 7R13 Processor cpu17@2649MHz",
+      "AMD EPYC 7R13 Processor cpu18@2516MHz",
+      "AMD EPYC 7R13 Processor cpu19@2516MHz",
+      "AMD EPYC 7R13 Processor cpu20@2516MHz",
+      "AMD EPYC 7R13 Processor cpu21@2516MHz",
+      "AMD EPYC 7R13 Processor cpu22@3083MHz",
+      "AMD EPYC 7R13 Processor cpu23@2649MHz",
+      "AMD EPYC 7R13 Processor cpu24@2516MHz",
+      "AMD EPYC 7R13 Processor cpu25@2516MHz",
+      "AMD EPYC 7R13 Processor cpu26@2516MHz",
+      "AMD EPYC 7R13 Processor cpu27@2516MHz",
+      "AMD EPYC 7R13 Processor cpu28@2649MHz",
+      "AMD EPYC 7R13 Processor cpu29@2516MHz",
+      "AMD EPYC 7R13 Processor cpu30@2516MHz",
+      "AMD EPYC 7R13 Processor cpu31@2516MHz",
+      "AMD EPYC 7R13 Processor cpu32@2516MHz",
+      "AMD EPYC 7R13 Processor cpu33@2516MHz",
+      "AMD EPYC 7R13 Processor cpu34@2649MHz",
+      "AMD EPYC 7R13 Processor cpu35@2649MHz",
+      "AMD EPYC 7R13 Processor cpu36@2516MHz",
+      "AMD EPYC 7R13 Processor cpu37@2516MHz",
+      "AMD EPYC 7R13 Processor cpu38@2516MHz",
+      "AMD EPYC 7R13 Processor cpu39@2649MHz",
+      "AMD EPYC 7R13 Processor cpu40@2649MHz",
+      "AMD EPYC 7R13 Processor cpu41@2649MHz",
+      "AMD EPYC 7R13 Processor cpu42@2516MHz",
+      "AMD EPYC 7R13 Processor cpu43@2516MHz",
+      "AMD EPYC 7R13 Processor cpu44@2881MHz",
+      "AMD EPYC 7R13 Processor cpu45@2516MHz",
+      "AMD EPYC 7R13 Processor cpu46@2516MHz",
+      "AMD EPYC 7R13 Processor cpu47@2649MHz",
+      "AMD EPYC 7R13 Processor cpu48@2516MHz",
+      "AMD EPYC 7R13 Processor cpu49@2516MHz",
+      "AMD EPYC 7R13 Processor cpu50@2516MHz",
+      "AMD EPYC 7R13 Processor cpu51@2649MHz",
+      "AMD EPYC 7R13 Processor cpu52@2649MHz",
+      "AMD EPYC 7R13 Processor cpu53@2649MHz",
+      "AMD EPYC 7R13 Processor cpu54@2516MHz",
+      "AMD EPYC 7R13 Processor cpu55@2516MHz",
+      "AMD EPYC 7R13 Processor cpu56@2516MHz",
+      "AMD EPYC 7R13 Processor cpu57@2516MHz",
+      "AMD EPYC 7R13 Processor cpu58@2516MHz",
+      "AMD EPYC 7R13 Processor cpu59@2516MHz",
+      "AMD EPYC 7R13 Processor cpu60@2516MHz",
+      "AMD EPYC 7R13 Processor cpu61@2516MHz",
+      "AMD EPYC 7R13 Processor cpu62@3007MHz",
+      "AMD EPYC 7R13 Processor cpu63@2649MHz",
+      "AMD EPYC 7R13 Processor cpu64@2649MHz",
+      "AMD EPYC 7R13 Processor cpu65@2649MHz",
+      "AMD EPYC 7R13 Processor cpu66@2649MHz",
+      "AMD EPYC 7R13 Processor cpu67@2649MHz",
+      "AMD EPYC 7R13 Processor cpu68@2649MHz",
+      "AMD EPYC 7R13 Processor cpu69@2649MHz",
+      "AMD EPYC 7R13 Processor cpu70@2649MHz",
+      "AMD EPYC 7R13 Processor cpu71@2649MHz",
+      "AMD EPYC 7R13 Processor cpu72@2649MHz",
+      "AMD EPYC 7R13 Processor cpu73@2649MHz",
+      "AMD EPYC 7R13 Processor cpu74@2649MHz",
+      "AMD EPYC 7R13 Processor cpu75@2649MHz",
+      "AMD EPYC 7R13 Processor cpu76@2649MHz",
+      "AMD EPYC 7R13 Processor cpu77@2649MHz",
+      "AMD EPYC 7R13 Processor cpu78@2516MHz",
+      "AMD EPYC 7R13 Processor cpu79@2516MHz",
+      "AMD EPYC 7R13 Processor cpu80@2516MHz",
+      "AMD EPYC 7R13 Processor cpu81@2649MHz",
+      "AMD EPYC 7R13 Processor cpu82@2649MHz",
+      "AMD EPYC 7R13 Processor cpu83@2649MHz",
+      "AMD EPYC 7R13 Processor cpu84@2516MHz",
+      "AMD EPYC 7R13 Processor cpu85@2516MHz",
+      "AMD EPYC 7R13 Processor cpu86@2649MHz",
+      "AMD EPYC 7R13 Processor cpu87@2649MHz",
+      "AMD EPYC 7R13 Processor cpu88@2649MHz",
+      "AMD EPYC 7R13 Processor cpu89@2649MHz",
+      "AMD EPYC 7R13 Processor cpu90@2649MHz",
+      "AMD EPYC 7R13 Processor cpu91@2649MHz",
+      "AMD EPYC 7R13 Processor cpu92@2649MHz",
+      "AMD EPYC 7R13 Processor cpu93@2649MHz",
+      "AMD EPYC 7R13 Processor cpu94@2649MHz",
+      "AMD EPYC 7R13 Processor cpu95@2649MHz"
+    ],
+    "memory": "1999.99 GB",
+    "os_name": "Debian GNU/Linux",
+    "os_version": "11",
+    "kernel": "5.15.0-1048-aws",
+    "hostname": "ip-26-0-161-153"
+  }
+}

results/meta-llama_Llama-3_1-8B-Instruct_2025-04-02-08-06-23.json

@@ -0,0 +1,951 @@
+{
+  "config": {
+    "max_vus": 128,
+    "duration_secs": 120,
+    "benchmark_kind": "Rate",
+    "warmup_duration_secs": 30,
+    "rates": [
+      0.8,
+      2.4,
+      4.0,
+      5.6,
+      7.2,
+      8.8,
+      10.4,
+      12.0,
+      13.6,
+      15.2,
+      16.8,
+      18.4,
+      20.0,
+      21.6,
+      23.2,
+      24.0
+    ],
+    "num_rates": 10,
+    "prompt_options": null,
+    "decode_options": {
+      "num_tokens": 200,
+      "min_tokens": 200,
+      "max_tokens": 200,
+      "variance": 100
+    },
+    "tokenizer": "meta-llama/Llama-3.1-8B-Instruct",
+    "model_name": "meta-llama/Llama-3.1-8B-Instruct",
+    "profile": null,
+    "meta": {
+      "tp": "1",
+      "version": "d2ed52f531cf8098ca62375248e007022eaadc65",
+      "engine": "TGI"
+    },
+    "run_id": "TGI-2.4.1"
+  },
+  "results": [
+    {
+      "id": "warmup",
+      "executor_type": "ConstantVUs",
+      "config": {
+        "max_vus": 1,
+        "duration_secs": 30,
+        "rate": null
+      },
+      "total_requests": 21,
+      "total_tokens": 3526,
+      "token_throughput_secs": 122.88666224584397,
+      "duration_ms": 28693,
+      "time_to_first_token_ms": {
+        "p50": 46.249,
+        "p60": 46.647,
+        "p70": 47.564,
+        "p80": 48.066,
+        "p90": 48.771,
+        "p95": 84.208,
+        "p99": 86.256,
+        "avg": 49.712
+      },
+      "inter_token_latency_ms": {
+        "p50": 7.881,
+        "p60": 7.884,
+        "p70": 7.887,
+        "p80": 7.912,
+        "p90": 7.984,
+        "p95": 7.986,
+        "p99": 7.999,
+        "avg": 7.821
+      },
+      "failed_requests": 0,
+      "successful_requests": 21,
+      "request_rate": 0.7318831273859113,
+      "total_tokens_sent": 1337,
+      "e2e_latency_ms": {
+        "p50": 1614.476,
+        "p60": 1615.106,
+        "p70": 1615.452,
+        "p80": 1619.496,
+        "p90": 1634.138,
+        "p95": 1637.031,
+        "p99": 1668.807,
+        "avg": 1366.18
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 0.8
+      },
+      "total_requests": 94,
+      "total_tokens": 17800,
+      "token_throughput_secs": 150.94862460675677,
+      "duration_ms": 117920,
+      "time_to_first_token_ms": {
+        "p50": 24.484,
+        "p60": 25.247,
+        "p70": 26.255,
+        "p80": 27.172,
+        "p90": 28.51,
+        "p95": 29.611,
+        "p99": 30.397,
+        "avg": 24.771
+      },
+      "inter_token_latency_ms": {
+        "p50": 8.272,
+        "p60": 8.28,
+        "p70": 8.286,
+        "p80": 8.292,
+        "p90": 8.326,
+        "p95": 8.376,
+        "p99": 8.475,
+        "avg": 8.281
+      },
+      "failed_requests": 0,
+      "successful_requests": 94,
+      "request_rate": 0.7971444220806255,
+      "total_tokens_sent": 11826,
+      "e2e_latency_ms": {
+        "p50": 1670.039,
+        "p60": 1671.341,
+        "p70": 1673.034,
+        "p80": 1675.422,
+        "p90": 1677.624,
+        "p95": 1684.827,
+        "p99": 1704.448,
+        "avg": 1584.628
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 2.4
+      },
+      "total_requests": 284,
+      "total_tokens": 48304,
+      "token_throughput_secs": 406.254834027723,
+      "duration_ms": 118900,
+      "time_to_first_token_ms": {
+        "p50": 25.492,
+        "p60": 26.266,
+        "p70": 27.157,
+        "p80": 28.116,
+        "p90": 29.53,
+        "p95": 30.532,
+        "p99": 46.87,
+        "avg": 26.066
+      },
+      "inter_token_latency_ms": {
+        "p50": 8.977,
+        "p60": 9.004,
+        "p70": 9.027,
+        "p80": 9.052,
+        "p90": 9.082,
+        "p95": 9.114,
+        "p99": 9.179,
+        "avg": 8.716
+      },
+      "failed_requests": 0,
+      "successful_requests": 284,
+      "request_rate": 2.3885469705174174,
+      "total_tokens_sent": 43082,
+      "e2e_latency_ms": {
+        "p50": 1809.698,
+        "p60": 1814.702,
+        "p70": 1820.864,
+        "p80": 1827.06,
+        "p90": 1833.784,
+        "p95": 1838.2,
+        "p99": 1849.697,
+        "avg": 1543.709
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 4.0
+      },
+      "total_requests": 472,
+      "total_tokens": 83257,
+      "token_throughput_secs": 697.279766847058,
+      "duration_ms": 119402,
+      "time_to_first_token_ms": {
+        "p50": 25.684,
+        "p60": 26.779,
+        "p70": 27.553,
+        "p80": 28.486,
+        "p90": 29.463,
+        "p95": 30.455,
+        "p99": 45.113,
+        "avg": 26.146
+      },
+      "inter_token_latency_ms": {
+        "p50": 9.418,
+        "p60": 9.445,
+        "p70": 9.461,
+        "p80": 9.49,
+        "p90": 9.526,
+        "p95": 9.532,
+        "p99": 9.547,
+        "avg": 9.371
+      },
+      "failed_requests": 0,
+      "successful_requests": 472,
+      "request_rate": 3.953013559842552,
+      "total_tokens_sent": 53690,
+      "e2e_latency_ms": {
+        "p50": 1898.831,
+        "p60": 1904.467,
+        "p70": 1907.994,
+        "p80": 1913.596,
+        "p90": 1920.946,
+        "p95": 1923.942,
+        "p99": 1928.04,
+        "avg": 1679.7
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 5.6
+      },
+      "total_requests": 662,
+      "total_tokens": 118380,
+      "token_throughput_secs": 989.0969943504368,
+      "duration_ms": 119684,
+      "time_to_first_token_ms": {
+        "p50": 26.614,
+        "p60": 27.599,
+        "p70": 28.411,
+        "p80": 29.519,
+        "p90": 40.521,
+        "p95": 45.386,
+        "p99": 47.427,
+        "avg": 27.98
+      },
+      "inter_token_latency_ms": {
+        "p50": 9.948,
+        "p60": 9.96,
+        "p70": 9.969,
+        "p80": 9.98,
+        "p90": 9.995,
+        "p95": 10.005,
+        "p99": 10.037,
+        "avg": 9.848
+      },
+      "failed_requests": 0,
+      "successful_requests": 662,
+      "request_rate": 5.531189476769633,
+      "total_tokens_sent": 111696,
+      "e2e_latency_ms": {
+        "p50": 2004.271,
+        "p60": 2007.295,
+        "p70": 2010.406,
+        "p80": 2012.985,
+        "p90": 2015.977,
+        "p95": 2017.766,
+        "p99": 2025.136,
+        "avg": 1791.085
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 7.2
+      },
+      "total_requests": 850,
+      "total_tokens": 150442,
+      "token_throughput_secs": 1255.1886063661461,
+      "duration_ms": 119856,
+      "time_to_first_token_ms": {
+        "p50": 42.393,
+        "p60": 44.487,
+        "p70": 45.999,
+        "p80": 46.482,
+        "p90": 47.894,
+        "p95": 48.207,
+        "p99": 48.37,
+        "avg": 37.807
+      },
+      "inter_token_latency_ms": {
+        "p50": 10.186,
+        "p60": 10.207,
+        "p70": 10.229,
+        "p80": 10.259,
+        "p90": 10.316,
+        "p95": 10.336,
+        "p99": 10.359,
+        "avg": 9.933
+      },
+      "failed_requests": 0,
+      "successful_requests": 850,
+      "request_rate": 7.091838152984034,
+      "total_tokens_sent": 168386,
+      "e2e_latency_ms": {
+        "p50": 2065.737,
+        "p60": 2070.244,
+        "p70": 2074.586,
+        "p80": 2078.847,
+        "p90": 2083.084,
+        "p95": 2085.846,
+        "p99": 2094.979,
+        "avg": 1832.984
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 8.8
+      },
+      "total_requests": 1044,
+      "total_tokens": 189381,
+      "token_throughput_secs": 1581.3245632308367,
+      "duration_ms": 119760,
+      "time_to_first_token_ms": {
+        "p50": 42.904,
+        "p60": 44.702,
+        "p70": 45.143,
+        "p80": 45.649,
+        "p90": 47.202,
+        "p95": 47.429,
+        "p99": 47.647,
+        "avg": 38.138
+      },
+      "inter_token_latency_ms": {
+        "p50": 10.601,
+        "p60": 10.617,
+        "p70": 10.64,
+        "p80": 10.669,
+        "p90": 10.709,
+        "p95": 10.745,
+        "p99": 10.837,
+        "avg": 10.457
+      },
+      "failed_requests": 0,
+      "successful_requests": 1044,
+      "request_rate": 8.717362586600522,
+      "total_tokens_sent": 164348,
+      "e2e_latency_ms": {
+        "p50": 2149.601,
+        "p60": 2152.628,
+        "p70": 2155.129,
+        "p80": 2157.878,
+        "p90": 2161.992,
+        "p95": 2166.405,
+        "p99": 2185.247,
+        "avg": 1952.745
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 10.4
+      },
+      "total_requests": 1227,
+      "total_tokens": 215756,
+      "token_throughput_secs": 1801.727543102545,
+      "duration_ms": 119749,
+      "time_to_first_token_ms": {
+        "p50": 28.996,
+        "p60": 29.94,
+        "p70": 31.219,
+        "p80": 33.038,
+        "p90": 43.255,
+        "p95": 45.051,
+        "p99": 47.322,
+        "avg": 30.686
+      },
+      "inter_token_latency_ms": {
+        "p50": 11.152,
+        "p60": 11.172,
+        "p70": 11.192,
+        "p80": 11.214,
+        "p90": 11.247,
+        "p95": 11.271,
+        "p99": 11.413,
+        "avg": 11.041
+      },
+      "failed_requests": 0,
+      "successful_requests": 1227,
+      "request_rate": 10.246388028081828,
+      "total_tokens_sent": 216589,
+      "e2e_latency_ms": {
+        "p50": 2246.941,
+        "p60": 2250.773,
+        "p70": 2254.922,
+        "p80": 2260.186,
+        "p90": 2267.355,
+        "p95": 2272.509,
+        "p99": 2299.613,
+        "avg": 1980.137
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 12.0
+      },
+      "total_requests": 1413,
+      "total_tokens": 257715,
+      "token_throughput_secs": 2152.3457887331883,
+      "duration_ms": 119736,
+      "time_to_first_token_ms": {
+        "p50": 32.129,
+        "p60": 41.129,
+        "p70": 42.905,
+        "p80": 43.618,
+        "p90": 45.582,
+        "p95": 47.077,
+        "p99": 47.609,
+        "avg": 34.989
+      },
+      "inter_token_latency_ms": {
+        "p50": 11.524,
+        "p60": 11.571,
+        "p70": 11.619,
+        "p80": 11.681,
+        "p90": 11.737,
+        "p95": 11.759,
+        "p99": 11.825,
+        "avg": 11.473
+      },
+      "failed_requests": 0,
+      "successful_requests": 1413,
+      "request_rate": 11.800883144093262,
+      "total_tokens_sent": 224022,
+      "e2e_latency_ms": {
+        "p50": 2327.912,
+        "p60": 2333.025,
+        "p70": 2339.512,
+        "p80": 2355.095,
+        "p90": 2364.745,
+        "p95": 2369.454,
+        "p99": 2381.415,
+        "avg": 2131.083
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 13.6
+      },
+      "total_requests": 1586,
+      "total_tokens": 273066,
+      "token_throughput_secs": 2286.726108001893,
+      "duration_ms": 119413,
+      "time_to_first_token_ms": {
+        "p50": 41.765,
+        "p60": 43.496,
+        "p70": 45.121,
+        "p80": 45.647,
+        "p90": 47.119,
+        "p95": 47.381,
+        "p99": 47.644,
+        "avg": 39.252
+      },
+      "inter_token_latency_ms": {
+        "p50": 21.018,
+        "p60": 21.058,
+        "p70": 21.094,
+        "p80": 21.136,
+        "p90": 21.204,
+        "p95": 21.361,
+        "p99": 21.474,
+        "avg": 20.088
+      },
+      "failed_requests": 0,
+      "successful_requests": 1586,
+      "request_rate": 13.281578839148787,
+      "total_tokens_sent": 263160,
+      "e2e_latency_ms": {
+        "p50": 4215.628,
+        "p60": 4224.579,
+        "p70": 4233.564,
+        "p80": 4241.334,
+        "p90": 4253.157,
+        "p95": 4281.605,
+        "p99": 4307.594,
+        "avg": 3499.01
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 15.2
+      },
+      "total_requests": 1767,
+      "total_tokens": 318782,
+      "token_throughput_secs": 2659.660245067388,
+      "duration_ms": 119858,
+      "time_to_first_token_ms": {
+        "p50": 41.941,
+        "p60": 42.561,
+        "p70": 43.982,
+        "p80": 44.599,
+        "p90": 46.402,
+        "p95": 47.889,
+        "p99": 48.314,
+        "avg": 38.969
+      },
+      "inter_token_latency_ms": {
+        "p50": 21.711,
+        "p60": 21.755,
+        "p70": 21.812,
+        "p80": 21.867,
+        "p90": 21.933,
+        "p95": 21.976,
+        "p99": 22.027,
+        "avg": 21.316
+      },
+      "failed_requests": 0,
+      "successful_requests": 1767,
+      "request_rate": 14.742424770012343,
+      "total_tokens_sent": 299995,
+      "e2e_latency_ms": {
+        "p50": 4354.219,
+        "p60": 4361.816,
+        "p70": 4377.337,
+        "p80": 4389.415,
+        "p90": 4402.615,
+        "p95": 4409.06,
+        "p99": 4416.879,
+        "avg": 3900.022
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 16.8
+      },
+      "total_requests": 1952,
+      "total_tokens": 349899,
+      "token_throughput_secs": 2924.5946715959903,
+      "duration_ms": 119640,
+      "time_to_first_token_ms": {
+        "p50": 42.018,
+        "p60": 42.508,
+        "p70": 43.793,
+        "p80": 44.153,
+        "p90": 44.83,
+        "p95": 46.44,
+        "p99": 48.232,
+        "avg": 38.955
+      },
+      "inter_token_latency_ms": {
+        "p50": 22.202,
+        "p60": 22.257,
+        "p70": 22.348,
+        "p80": 22.464,
+        "p90": 22.555,
+        "p95": 22.618,
+        "p99": 22.716,
+        "avg": 21.964
+      },
+      "failed_requests": 0,
+      "successful_requests": 1952,
+      "request_rate": 16.315590495987053,
+      "total_tokens_sent": 308143,
+      "e2e_latency_ms": {
+        "p50": 4448.481,
+        "p60": 4462.093,
+        "p70": 4478.598,
+        "p80": 4507.043,
+        "p90": 4525.206,
+        "p95": 4538.11,
+        "p99": 4560.079,
+        "avg": 3971.144
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 18.4
+      },
+      "total_requests": 2135,
+      "total_tokens": 377085,
+      "token_throughput_secs": 3157.5918029420154,
+      "duration_ms": 119421,
+      "time_to_first_token_ms": {
+        "p50": 42.037,
+        "p60": 42.627,
+        "p70": 43.838,
+        "p80": 44.183,
+        "p90": 45.567,
+        "p95": 46.603,
+        "p99": 48.266,
+        "avg": 39.44
+      },
+      "inter_token_latency_ms": {
+        "p50": 22.519,
+        "p60": 22.586,
+        "p70": 22.678,
+        "p80": 22.785,
+        "p90": 22.851,
+        "p95": 22.888,
+        "p99": 22.962,
+        "avg": 22.143
+      },
+      "failed_requests": 0,
+      "successful_requests": 2135,
+      "request_rate": 17.87782197457126,
+      "total_tokens_sent": 378767,
+      "e2e_latency_ms": {
+        "p50": 4511.199,
+        "p60": 4525.581,
+        "p70": 4542.862,
+        "p80": 4569.463,
+        "p90": 4584.328,
+        "p95": 4592.49,
+        "p99": 4605.219,
+        "avg": 3978.179
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 20.0
+      },
+      "total_requests": 2314,
+      "total_tokens": 410851,
+      "token_throughput_secs": 3431.8999694521895,
+      "duration_ms": 119715,
+      "time_to_first_token_ms": {
+        "p50": 42.054,
+        "p60": 42.898,
+        "p70": 43.131,
+        "p80": 43.857,
+        "p90": 45.58,
+        "p95": 47.057,
+        "p99": 53.486,
+        "avg": 40.178
+      },
+      "inter_token_latency_ms": {
+        "p50": 23.178,
+        "p60": 23.241,
+        "p70": 23.306,
+        "p80": 23.362,
+        "p90": 23.489,
+        "p95": 23.649,
+        "p99": 24.696,
+        "avg": 22.962
+      },
+      "failed_requests": 0,
+      "successful_requests": 2314,
+      "request_rate": 19.329188755320946,
+      "total_tokens_sent": 428632,
+      "e2e_latency_ms": {
+        "p50": 4641.862,
+        "p60": 4658.169,
+        "p70": 4671.517,
+        "p80": 4685.088,
+        "p90": 4705.846,
+        "p95": 4725.362,
+        "p99": 4949.917,
+        "avg": 4118.614
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 21.6
+      },
+      "total_requests": 2496,
+      "total_tokens": 425184,
+      "token_throughput_secs": 3546.9270370066156,
+      "duration_ms": 119873,
+      "time_to_first_token_ms": {
+        "p50": 47.532,
+        "p60": 51.096,
+        "p70": 55.311,
+        "p80": 59.955,
+        "p90": 64.818,
+        "p95": 68.566,
+        "p99": 73.174,
+        "avg": 49.832
+      },
+      "inter_token_latency_ms": {
+        "p50": 23.615,
+        "p60": 23.819,
+        "p70": 23.937,
+        "p80": 24.142,
+        "p90": 24.335,
+        "p95": 24.424,
+        "p99": 24.675,
+        "avg": 23.117
+      },
+      "failed_requests": 0,
+      "successful_requests": 2496,
+      "request_rate": 20.821879196697225,
+      "total_tokens_sent": 449445,
+      "e2e_latency_ms": {
+        "p50": 4705.977,
+        "p60": 4760.663,
+        "p70": 4802.56,
+        "p80": 4835.503,
+        "p90": 4887.331,
+        "p95": 4911.378,
+        "p99": 4950.871,
+        "avg": 4041.088
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 23.2
+      },
+      "total_requests": 2693,
+      "total_tokens": 477598,
+      "token_throughput_secs": 3987.677624302409,
+      "duration_ms": 119768,
+      "time_to_first_token_ms": {
+        "p50": 48.917,
+        "p60": 53.875,
+        "p70": 58.427,
+        "p80": 63.212,
+        "p90": 68.231,
+        "p95": 71.137,
+        "p99": 75.227,
+        "avg": 51.515
+      },
+      "inter_token_latency_ms": {
+        "p50": 24.172,
+        "p60": 24.458,
+        "p70": 24.565,
+        "p80": 24.718,
+        "p90": 24.831,
+        "p95": 25.263,
+        "p99": 25.541,
+        "avg": 23.951
+      },
+      "failed_requests": 0,
+      "successful_requests": 2693,
+      "request_rate": 22.485051952157228,
+      "total_tokens_sent": 418148,
+      "e2e_latency_ms": {
+        "p50": 4820.55,
+        "p60": 4895.607,
+        "p70": 4932.562,
+        "p80": 4963.156,
+        "p90": 4990.976,
+        "p95": 5047.438,
+        "p99": 5133.803,
+        "avg": 4302.865
+      }
+    },
+    {
+      "id": "[email protected]/s",
+      "executor_type": "ConstantArrivalRate",
+      "config": {
+        "max_vus": 128,
+        "duration_secs": 120,
+        "rate": 24.0
+      },
+      "total_requests": 2770,
+      "total_tokens": 491476,
+      "token_throughput_secs": 4122.591533026405,
+      "duration_ms": 119215,
+      "time_to_first_token_ms": {
+        "p50": 49.685,
+        "p60": 54.302,
+        "p70": 59.464,
+        "p80": 64.151,
+        "p90": 69.885,
+        "p95": 72.497,
+        "p99": 75.82,
+        "avg": 51.673
+      },
+      "inter_token_latency_ms": {
+        "p50": 24.572,
+        "p60": 24.655,
+        "p70": 24.758,
+        "p80": 24.89,
+        "p90": 25.177,
+        "p95": 25.712,
+        "p99": 25.93,
+        "avg": 24.429
+      },
+      "failed_requests": 0,
+      "successful_requests": 2770,
+      "request_rate": 23.235272010196105,
+      "total_tokens_sent": 417041,
+      "e2e_latency_ms": {
+        "p50": 4917.332,
+        "p60": 4944.149,
+        "p70": 4968.491,
+        "p80": 4997.781,
+        "p90": 5053.09,
+        "p95": 5149.163,
+        "p99": 5204.539,
+        "avg": 4376.615
+      }
+    }
+  ],
+  "start_time": "2025-04-02T07:33:13.875090549+00:00",
+  "end_time": "2025-04-02T08:06:23.692065899+00:00",
+  "system": {
+    "cpu": [
+      "AMD EPYC 7R13 Processor cpu0@2363MHz",
+      "AMD EPYC 7R13 Processor cpu1@2649MHz",
+      "AMD EPYC 7R13 Processor cpu2@2673MHz",
+      "AMD EPYC 7R13 Processor cpu3@2673MHz",
+      "AMD EPYC 7R13 Processor cpu4@2649MHz",
+      "AMD EPYC 7R13 Processor cpu5@2673MHz",
+      "AMD EPYC 7R13 Processor cpu6@2673MHz",
+      "AMD EPYC 7R13 Processor cpu7@2649MHz",
+      "AMD EPYC 7R13 Processor cpu8@2649MHz",
+      "AMD EPYC 7R13 Processor cpu9@2673MHz",
+      "AMD EPYC 7R13 Processor cpu10@2673MHz",
+      "AMD EPYC 7R13 Processor cpu11@2673MHz",
+      "AMD EPYC 7R13 Processor cpu12@2649MHz",
+      "AMD EPYC 7R13 Processor cpu13@2673MHz",
+      "AMD EPYC 7R13 Processor cpu14@2673MHz",
+      "AMD EPYC 7R13 Processor cpu15@2673MHz",
+      "AMD EPYC 7R13 Processor cpu16@2673MHz",
+      "AMD EPYC 7R13 Processor cpu17@2673MHz",
+      "AMD EPYC 7R13 Processor cpu18@2673MHz",
+      "AMD EPYC 7R13 Processor cpu19@2649MHz",
+      "AMD EPYC 7R13 Processor cpu20@2649MHz",
+      "AMD EPYC 7R13 Processor cpu21@2649MHz",
+      "AMD EPYC 7R13 Processor cpu22@2649MHz",
+      "AMD EPYC 7R13 Processor cpu23@2673MHz",
+      "AMD EPYC 7R13 Processor cpu24@2649MHz",
+      "AMD EPYC 7R13 Processor cpu25@2649MHz",
+      "AMD EPYC 7R13 Processor cpu26@2649MHz",
+      "AMD EPYC 7R13 Processor cpu27@2649MHz",
+      "AMD EPYC 7R13 Processor cpu28@2649MHz",
+      "AMD EPYC 7R13 Processor cpu29@2649MHz",
+      "AMD EPYC 7R13 Processor cpu30@2673MHz",
+      "AMD EPYC 7R13 Processor cpu31@3391MHz",
+      "AMD EPYC 7R13 Processor cpu32@2649MHz",
+      "AMD EPYC 7R13 Processor cpu33@2649MHz",
+      "AMD EPYC 7R13 Processor cpu34@2649MHz",
+      "AMD EPYC 7R13 Processor cpu35@2649MHz",
+      "AMD EPYC 7R13 Processor cpu36@2649MHz",
+      "AMD EPYC 7R13 Processor cpu37@2649MHz",
+      "AMD EPYC 7R13 Processor cpu38@2649MHz",
+      "AMD EPYC 7R13 Processor cpu39@2673MHz",
+      "AMD EPYC 7R13 Processor cpu40@2649MHz",
+      "AMD EPYC 7R13 Processor cpu41@2673MHz",
+      "AMD EPYC 7R13 Processor cpu42@2673MHz",
+      "AMD EPYC 7R13 Processor cpu43@2673MHz",
+      "AMD EPYC 7R13 Processor cpu44@2673MHz",
+      "AMD EPYC 7R13 Processor cpu45@2673MHz",
+      "AMD EPYC 7R13 Processor cpu46@2673MHz",
+      "AMD EPYC 7R13 Processor cpu47@2649MHz",
+      "AMD EPYC 7R13 Processor cpu48@2649MHz",
+      "AMD EPYC 7R13 Processor cpu49@2649MHz",
+      "AMD EPYC 7R13 Processor cpu50@2649MHz",
+      "AMD EPYC 7R13 Processor cpu51@2649MHz",
+      "AMD EPYC 7R13 Processor cpu52@2649MHz",
+      "AMD EPYC 7R13 Processor cpu53@2649MHz",
+      "AMD EPYC 7R13 Processor cpu54@2673MHz",
+      "AMD EPYC 7R13 Processor cpu55@2673MHz",
+      "AMD EPYC 7R13 Processor cpu56@2673MHz",
+      "AMD EPYC 7R13 Processor cpu57@2649MHz",
+      "AMD EPYC 7R13 Processor cpu58@2649MHz",
+      "AMD EPYC 7R13 Processor cpu59@2649MHz",
+      "AMD EPYC 7R13 Processor cpu60@2673MHz",
+      "AMD EPYC 7R13 Processor cpu61@2649MHz",
+      "AMD EPYC 7R13 Processor cpu62@2649MHz",
+      "AMD EPYC 7R13 Processor cpu63@2649MHz",
+      "AMD EPYC 7R13 Processor cpu64@2649MHz",
+      "AMD EPYC 7R13 Processor cpu65@2649MHz",
+      "AMD EPYC 7R13 Processor cpu66@2649MHz",
+      "AMD EPYC 7R13 Processor cpu67@2649MHz",
+      "AMD EPYC 7R13 Processor cpu68@2649MHz",
+      "AMD EPYC 7R13 Processor cpu69@2649MHz",
+      "AMD EPYC 7R13 Processor cpu70@2649MHz",
+      "AMD EPYC 7R13 Processor cpu71@2649MHz",
+      "AMD EPYC 7R13 Processor cpu72@2649MHz",
+      "AMD EPYC 7R13 Processor cpu73@2649MHz",
+      "AMD EPYC 7R13 Processor cpu74@2649MHz",
+      "AMD EPYC 7R13 Processor cpu75@2649MHz",
+      "AMD EPYC 7R13 Processor cpu76@2649MHz",
+      "AMD EPYC 7R13 Processor cpu77@2649MHz",
+      "AMD EPYC 7R13 Processor cpu78@2649MHz",
+      "AMD EPYC 7R13 Processor cpu79@2649MHz",
+      "AMD EPYC 7R13 Processor cpu80@2649MHz",
+      "AMD EPYC 7R13 Processor cpu81@2649MHz",
+      "AMD EPYC 7R13 Processor cpu82@2649MHz",
+      "AMD EPYC 7R13 Processor cpu83@2649MHz",
+      "AMD EPYC 7R13 Processor cpu84@2673MHz",
+      "AMD EPYC 7R13 Processor cpu85@2673MHz",
+      "AMD EPYC 7R13 Processor cpu86@3111MHz",
+      "AMD EPYC 7R13 Processor cpu87@2649MHz",
+      "AMD EPYC 7R13 Processor cpu88@2649MHz",
+      "AMD EPYC 7R13 Processor cpu89@2649MHz",
+      "AMD EPYC 7R13 Processor cpu90@3111MHz",
+      "AMD EPYC 7R13 Processor cpu91@2649MHz",
+      "AMD EPYC 7R13 Processor cpu92@2649MHz",
+      "AMD EPYC 7R13 Processor cpu93@2649MHz",
+      "AMD EPYC 7R13 Processor cpu94@2649MHz",
+      "AMD EPYC 7R13 Processor cpu95@2649MHz"
+    ],
+    "memory": "1999.99 GB",
+    "os_name": "Debian GNU/Linux",
+    "os_version": "11",
+    "kernel": "5.15.0-1048-aws",
+    "hostname": "ip-26-0-161-123"
+  }
+}