{
  "schemaVersion": 1,
  "id": "provider-latency/2026-08",
  "series": "provider-latency",
  "title": "Time to first token by provider, August 2026",
  "topic": "latency",
  "question": "Which LLM provider returns the first token fastest?",
  "questions": [
    "Which LLM API has the lowest latency?",
    "What is the median time to first token for LLM providers?",
    "Which provider streams the most tokens per second?",
    "How long does a full streamed LLM response take?"
  ],
  "description": "Median and p90 time to first token, median total response time, and median output tokens per second, for streamed successful requests per provider.",
  "period": "2026-08",
  "period_start": "2026-08-01",
  "period_end": "2026-08-31",
  "updated": "2026-11-11",
  "is_latest": false,
  "headline": "Sference had the fastest median time to first token among major providers in August 2026 at 55 ms. The median major provider took 1.33 s.",
  "key_findings": [
    "Fastest median TTFT: Sference at 55 ms, then Alibaba at 200 ms.",
    "Highest median output speed: Vertex (Gemini) at 192 tokens per second.",
    "Fastest median full streamed response: Alibaba at 1.07 s.",
    "Novita moved the most since July 2026: median TTFT went from 3.71 s to 1.40 s (-2.31 s)."
  ],
  "columns": [
    {
      "key": "provider",
      "label": "Provider",
      "unit": "text"
    },
    {
      "key": "ttft_p50_ms",
      "label": "Median TTFT",
      "unit": "milliseconds"
    },
    {
      "key": "ttft_p90_ms",
      "label": "p90 TTFT",
      "unit": "milliseconds"
    },
    {
      "key": "stream_total_p50_ms",
      "label": "Median total",
      "unit": "milliseconds",
      "description": "Time until the last streamed token arrived."
    },
    {
      "key": "tps_p50",
      "label": "Median tokens/s",
      "unit": "ratio",
      "description": "Visible output tokens per second after the first token."
    },
    {
      "key": "stream_share",
      "label": "Streamed share",
      "unit": "fraction"
    },
    {
      "key": "sample_band",
      "label": "Organizations (band)",
      "unit": "text",
      "description": "Number of distinct organizations behind the row, in bands."
    }
  ],
  "rows": [
    {
      "provider": "sference",
      "ttft_p50_ms": 55,
      "ttft_p90_ms": 142,
      "stream_total_p50_ms": 1796,
      "tps_p50": 152.7,
      "stream_share": 0.7424,
      "sample_band": "500+",
      "provider_label": "Sference"
    },
    {
      "provider": "alibaba",
      "ttft_p50_ms": 200,
      "ttft_p90_ms": 4537,
      "stream_total_p50_ms": 1071,
      "tps_p50": 110.2,
      "stream_share": 0.3376,
      "sample_band": "100-499",
      "provider_label": "Alibaba"
    },
    {
      "provider": "nebius",
      "ttft_p50_ms": 542,
      "ttft_p90_ms": 3197,
      "stream_total_p50_ms": 2222,
      "tps_p50": 130.7,
      "stream_share": 0.2346,
      "sample_band": "500+",
      "provider_label": "Nebius"
    },
    {
      "provider": "openai",
      "ttft_p50_ms": 547,
      "ttft_p90_ms": 4311,
      "stream_total_p50_ms": 3628,
      "tps_p50": 69.9,
      "stream_share": 0.1371,
      "sample_band": "500+",
      "provider_label": "OpenAI"
    },
    {
      "provider": "azure",
      "ttft_p50_ms": 608,
      "ttft_p90_ms": 1637,
      "stream_total_p50_ms": 1984,
      "tps_p50": 52.8,
      "stream_share": 0.5469,
      "sample_band": "500+",
      "provider_label": "Azure"
    },
    {
      "provider": "deepseek",
      "ttft_p50_ms": 857,
      "ttft_p90_ms": 1452,
      "stream_total_p50_ms": 9211,
      "tps_p50": 38.7,
      "stream_share": 0.5046,
      "sample_band": "500+",
      "provider_label": "DeepSeek"
    },
    {
      "provider": "deepinfra",
      "ttft_p50_ms": 930,
      "ttft_p90_ms": 3830,
      "stream_total_p50_ms": 3186,
      "tps_p50": 73.6,
      "stream_share": 0.2753,
      "sample_band": "500+",
      "provider_label": "DeepInfra"
    },
    {
      "provider": "fireworks",
      "ttft_p50_ms": 1190,
      "ttft_p90_ms": 3708,
      "stream_total_p50_ms": 4668,
      "tps_p50": 88.3,
      "stream_share": 0.3453,
      "sample_band": "500+",
      "provider_label": "Fireworks"
    },
    {
      "provider": "vertex_claude",
      "ttft_p50_ms": 1330,
      "ttft_p90_ms": 3079,
      "stream_total_p50_ms": 5796,
      "tps_p50": 77.7,
      "stream_share": 0.4449,
      "sample_band": "500+",
      "provider_label": "Vertex (Claude)"
    },
    {
      "provider": "tensorx",
      "ttft_p50_ms": 1355,
      "ttft_p90_ms": 5310,
      "stream_total_p50_ms": 4965,
      "tps_p50": 80.2,
      "stream_share": 0.2896,
      "sample_band": "500+",
      "provider_label": "TensorX"
    },
    {
      "provider": "novita",
      "ttft_p50_ms": 1396,
      "ttft_p90_ms": 2947,
      "stream_total_p50_ms": 25286,
      "tps_p50": 13.1,
      "stream_share": 0.4245,
      "sample_band": "500+",
      "provider_label": "Novita"
    },
    {
      "provider": "xai",
      "ttft_p50_ms": 1470,
      "ttft_p90_ms": 3397,
      "stream_total_p50_ms": 9673,
      "tps_p50": 44.6,
      "stream_share": 0.0993,
      "sample_band": "100-499",
      "provider_label": "xAI"
    },
    {
      "provider": "mistral",
      "ttft_p50_ms": 1548,
      "ttft_p90_ms": 10436,
      "stream_total_p50_ms": 3011,
      "tps_p50": 117.8,
      "stream_share": 0.2015,
      "sample_band": "500+",
      "provider_label": "Mistral"
    },
    {
      "provider": "anthropic",
      "ttft_p50_ms": 1596,
      "ttft_p90_ms": 3372,
      "stream_total_p50_ms": 7268,
      "tps_p50": 59.6,
      "stream_share": 0.2435,
      "sample_band": "500+",
      "provider_label": "Anthropic"
    },
    {
      "provider": "bedrock",
      "ttft_p50_ms": 1735,
      "ttft_p90_ms": 3299,
      "stream_total_p50_ms": 6078,
      "tps_p50": 80.4,
      "stream_share": 0.3002,
      "sample_band": "500+",
      "provider_label": "Bedrock"
    },
    {
      "provider": "vertex_gemini",
      "ttft_p50_ms": 1935,
      "ttft_p90_ms": 9173,
      "stream_total_p50_ms": 3909,
      "tps_p50": 192.4,
      "stream_share": 0.0508,
      "sample_band": "500+",
      "provider_label": "Vertex (Gemini)"
    },
    {
      "provider": "nvidia",
      "ttft_p50_ms": 2046,
      "ttft_p90_ms": 14026,
      "stream_total_p50_ms": 8478,
      "tps_p50": 60.7,
      "stream_share": 0.6686,
      "sample_band": "500+",
      "provider_label": "NVIDIA"
    }
  ],
  "totals": {
    "withheld_request_share": 0.103
  },
  "floors": {
    "min_requests": 20,
    "min_orgs": 5,
    "max_top_org_share": 0.8
  },
  "methodology": [
    "Time to first token and tokens per second are measured on streamed, successful requests only, with the same formulas /rankings uses. Tokens per second counts visible output tokens after the first token, with a 250 ms floor on decode time and at least 20 output tokens.",
    "A provider needs at least 500 streamed successes in the month for latency to be published. Only providers with at least 0.5% of published traffic are shown.",
    "A row is published only when it has at least 20 requests, at least 5 distinct organizations, and no single organization above 80% of its traffic. These are the same floors /rankings uses. Rows below the floors are withheld, not estimated.",
    "Only shares, rates, medians and organization bands are published. Absolute request, token and spend totals stay private."
  ],
  "definitions": [
    {
      "term": "TTFT",
      "text": "Time from the gateway sending the request to the first streamed token from the provider."
    },
    {
      "term": "p90",
      "text": "90% of requests were faster than this."
    },
    {
      "term": "Total",
      "text": "Time from the gateway sending the request to the last streamed token."
    }
  ],
  "caveats": [
    "Latency depends on the model mix. A provider serving mostly small models will look faster than one serving large reasoning models.",
    "Reasoning models think before the first visible token, which raises TTFT."
  ],
  "source": "Requesty production gateway",
  "license": {
    "name": "CC BY 4.0",
    "url": "https://creativecommons.org/licenses/by/4.0/"
  },
  "citation": {
    "apa": "Requesty (2026). Time to first token by provider, August 2026. Requesty Data. https://www.requesty.ai/data/provider-latency/2026-08",
    "bibtex": "@misc{requesty_provider_latency_2026_08,\n  author       = {{Requesty}},\n  title        = {Time to first token by provider, August 2026},\n  year         = {2026},\n  howpublished = {\\url{https://www.requesty.ai/data/provider-latency/2026-08}},\n  note         = {Requesty Data}\n}"
  },
  "permalink": "https://www.requesty.ai/data/provider-latency/2026-08",
  "latest_url": "https://www.requesty.ai/data/provider-latency",
  "previous": "https://www.requesty.ai/data/provider-latency/2026-07",
  "next": "https://www.requesty.ai/data/provider-latency/2026-09",
  "periods": [
    {
      "period": "2026-05",
      "url": "https://www.requesty.ai/data/provider-latency/2026-05",
      "json": "https://www.requesty.ai/data/provider-latency/2026-05/data.json"
    },
    {
      "period": "2026-06",
      "url": "https://www.requesty.ai/data/provider-latency/2026-06",
      "json": "https://www.requesty.ai/data/provider-latency/2026-06/data.json"
    },
    {
      "period": "2026-07",
      "url": "https://www.requesty.ai/data/provider-latency/2026-07",
      "json": "https://www.requesty.ai/data/provider-latency/2026-07/data.json"
    },
    {
      "period": "2026-08",
      "url": "https://www.requesty.ai/data/provider-latency/2026-08",
      "json": "https://www.requesty.ai/data/provider-latency/2026-08/data.json"
    },
    {
      "period": "2026-09",
      "url": "https://www.requesty.ai/data/provider-latency/2026-09",
      "json": "https://www.requesty.ai/data/provider-latency/2026-09/data.json"
    },
    {
      "period": "2026-10",
      "url": "https://www.requesty.ai/data/provider-latency/2026-10",
      "json": "https://www.requesty.ai/data/provider-latency/2026-10/data.json"
    }
  ],
  "downloads": {
    "json": "https://www.requesty.ai/data/provider-latency/2026-08/data.json",
    "csv": "https://www.requesty.ai/data/provider-latency/2026-08/data.csv",
    "markdown": "https://www.requesty.ai/data/provider-latency/2026-08/data.md"
  },
  "related": [
    "https://www.requesty.ai/data/provider-error-rates",
    "https://www.requesty.ai/data/provider-cache-hit-rate"
  ],
  "rankings": "https://www.requesty.ai/rankings"
}