{
  "title": "GlobalVoice-Bench",
  "subtitle": "Measuring linguistic equity gaps in multilingual speech AI",
  "url": "https://www.datoric.com/research/globalvoice-bench",
  "version": "2026-07-21 report edition",
  "publishedOn": "2026-05-24",
  "lastReviewed": "2026-07-21",
  "authors": [
    {
      "name": "Jeffrey Lin",
      "affiliation": "Datoric Research"
    },
    {
      "name": "Nikhil Reddy",
      "affiliation": "Datoric Research"
    }
  ],
  "headlineStat": {
    "value": "3 of 5",
    "label": "dedicated ASR providers cannot serve the low-resource tier reliably: Deepgram returns HTTP 400; AssemblyAI Universal-2 returns WER above 0.85"
  },
  "datasetSummary": "Four axes: 800 FLEURS samples across 20 languages in 3 tiers (200 evaluated for per-tier transcription), 150 ASCEND Mandarin–English code-switch samples, 745 Common Voice 17 accent samples across 7 languages × 22 accent cells, and 800 culturally-grounded recordings (40 per language × 20 languages). 12 frontier systems. Bootstrap 95% CI.",
  "changelog": [
    {
      "date": "2026-07-21",
      "summary": "Corrected cultural-QA coverage to identify ElevenLabs Scribe and GPT-4o-mini Audio as the two systems with 100% effective coverage."
    },
    {
      "date": "2026-07-21",
      "summary": "Added named authors, versioned report resources, downloadable tables, structured citations, and synchronized HTML and PDF editions."
    },
    {
      "date": "2026-07-17",
      "summary": "Aligned public asset descriptions and added primary-source links."
    }
  ],
  "tables": [
    {
      "caption": "Table 1. Error rate by resource tier. WER for whitespace-tokenized languages, CER for CJK. Lower is better. \"unsup.\" = unsupported (HTTP 400 or empty transcript). Claude text reasoners (control) are omitted; all three score under 0.02 on every tier because they receive the reference transcript.",
      "headers": [
        "Model",
        "High",
        "Mid",
        "Low"
      ],
      "rows": [
        [
          "Whisper v3",
          "0.214",
          "0.232",
          "0.529"
        ],
        [
          "Deepgram Nova-3",
          "0.210",
          "0.228",
          "unsup."
        ],
        [
          "Deepgram Nova-2",
          "0.213",
          "0.246",
          "unsup."
        ],
        [
          "AssemblyAI Univ.-2",
          "0.201",
          "0.217",
          "0.856"
        ],
        [
          "ElevenLabs Scribe",
          "0.230",
          "0.217",
          "0.409"
        ],
        [
          "GPT-4o Audio",
          "0.742",
          "0.456",
          "2.829"
        ],
        [
          "GPT-4o-mini Audio",
          "0.838",
          "0.574",
          "2.377"
        ],
        [
          "Gemini 2.5 Pro",
          "0.257",
          "0.198",
          "0.413"
        ],
        [
          "Gemini 2.5 Flash",
          "0.239",
          "0.208",
          "0.510"
        ]
      ]
    },
    {
      "caption": "Table 2. ASCEND Mandarin–English code-switching. \"Refuse %\" is the fraction of samples returning an empty transcript; \"Boundary WER\" is computed on ±50-character windows around the 356 annotated switch points, averaged over non-empty windows.",
      "headers": [
        "Model",
        "Refuse %",
        "Boundary WER ↓",
        "LID acc ↑"
      ],
      "rows": [
        [
          "ElevenLabs Scribe",
          "0.0",
          "0.932",
          "0.343"
        ],
        [
          "Whisper v3",
          "0.7",
          "0.948",
          "0.164"
        ],
        [
          "AssemblyAI Univ.-2",
          "2.0",
          "0.989",
          "0.145"
        ],
        [
          "Deepgram Nova-3",
          "6.7",
          "2.065",
          "0.000"
        ],
        [
          "Deepgram Nova-2",
          "49.3",
          "1.501",
          "0.000"
        ],
        [
          "GPT-4o Audio",
          "4.7",
          "1.064",
          "0.135"
        ],
        [
          "GPT-4o-mini Audio",
          "0.0",
          "1.649",
          "0.067"
        ],
        [
          "Gemini 2.5 Pro",
          "2.7",
          "0.821",
          "0.206"
        ],
        [
          "Gemini 2.5 Flash",
          "28.7",
          "0.781",
          "0.288"
        ]
      ]
    }
  ],
  "figures": [
    {
      "variant": "gap-bars",
      "caption": "Figure 1. WER by resource tier across the nine audio-facing systems (lower is better). Deepgram Nova-3 and Nova-2 are unsupported on the low-resource tier; AssemblyAI Universal-2 returns a low-resource WER above 0.85 and is not practically usable. GPT-4o Audio's bars exceed 1.0 because the model hallucinates more text than the reference contains.",
      "data": {
        "tiers": [
          "High resource",
          "Mid resource",
          "Low resource"
        ],
        "models": [
          {
            "name": "Whisper v3",
            "scores": [
              0.214,
              0.232,
              0.529
            ]
          },
          {
            "name": "Deepgram Nova-3",
            "scores": [
              0.21,
              0.228,
              null
            ]
          },
          {
            "name": "Deepgram Nova-2",
            "scores": [
              0.213,
              0.246,
              null
            ]
          },
          {
            "name": "AssemblyAI Univ.-2",
            "scores": [
              0.201,
              0.217,
              0.856
            ]
          },
          {
            "name": "ElevenLabs Scribe",
            "scores": [
              0.23,
              0.217,
              0.409
            ]
          },
          {
            "name": "GPT-4o Audio",
            "scores": [
              0.742,
              0.456,
              2.829
            ]
          },
          {
            "name": "GPT-4o-mini Audio",
            "scores": [
              0.838,
              0.574,
              2.377
            ]
          },
          {
            "name": "Gemini 2.5 Pro",
            "scores": [
              0.257,
              0.198,
              0.413
            ]
          },
          {
            "name": "Gemini 2.5 Flash",
            "scores": [
              0.239,
              0.208,
              0.51
            ]
          }
        ]
      }
    },
    {
      "variant": "bar-vertical",
      "caption": "Figure 2. Mean across-accent WER standard deviation across seven languages (lower = more accent-robust). Gemini 2.5 Flash and Deepgram Nova-2 are statistically tied at the top; AssemblyAI Universal-2's higher figure is driven by its Arabic cell.",
      "data": {
        "models": [
          "Gemini 2.5 Flash",
          "Deepgram Nova-2",
          "Whisper v3",
          "ElevenLabs Scribe",
          "Deepgram Nova-3",
          "AssemblyAI Univ.-2"
        ],
        "scores": [
          0.039,
          0.039,
          0.04,
          0.041,
          0.042,
          0.051
        ]
      }
    }
  ],
  "sources": [
    {
      "label": "OpenAI GPT-4o model documentation",
      "url": "https://developers.openai.com/api/docs/models/gpt-4o",
      "note": "Model identity and version reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Anthropic model IDs and versioning",
      "url": "https://platform.claude.com/docs/en/about-claude/models/model-ids-and-versions",
      "note": "Model identity and version reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Google Gemini model documentation",
      "url": "https://ai.google.dev/gemini-api/docs/models",
      "note": "Model identity and version reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "OpenAI audio documentation",
      "url": "https://developers.openai.com/api/docs/guides/audio",
      "note": "Audio model identity and interface reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Deepgram models and languages",
      "url": "https://developers.deepgram.com/docs/models-languages-overview",
      "note": "Provider model and language coverage reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "AssemblyAI speech-to-text models",
      "url": "https://www.assemblyai.com/docs/getting-started/models",
      "note": "Provider model identity reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "ElevenLabs speech-to-text API",
      "url": "https://elevenlabs.io/docs/api-reference/speech-to-text/convert",
      "note": "Provider model and interface reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "OpenAI Whisper large-v3 model card",
      "url": "https://huggingface.co/openai/whisper-large-v3",
      "note": "Model identity and model-card reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "FLEURS",
      "url": "https://huggingface.co/datasets/google/fleurs",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Common Voice",
      "url": "https://commonvoice.mozilla.org/en/datasets",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "VoxPopuli",
      "url": "https://github.com/facebookresearch/voxpopuli",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "ASCEND",
      "url": "https://huggingface.co/datasets/CAiRE/ASCEND",
      "accessedOn": "2026-07-21"
    }
  ]
}