{ "title": "VoicePro-Bench", "subtitle": "Evaluating frontier voice AI models on professional speech understanding", "url": "https://www.datoric.com/research/voicepro-bench", "version": "2026-07-21 report edition", "publishedOn": "2026-05-10", "lastReviewed": "2026-07-21", "authors": [ { "name": "Jeffrey Lin", "affiliation": "Datoric Research" }, { "name": "Nikhil Reddy", "affiliation": "Datoric Research" } ], "headlineStat": { "value": "27 pp", "label": "SLURP intent F1 gap: best text-reasoner control over the best audio-native MLLM: audio understanding is the dominant source of error" }, "datasetSummary": "Five axes: 760-sample FLEURS + VoxPopuli transcription (200 evaluated), 200 SLURP intent + entity, 200 MELD emotion, 248 AMI reasoning questions, 200-sample MUSAN/WHAM noise cliff at four SNR levels. 12 models. Bootstrap 95% CI.", "changelog": [ { "date": "2026-07-21", "summary": "Added named authors, versioned report resources, downloadable tables, structured citations, and synchronized HTML and PDF editions." }, { "date": "2026-07-17", "summary": "Aligned public asset descriptions and added primary-source links." } ], "tables": [ { "caption": "Table 1. VoicePro-Bench transcription results, lower-is-better. WER and CER on a 200-sample evaluation subset across English and six accented language variants.", "headers": [ "Model", "Class", "WER", "CER" ], "rows": [ [ "Whisper v3", "Dedicated ASR", "0.432", "0.137" ], [ "Deepgram Nova-3", "Dedicated ASR", "0.429", "0.144" ], [ "Deepgram Nova-2", "Dedicated ASR", "0.432", "0.144" ], [ "AssemblyAI Univ.-2", "Dedicated ASR", "0.427", "0.125" ], [ "ElevenLabs Scribe", "Dedicated ASR", "0.408", "0.132" ], [ "GPT-4o Audio", "Audio MLLM", "0.612", "0.308" ], [ "GPT-4o-mini Audio", "Audio MLLM", "0.611", "0.334" ], [ "Gemini 2.5 Pro", "Audio MLLM", "0.411", "0.132" ], [ "Gemini 2.5 Flash", "Audio MLLM", "0.429", "0.143" ], [ "Claude Opus 4.5", "Text control", "0.391", "0.116" ], [ "Claude Sonnet 4.5", "Text control", "0.395", "0.128" ], [ "Claude Haiku 4.5", "Text control", "0.385", "0.106" ] ] } ], "figures": [ { "variant": "bar-horizontal", "caption": "Figure 1. WER (lower is better) across all twelve evaluated systems, post the 2026-04-22 rate-limit backfill on the four audio-native MLLMs. ElevenLabs Scribe and Gemini 2.5 Pro are tied at the top of the audio-facing tier on point estimate (0.408 vs 0.411) with overlapping CIs; the rest of the dedicated-ASR tier clusters within 0.024 WER; GPT-4o Audio and its mini variant remain outliers.", "data": { "tasks": [ "WER" ], "models": [ { "name": "ElevenLabs Scribe", "scores": [ 0.408 ], "color": "#0A0A0A" }, { "name": "Gemini 2.5 Pro", "scores": [ 0.411 ], "color": "#1F1F1F" }, { "name": "AssemblyAI Univ.-2", "scores": [ 0.427 ], "color": "#3A3A3A" }, { "name": "Deepgram Nova-3", "scores": [ 0.429 ], "color": "#555555" }, { "name": "Gemini 2.5 Flash", "scores": [ 0.429 ], "color": "#6B6B6B" }, { "name": "Whisper v3", "scores": [ 0.432 ], "color": "#7B7B7B" }, { "name": "Deepgram Nova-2", "scores": [ 0.432 ], "color": "#8C8C8C" }, { "name": "GPT-4o-mini Audio", "scores": [ 0.611 ], "color": "#A0A0A0" }, { "name": "GPT-4o Audio", "scores": [ 0.612 ], "color": "#C0C0BC" }, { "name": "Claude Haiku 4.5 (text)", "scores": [ 0.385 ], "color": "#D6D6D2" }, { "name": "Claude Opus 4.5 (text)", "scores": [ 0.391 ], "color": "#DEDED9" }, { "name": "Claude Sonnet 4.5 (text)", "scores": [ 0.395 ], "color": "#E6E6E1" } ] } } ], "sources": [ { "label": "OpenAI GPT-4o model documentation", "url": "https://developers.openai.com/api/docs/models/gpt-4o", "note": "Model identity and version reference.", "accessedOn": "2026-07-21" }, { "label": "Anthropic model IDs and versioning", "url": "https://platform.claude.com/docs/en/about-claude/models/model-ids-and-versions", "note": "Model identity and version reference.", "accessedOn": "2026-07-21" }, { "label": "Google Gemini model documentation", "url": "https://ai.google.dev/gemini-api/docs/models", "note": "Model identity and version reference.", "accessedOn": "2026-07-21" }, { "label": "OpenAI audio documentation", "url": "https://developers.openai.com/api/docs/guides/audio", "note": "Audio model identity and interface reference.", "accessedOn": "2026-07-21" }, { "label": "Deepgram models and languages", "url": "https://developers.deepgram.com/docs/models-languages-overview", "note": "Provider model and language coverage reference.", "accessedOn": "2026-07-21" }, { "label": "AssemblyAI speech-to-text models", "url": "https://www.assemblyai.com/docs/getting-started/models", "note": "Provider model identity reference.", "accessedOn": "2026-07-21" }, { "label": "ElevenLabs speech-to-text API", "url": "https://elevenlabs.io/docs/api-reference/speech-to-text/convert", "note": "Provider model and interface reference.", "accessedOn": "2026-07-21" }, { "label": "OpenAI Whisper large-v3 model card", "url": "https://huggingface.co/openai/whisper-large-v3", "note": "Model identity and model-card reference.", "accessedOn": "2026-07-21" }, { "label": "FLEURS", "url": "https://huggingface.co/datasets/google/fleurs", "accessedOn": "2026-07-21" }, { "label": "VoxPopuli", "url": "https://github.com/facebookresearch/voxpopuli", "accessedOn": "2026-07-21" }, { "label": "SLURP", "url": "https://github.com/pswietojanski/slurp", "accessedOn": "2026-07-21" }, { "label": "MELD", "url": "https://affective-meld.github.io/", "accessedOn": "2026-07-21" }, { "label": "AMI Meeting Corpus", "url": "https://groups.inf.ed.ac.uk/ami/corpus/", "accessedOn": "2026-07-21" }, { "label": "MUSAN", "url": "https://www.openslr.org/17/", "accessedOn": "2026-07-21" }, { "label": "WHAM!", "url": "http://wham.whisper.ai/", "accessedOn": "2026-07-21" } ] }