{
  "title": "VideoTruth-Bench",
  "subtitle": "Cross-modal consistency verification across six contradiction levels",
  "url": "https://www.datoric.com/research/videotruth-bench",
  "version": "2026-07-21 report edition",
  "publishedOn": "2026-06-24",
  "lastReviewed": "2026-07-21",
  "authors": [
    {
      "name": "Jeffrey Lin",
      "affiliation": "Datoric Research"
    },
    {
      "name": "Nikhil Reddy",
      "affiliation": "Datoric Research"
    }
  ],
  "headlineStat": {
    "value": "24–60 pp",
    "label": "sycophancy-gap range across six frontier models under an \"already-verified\" preamble: no model in the frontier tier is sycophancy-immune"
  },
  "datasetSummary": "100 VATEX videos; 566 model×task evaluations per model. Four axes: six-level contradiction taxonomy, temporal ordering, hallucination refusal, and sycophancy under three prompt framings. 6 frontier multimodal models, eight real video frames per API call. Bootstrap 95% CI. A caption-only ablation is described in the report.",
  "changelog": [
    {
      "date": "2026-07-21",
      "summary": "Added named authors, versioned report resources, downloadable tables, structured citations, and synchronized HTML and PDF editions."
    },
    {
      "date": "2026-07-17",
      "summary": "Aligned public asset descriptions and added primary-source links."
    }
  ],
  "tables": [
    {
      "caption": "Table 1. Contradiction-detection accuracy by level across all six evaluated models. L1–L4 saturate for the top three; L5 and L6 spread the field. L6 (omission) is the universal ceiling: no model exceeds 0.74.",
      "headers": [
        "Model",
        "L1 Entity",
        "L2 Temp.",
        "L3 Quant.",
        "L4 Attr.",
        "L5 Causal",
        "L6 Omiss.",
        "Overall",
        "n"
      ],
      "rows": [
        [
          "Claude Haiku 4.5",
          "0.889",
          "0.900",
          "1.000",
          "0.760",
          "0.962",
          "0.733",
          "0.870",
          "162"
        ],
        [
          "Gemini 2.5 Flash",
          "1.000",
          "0.909",
          "0.812",
          "1.000",
          "0.952",
          "0.593",
          "0.869",
          "130"
        ],
        [
          "GPT-4o",
          "1.000",
          "0.833",
          "0.875",
          "0.920",
          "0.846",
          "0.733",
          "0.864",
          "162"
        ],
        [
          "Claude Sonnet 4.5",
          "0.889",
          "0.900",
          "0.750",
          "0.760",
          "0.962",
          "0.533",
          "0.796",
          "162"
        ],
        [
          "Claude Opus 4.5",
          "0.926",
          "0.833",
          "0.750",
          "0.800",
          "0.769",
          "0.500",
          "0.759",
          "162"
        ],
        [
          "Gemini 2.5 Pro",
          "0.731",
          "0.733",
          "0.625",
          "0.760",
          "0.577",
          "0.500",
          "0.652",
          "161"
        ]
      ]
    },
    {
      "caption": "Table 2. Detection rates by prompt variant across all six evaluated models. The sycophancy gap is direct minus adversarial. Each framing is evaluated at n=30 per model where coverage permits.",
      "headers": [
        "Model",
        "Direct",
        "Indirect",
        "Adversarial",
        "Gap (pp)"
      ],
      "rows": [
        [
          "Gemini 2.5 Flash",
          "0.967",
          "0.967",
          "0.731",
          "23.6"
        ],
        [
          "Claude Haiku 4.5",
          "0.933",
          "0.933",
          "0.667",
          "26.7"
        ],
        [
          "Gemini 2.5 Pro",
          "0.933",
          "1.000",
          "0.667",
          "26.7"
        ],
        [
          "GPT-4o",
          "1.000",
          "0.967",
          "0.667",
          "33.3"
        ],
        [
          "Claude Sonnet 4.5",
          "0.833",
          "1.000",
          "0.433",
          "40.0"
        ],
        [
          "Claude Opus 4.5",
          "0.900",
          "0.900",
          "0.300",
          "60.0"
        ],
        [
          "Macro-avg",
          "0.928",
          "0.961",
          "0.578",
          "35.7"
        ]
      ]
    },
    {
      "caption": "Table 3. Temporal binary before/after accuracy and hallucination refusal rate (fraction of probes about absent events that are correctly refused: higher is better).",
      "headers": [
        "Model",
        "Temporal (B/A)",
        "n_t",
        "Halluc. Refusal",
        "n_h"
      ],
      "rows": [
        [
          "Claude Haiku 4.5",
          "0.667",
          "60",
          "0.930",
          "100"
        ],
        [
          "Claude Opus 4.5",
          "0.617",
          "60",
          "0.880",
          "100"
        ],
        [
          "Claude Sonnet 4.5",
          "0.567",
          "60",
          "0.950",
          "100"
        ],
        [
          "Gemini 2.5 Flash",
          "0.564",
          "55",
          "0.663",
          "95"
        ],
        [
          "Gemini 2.5 Pro",
          "0.577",
          "52",
          "0.589",
          "95"
        ],
        [
          "GPT-4o",
          "0.583",
          "60",
          "0.950",
          "100"
        ]
      ]
    }
  ],
  "figures": [
    {
      "variant": "gap-bars",
      "caption": "Figure 1. Contradiction detection under direct versus adversarial framing across all six evaluated models. The gap between the two bars is the sycophancy effect. At full video-frame coverage the range is 23.6 to 60.0 pp: no model is immune. Gemini 2.5 Flash is the most robust (23.6 pp); Claude Opus 4.5 is the most susceptible (60.0 pp).",
      "data": {
        "tiers": [
          "Direct framing",
          "Adversarial framing"
        ],
        "models": [
          {
            "name": "Gemini 2.5 Flash",
            "scores": [
              0.967,
              0.731
            ]
          },
          {
            "name": "Claude Haiku 4.5",
            "scores": [
              0.933,
              0.667
            ]
          },
          {
            "name": "Gemini 2.5 Pro",
            "scores": [
              0.933,
              0.667
            ]
          },
          {
            "name": "GPT-4o",
            "scores": [
              1,
              0.667
            ]
          },
          {
            "name": "Claude Sonnet 4.5",
            "scores": [
              0.833,
              0.433
            ]
          },
          {
            "name": "Claude Opus 4.5",
            "scores": [
              0.9,
              0.3
            ]
          }
        ]
      }
    }
  ],
  "sources": [
    {
      "label": "OpenAI GPT-4o model documentation",
      "url": "https://developers.openai.com/api/docs/models/gpt-4o",
      "note": "Model identity and version reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Anthropic model IDs and versioning",
      "url": "https://platform.claude.com/docs/en/about-claude/models/model-ids-and-versions",
      "note": "Model identity and version reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Google Gemini model documentation",
      "url": "https://ai.google.dev/gemini-api/docs/models",
      "note": "Model identity and version reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "VATEX",
      "url": "https://eric-xw.github.io/vatex-website/",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Synthetic adversarial constructions",
      "note": "Study-created benchmark materials described in the report.",
      "accessedOn": "2026-07-21"
    }
  ]
}