{
  "title": "VidWork-Bench",
  "subtitle": "A five-axis benchmark for procedural video understanding",
  "url": "https://www.datoric.com/research/vidwork-bench",
  "version": "2026-07-21 report edition",
  "publishedOn": "2026-06-10",
  "lastReviewed": "2026-07-21",
  "authors": [
    {
      "name": "Jeffrey Lin",
      "affiliation": "Datoric Research"
    },
    {
      "name": "Nikhil Reddy",
      "affiliation": "Datoric Research"
    }
  ],
  "headlineStat": {
    "value": "0.446",
    "label": "composite leaderboard winner: Claude Sonnet 4.5 narrowly ahead of Haiku 4.5 and Opus 4.5, with the GPT-4o family 8–13 points behind"
  },
  "datasetSummary": "171 procedural clips (30s–5 min) across cooking, repair/manufacturing, and first-aid/safety. 2,092 QA items across 5 axes. 10,686 scored model responses. 6 frontier vision–language models. Bootstrap 95% CI.",
  "changelog": [
    {
      "date": "2026-07-21",
      "summary": "Added named authors, versioned report resources, downloadable tables, structured citations, and synchronized HTML and PDF editions."
    },
    {
      "date": "2026-07-17",
      "summary": "Aligned public asset descriptions and added primary-source links."
    }
  ],
  "tables": [
    {
      "caption": "Table 1. Paired 1-frame vs 8-frame ablation on temporal ordering (n = 90) and causal reasoning (n = 225). Δ is per-item score difference (1f − 8f); positive means 1-frame better. Paired bootstrap, 1,000 resamples, seed 42.",
      "headers": [
        "Model",
        "Axis",
        "n",
        "1f",
        "8f",
        "Δ (1f − 8f)",
        "Sig."
      ],
      "rows": [
        [
          "Sonnet 4.5",
          "Temporal",
          "90",
          "0.365",
          "0.387",
          "−0.022",
          "ns"
        ],
        [
          "Sonnet 4.5",
          "Causal",
          "225",
          "0.491",
          "0.503",
          "−0.012",
          "ns"
        ],
        [
          "GPT-4o",
          "Temporal",
          "90",
          "0.203",
          "0.218",
          "−0.014",
          "ns"
        ],
        [
          "GPT-4o",
          "Causal",
          "225",
          "0.397",
          "0.383",
          "+0.014",
          "p < 0.05"
        ]
      ]
    },
    {
      "caption": "Table 2. VidWork-Bench leaderboard. Scores are per-axis mean accuracy/F1 (higher is better). Composite is the unweighted mean across the five axes, reported only for models with full coverage. Gemini 2.5 Flash completed only step recognition (30% 503 rate).",
      "headers": [
        "Model",
        "Step (F1)",
        "Temporal",
        "Causal",
        "X-Modal",
        "Error Det.",
        "Composite"
      ],
      "rows": [
        [
          "Claude Sonnet 4.5",
          "0.048",
          "0.387",
          "0.503",
          "0.343",
          "0.947",
          "0.446"
        ],
        [
          "Claude Haiku 4.5",
          "0.036",
          "0.308",
          "0.487",
          "0.342",
          "0.936",
          "0.422"
        ],
        [
          "Claude Opus 4.5",
          "0.035",
          "0.335",
          "0.457",
          "0.343",
          "0.933",
          "0.420"
        ],
        [
          "GPT-4o-mini",
          "0.094",
          "0.238",
          "0.426",
          "0.270",
          "0.647",
          "0.335"
        ],
        [
          "GPT-4o",
          "0.104",
          "0.218",
          "0.383",
          "0.223",
          "0.639",
          "0.313"
        ],
        [
          "Gemini 2.5 Flash",
          "0.056",
          "N/A",
          "N/A",
          "N/A",
          "N/A",
          "N/A"
        ]
      ]
    },
    {
      "caption": "Table 3. Adversarial error-detection rate by error type. \"Detection\" = the model flags the adversarial description as incorrect. Best per row in italics in the paper; here the weighted-mean column is highlighted. The degenerate insufficient-input category (n = 1) and step_modification (n = 3) are saturated for every model and omitted here.",
      "headers": [
        "Error type",
        "n",
        "GPT-4o",
        "GPT-4o-mini",
        "Haiku 4.5",
        "Sonnet 4.5",
        "Opus 4.5"
      ],
      "rows": [
        [
          "step_swap",
          "428",
          "0.729",
          "0.755",
          "0.956",
          "0.967",
          "0.967"
        ],
        [
          "step_omission",
          "206",
          "0.592",
          "0.738",
          "0.966",
          "0.947",
          "0.850"
        ],
        [
          "action_modification",
          "181",
          "0.597",
          "0.481",
          "0.895",
          "0.934",
          "0.945"
        ],
        [
          "tool_substitution",
          "124",
          "0.532",
          "0.524",
          "0.911",
          "0.952",
          "0.968"
        ],
        [
          "causal_reversal",
          "123",
          "0.602",
          "0.537",
          "0.911",
          "0.911",
          "0.911"
        ],
        [
          "quantity_error",
          "24",
          "0.458",
          "0.375",
          "0.917",
          "0.875",
          "0.917"
        ],
        [
          "Weighted mean",
          "1,090",
          "0.639",
          "0.647",
          "0.936",
          "0.947",
          "0.933"
        ]
      ]
    }
  ],
  "figures": [],
  "sources": [
    {
      "label": "OpenAI GPT-4o model documentation",
      "url": "https://developers.openai.com/api/docs/models/gpt-4o",
      "note": "Model identity and version reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Anthropic model IDs and versioning",
      "url": "https://platform.claude.com/docs/en/about-claude/models/model-ids-and-versions",
      "note": "Model identity and version reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Google Gemini model documentation",
      "url": "https://ai.google.dev/gemini-api/docs/models",
      "note": "Model identity and version reference.",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "YouCook2",
      "url": "http://youcook2.eecs.umich.edu/",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "COIN",
      "url": "https://coin-dataset.github.io/",
      "accessedOn": "2026-07-21"
    },
    {
      "label": "Curated instructional content",
      "note": "Study-created benchmark subset described in the report.",
      "accessedOn": "2026-07-21"
    }
  ]
}