{
  "id": "2401.09861",
  "version": "v1",
  "published": "2024-01-18T10:18:48.000Z",
  "updated": "2024-01-18T10:18:48.000Z",
  "title": "Temporal Insight Enhancement: Mitigating Temporal Hallucination in Multimodal Large Language Models",
  "authors": [
    "Li Sun",
    "Liuan Wang",
    "Jun Sun",
    "Takayuki Okatani"
  ],
  "comment": "7 pages, 7 figures",
  "categories": [
    "cs.CV",
    "cs.AI"
  ],
  "abstract": "Recent advancements in Multimodal Large Language Models (MLLMs) have significantly enhanced the comprehension of multimedia content, bringing together diverse modalities such as text, images, and videos. However, a critical challenge faced by these models, especially when processing video inputs, is the occurrence of hallucinations - erroneous perceptions or interpretations, particularly at the event level. This study introduces an innovative method to address event-level hallucinations in MLLMs, focusing on specific temporal understanding in video content. Our approach leverages a novel framework that extracts and utilizes event-specific information from both the event query and the provided video to refine MLLMs' response. We propose a unique mechanism that decomposes on-demand event queries into iconic actions. Subsequently, we employ models like CLIP and BLIP2 to predict specific timestamps for event occurrences. Our evaluation, conducted using the Charades-STA dataset, demonstrates a significant reduction in temporal hallucinations and an improvement in the quality of event-related responses. This research not only provides a new perspective in addressing a critical limitation of MLLMs but also contributes a quantitatively measurable method for evaluating MLLMs in the context of temporal-related questions.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2024-01-18T10:18:48.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "multimodal large language models",
      "temporal insight enhancement",
      "mitigating temporal hallucination",
      "event query",
      "decomposes on-demand event queries"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 7,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}