{
  "id": "2104.11178",
  "version": "v1",
  "published": "2021-04-22T17:07:41.000Z",
  "updated": "2021-04-22T17:07:41.000Z",
  "title": "VATT: Transformers for Multimodal Self-Supervised Learning from Raw Video, Audio and Text",
  "authors": [
    "Hassan Akbari",
    "Linagzhe Yuan",
    "Rui Qian",
    "Wei-Hong Chuang",
    "Shih-Fu Chang",
    "Yin Cui",
    "Boqing Gong"
  ],
  "categories": [
    "cs.CV",
    "cs.AI",
    "cs.LG",
    "cs.MM",
    "eess.IV"
  ],
  "abstract": "We present a framework for learning multimodal representations from unlabeled data using convolution-free Transformer architectures. Specifically, our Video-Audio-Text Transformer (VATT) takes raw signals as inputs and extracts multimodal representations that are rich enough to benefit a variety of downstream tasks. We train VATT end-to-end from scratch using multimodal contrastive losses and evaluate its performance by the downstream tasks of video action recognition, audio event classification, image classification, and text-to-video retrieval. Furthermore, we study a modality-agnostic single-backbone Transformer by sharing weights among the three modalities. We show that the convolution-free VATT outperforms state-of-the-art ConvNet-based architectures in the downstream tasks. Especially, VATT's vision Transformer achieves the top-1 accuracy of 82.1% on Kinetics-400, 83.6% on Kinetics-600,and 41.1% on Moments in Time, new records while avoiding supervised pre-training. Transferring to image classification leads to 78.7% top-1 accuracy on ImageNet compared to 64.7% by training the same Transformer from scratch, showing the generalizability of our model despite the domain gap between videos and images. VATT's audio Transformer also sets a new record on waveform-based audio event recognition by achieving the mAP of 39.4% on AudioSet without any supervised pre-training.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2021-04-22T17:07:41.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "multimodal self-supervised learning",
      "raw video",
      "downstream tasks",
      "vatt outperforms state-of-the-art convnet-based architectures",
      "vatts vision transformer achieves"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}