{
  "id": "2203.10789",
  "version": "v1",
  "published": "2022-03-21T08:07:46.000Z",
  "updated": "2022-03-21T08:07:46.000Z",
  "title": "Domain Generalization by Mutual-Information Regularization with Pre-trained Models",
  "authors": [
    "Junbum Cha",
    "Kyungjae Lee",
    "Sungrae Park",
    "Sanghyuk Chun"
  ],
  "categories": [
    "cs.LG",
    "cs.CV"
  ],
  "abstract": "Domain generalization (DG) aims to learn a generalized model to an unseen target domain using only limited source domains. Previous attempts to DG fail to learn domain-invariant representations only from the source domains due to the significant domain shifts between training and test domains. Instead, we re-formulate the DG objective using mutual information with the oracle model, a model generalized to any possible domain. We derive a tractable variational lower bound via approximating the oracle model by a pre-trained model, called Mutual Information Regularization with Oracle (MIRO). Our extensive experiments show that MIRO significantly improves the out-of-distribution performance. Furthermore, our scaling experiments show that the larger the scale of the pre-trained model, the greater the performance improvement of MIRO. Source code is available at https://github.com/kakaobrain/miro.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2022-03-21T08:07:46.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "pre-trained model",
      "domain generalization",
      "mutual-information regularization",
      "source domains",
      "oracle model"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}