{
  "id": "2405.16027",
  "version": "v1",
  "published": "2024-05-25T03:00:06.000Z",
  "updated": "2024-05-25T03:00:06.000Z",
  "title": "Feature Protection For Out-of-distribution Generalization",
  "authors": [
    "Lu Tan",
    "Huei Zhou",
    "Yinxiang Huang",
    "Zeming Zheng",
    "Yujiu Yang"
  ],
  "comment": "arXiv admin note: substantial text overlap with arXiv:2309.06256",
  "categories": [
    "cs.LG"
  ],
  "abstract": "With the availability of large pre-trained models, a modern workflow for building real-world machine learning solutions is to fine-tune such models on a downstream task with a relatively small domain-specific dataset. In such applications, one major challenge is that the small fine-tuning dataset does not have sufficient coverage of the distribution encountered when the model is deployed. It is thus important to design fine-tuning methods that are robust to out-of-distribution (OOD) data that are under-represented by the training data. This paper compares common fine-tuning methods to investigate their OOD performance and demonstrates that standard methods will result in a significant change to the pre-trained model so that the fine-tuned features overfit the fine-tuning dataset. However, this causes deteriorated OOD performance. To overcome this issue, we show that protecting pre-trained features leads to a fine-tuned model more robust to OOD generalization. We validate the feature protection methods with extensive experiments of fine-tuning CLIP on ImageNet and DomainNet.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2024-05-25T03:00:06.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "out-of-distribution generalization",
      "ood performance",
      "fine-tuning dataset",
      "relatively small domain-specific dataset",
      "fine-tuning methods"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}