{
  "id": "2312.10136",
  "version": "v1",
  "published": "2023-12-15T18:59:05.000Z",
  "updated": "2023-12-15T18:59:05.000Z",
  "title": "Gradient-based Parameter Selection for Efficient Fine-Tuning",
  "authors": [
    "Zhi Zhang",
    "Qizhe Zhang",
    "Zijun Gao",
    "Renrui Zhang",
    "Ekaterina Shutova",
    "Shiji Zhou",
    "Shanghang Zhang"
  ],
  "categories": [
    "cs.CV"
  ],
  "abstract": "With the growing size of pre-trained models, full fine-tuning and storing all the parameters for various downstream tasks is costly and infeasible. In this paper, we propose a new parameter-efficient fine-tuning method, Gradient-based Parameter Selection (GPS), demonstrating that only tuning a few selected parameters from the pre-trained model while keeping the remainder of the model frozen can generate similar or better performance compared with the full model fine-tuning method. Different from the existing popular and state-of-the-art parameter-efficient fine-tuning approaches, our method does not introduce any additional parameters and computational costs during both the training and inference stages. Another advantage is the model-agnostic and non-destructive property, which eliminates the need for any other design specific to a particular model. Compared with the full fine-tuning, GPS achieves 3.33% (91.78% vs. 88.45%, FGVC) and 9.61% (73.1% vs. 65.57%, VTAB) improvement of the accuracy with tuning only 0.36% parameters of the pre-trained model on average over 24 image classification tasks; it also demonstrates a significant improvement of 17% and 16.8% in mDice and mIoU, respectively, on medical image segmentation task. Moreover, GPS achieves state-of-the-art performance compared with existing PEFT methods.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2023-12-15T18:59:05.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "gradient-based parameter selection",
      "pre-trained model",
      "gps achieves state-of-the-art performance",
      "medical image segmentation task",
      "full model fine-tuning method"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}