{
  "id": "2403.09296",
  "version": "v1",
  "published": "2024-03-14T11:36:36.000Z",
  "updated": "2024-03-14T11:36:36.000Z",
  "title": "Select and Distill: Selective Dual-Teacher Knowledge Transfer for Continual Learning on Vision-Language Models",
  "authors": [
    "Yu-Chu Yu",
    "Chi-Pin Huang",
    "Jr-Jen Chen",
    "Kai-Po Chang",
    "Yung-Hsuan Lai",
    "Fu-En Yang",
    "Yu-Chiang Frank Wang"
  ],
  "categories": [
    "cs.CV"
  ],
  "abstract": "Large-scale vision-language models (VLMs) have shown a strong zero-shot generalization capability on unseen-domain data. However, when adapting pre-trained VLMs to a sequence of downstream tasks, they are prone to forgetting previously learned knowledge and degrade their zero-shot classification capability. To tackle this problem, we propose a unique Selective Dual-Teacher Knowledge Transfer framework that leverages the most recent fine-tuned and the original pre-trained VLMs as dual teachers to preserve the previously learned knowledge and zero-shot capabilities, respectively. With only access to an unlabeled reference dataset, our proposed framework performs a selective knowledge distillation mechanism by measuring the feature discrepancy from the dual teacher VLMs. Consequently, our selective dual-teacher knowledge distillation would mitigate catastrophic forgetting of previously learned knowledge while preserving the zero-shot capabilities from pre-trained VLMs. Through extensive experiments on benchmark datasets, we show that our proposed framework is favorable against state-of-the-art continual learning approaches for preventing catastrophic forgetting and zero-shot degradation.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2024-03-14T11:36:36.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "vision-language models",
      "continual learning",
      "pre-trained vlms",
      "learned knowledge",
      "selective dual-teacher knowledge transfer framework"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}