{
  "id": "2303.08360",
  "version": "v1",
  "published": "2023-03-15T04:39:01.000Z",
  "updated": "2023-03-15T04:39:01.000Z",
  "title": "Knowledge Distillation from Single to Multi Labels: an Empirical Study",
  "authors": [
    "Youcai Zhang",
    "Yuzhuo Qin",
    "Hengwei Liu",
    "Yanhao Zhang",
    "Yaqian Li",
    "Xiaodong Gu"
  ],
  "categories": [
    "cs.CV"
  ],
  "abstract": "Knowledge distillation (KD) has been extensively studied in single-label image classification. However, its efficacy for multi-label classification remains relatively unexplored. In this study, we firstly investigate the effectiveness of classical KD techniques, including logit-based and feature-based methods, for multi-label classification. Our findings indicate that the logit-based method is not well-suited for multi-label classification, as the teacher fails to provide inter-category similarity information or regularization effect on student model's training. Moreover, we observe that feature-based methods struggle to convey compact information of multiple labels simultaneously. Given these limitations, we propose that a suitable dark knowledge should incorporate class-wise information and be highly correlated with the final classification results. To address these issues, we introduce a novel distillation method based on Class Activation Maps (CAMs), which is both effective and straightforward to implement. Across a wide range of settings, CAMs-based distillation consistently outperforms other methods.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2023-03-15T04:39:01.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "knowledge distillation",
      "multi labels",
      "empirical study",
      "feature-based methods",
      "single-label image classification"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}