{
  "id": "2310.09250",
  "version": "v1",
  "published": "2023-10-13T17:06:34.000Z",
  "updated": "2023-10-13T17:06:34.000Z",
  "title": "It's an Alignment, Not a Trade-off: Revisiting Bias and Variance in Deep Models",
  "authors": [
    "Lin Chen",
    "Michal Lukasik",
    "Wittawat Jitkrittum",
    "Chong You",
    "Sanjiv Kumar"
  ],
  "categories": [
    "cs.LG",
    "cs.AI",
    "stat.ML"
  ],
  "abstract": "Classical wisdom in machine learning holds that the generalization error can be decomposed into bias and variance, and these two terms exhibit a \\emph{trade-off}. However, in this paper, we show that for an ensemble of deep learning based classification models, bias and variance are \\emph{aligned} at a sample level, where squared bias is approximately \\emph{equal} to variance for correctly classified sample points. We present empirical evidence confirming this phenomenon in a variety of deep learning models and datasets. Moreover, we study this phenomenon from two theoretical perspectives: calibration and neural collapse. We first show theoretically that under the assumption that the models are well calibrated, we can observe the bias-variance alignment. Second, starting from the picture provided by the neural collapse theory, we show an approximate correlation between bias and variance.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2023-10-13T17:06:34.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "deep models",
      "revisiting bias",
      "neural collapse theory",
      "classification models",
      "correctly classified sample points"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}