{
  "id": "2001.10509",
  "version": "v1",
  "published": "2020-01-28T18:25:15.000Z",
  "updated": "2020-01-28T18:25:15.000Z",
  "title": "MSE-Optimal Neural Network Initialization via Layer Fusion",
  "authors": [
    "Ramina Ghods",
    "Andrew S. Lan",
    "Tom Goldstein",
    "Christoph Studer"
  ],
  "comment": "Extended version of the CISS 2020 paper containing the proof for convolutional layers",
  "categories": [
    "cs.LG",
    "eess.SP",
    "stat.ML"
  ],
  "abstract": "Deep neural networks achieve state-of-the-art performance for a range of classification and inference tasks. However, the use of stochastic gradient descent combined with the nonconvexity of the underlying optimization problems renders parameter learning susceptible to initialization. To address this issue, a variety of methods that rely on random parameter initialization or knowledge distillation have been proposed in the past. In this paper, we propose FuseInit, a novel method to initialize shallower networks by fusing neighboring layers of deeper networks that are trained with random initialization. We develop theoretical results and efficient algorithms for mean-square error (MSE)-optimal fusion of neighboring dense-dense, convolutional-dense, and convolutional-convolutional layers. We show experiments for a range of classification and regression datasets, which suggest that deeper neural networks are less sensitive to initialization and shallower networks can perform better (sometimes as well as their deeper counterparts) if initialized with FuseInit.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2020-01-28T18:25:15.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "mse-optimal neural network initialization",
      "layer fusion",
      "problems renders parameter learning",
      "neural networks achieve state-of-the-art performance",
      "deep neural networks achieve state-of-the-art"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}