{
  "id": "1602.04485",
  "version": "v1",
  "published": "2016-02-14T18:36:59.000Z",
  "updated": "2016-02-14T18:36:59.000Z",
  "title": "Benefits of depth in neural networks",
  "authors": [
    "Matus Telgarsky"
  ],
  "comment": "For a simplified version, see http://arxiv.org/abs/1509.08101",
  "categories": [
    "cs.LG",
    "cs.NE",
    "stat.ML"
  ],
  "abstract": "For any positive integer $k$, there exist neural networks with $\\Theta(k^3)$ layers, $\\Theta(1)$ nodes per layer, and $\\Theta(1)$ distinct parameters which can not be approximated by networks with $\\mathcal{O}(k)$ layers unless they are exponentially large --- they must possess $\\Omega(2^k)$ nodes. This result is proved here for a class of nodes termed \"semi-algebraic gates\" which includes the common choices of ReLU, maximum, indicator, and piecewise polynomial functions, therefore establishing benefits of depth against not just standard networks with ReLU gates, but also convolutional networks with ReLU and maximization gates, and boosted decision trees (in this last case with a stronger separation: $\\Omega(2^{k^3})$ total tree nodes are required).",
  "revisions": [
    {
      "version": "v1",
      "updated": "2016-02-14T18:36:59.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "neural networks",
      "total tree nodes",
      "distinct parameters",
      "stronger separation",
      "boosted decision trees"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable",
      "adsabs": "2016arXiv160204485T"
    }
  }
}