{
  "id": "2310.14062",
  "version": "v1",
  "published": "2023-10-21T16:47:18.000Z",
  "updated": "2023-10-21T16:47:18.000Z",
  "title": "On the Neural Tangent Kernel of Equilibrium Models",
  "authors": [
    "Zhili Feng",
    "J. Zico Kolter"
  ],
  "categories": [
    "cs.LG",
    "cs.AI"
  ],
  "abstract": "This work studies the neural tangent kernel (NTK) of the deep equilibrium (DEQ) model, a practical ``infinite-depth'' architecture which directly computes the infinite-depth limit of a weight-tied network via root-finding. Even though the NTK of a fully-connected neural network can be stochastic if its width and depth both tend to infinity simultaneously, we show that contrarily a DEQ model still enjoys a deterministic NTK despite its width and depth going to infinity at the same time under mild conditions. Moreover, this deterministic NTK can be found efficiently via root-finding.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2023-10-21T16:47:18.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "neural tangent kernel",
      "equilibrium models",
      "deterministic ntk despite",
      "infinite-depth",
      "mild conditions"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}