{
  "id": "2303.05924",
  "version": "v2",
  "published": "2023-03-09T00:47:30.000Z",
  "updated": "2023-04-06T04:03:09.000Z",
  "title": "Variational formulations of ODE-Net as a mean-field optimal control problem and existence results",
  "authors": [
    "Noboru Isobe",
    "Mizuho Okumura"
  ],
  "comment": "33 pages",
  "categories": [
    "math.AP",
    "cs.LG",
    "math.OC"
  ],
  "abstract": "This paper presents a mathematical analysis of ODE-Net, a continuum model of deep neural networks (DNNs). In recent years, Machine Learning researchers have introduced ideas of replacing the deep structure of DNNs with ODEs as a continuum limit. These studies regard the \"learning\" of ODE-Net as the minimization of a \"loss\" constrained by a parametric ODE. Although the existence of a minimizer for this minimization problem needs to be assumed, only a few studies have investigated its existence analytically in detail. In the present paper, the existence of a minimizer is discussed based on a formulation of ODE-Net as a measure-theoretic mean-field optimal control problem. The existence result is proved when a neural network, which describes a vector field of ODE-Net, is linear with respect to learnable parameters. The proof employs the measure-theoretic formulation combined with the direct method of Calculus of Variations. Secondly, an idealized minimization problem is proposed to remove the above linearity assumption. Such a problem is inspired by a kinetic regularization associated with the Benamou--Brenier formula and universal approximation theorems for neural networks. The proofs of these existence results use variational methods, differential equations, and mean-field optimal control theory. They will stand for a new analytic way to investigate the learning process of deep neural networks.",
  "revisions": [
    {
      "version": "v2",
      "updated": "2023-04-06T04:03:09.000Z"
    }
  ],
  "analyses": {
    "subjects": [
      "49J20",
      "49Q22",
      "68T07",
      "35A35"
    ],
    "keywords": [
      "existence result",
      "variational formulations",
      "deep neural networks",
      "measure-theoretic mean-field optimal control problem",
      "minimization problem"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 33,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}