{
  "id": "2207.08406",
  "version": "v1",
  "published": "2022-07-18T06:40:46.000Z",
  "updated": "2022-07-18T06:40:46.000Z",
  "title": "Kullback-Leibler and Renyi divergences in reproducing kernel Hilbert space and Gaussian process settings",
  "authors": [
    "Minh Ha Quang"
  ],
  "comment": "74 pages",
  "categories": [
    "stat.ML",
    "cs.LG"
  ],
  "abstract": "In this work, we present formulations for regularized Kullback-Leibler and R\\'enyi divergences via the Alpha Log-Determinant (Log-Det) divergences between positive Hilbert-Schmidt operators on Hilbert spaces in two different settings, namely (i) covariance operators and Gaussian measures defined on reproducing kernel Hilbert spaces (RKHS); and (ii) Gaussian processes with squared integrable sample paths. For characteristic kernels, the first setting leads to divergences between arbitrary Borel probability measures on a complete, separable metric space. We show that the Alpha Log-Det divergences are continuous in the Hilbert-Schmidt norm, which enables us to apply laws of large numbers for Hilbert space-valued random variables. As a consequence of this, we show that, in both settings, the infinite-dimensional divergences can be consistently and efficiently estimated from their finite-dimensional versions, using finite-dimensional Gram matrices/Gaussian measures and finite sample data, with {\\it dimension-independent} sample complexities in all cases. RKHS methodology plays a central role in the theoretical analysis in both settings. The mathematical formulation is illustrated by numerical experiments.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2022-07-18T06:40:46.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "reproducing kernel hilbert space",
      "gaussian process settings",
      "renyi divergences",
      "kullback-leibler",
      "arbitrary borel probability measures"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 74,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}