{
  "id": "2401.01270",
  "version": "v1",
  "published": "2024-01-02T16:14:35.000Z",
  "updated": "2024-01-02T16:14:35.000Z",
  "title": "Optimal Rates of Kernel Ridge Regression under Source Condition in Large Dimensions",
  "authors": [
    "Haobo Zhang",
    "Yicheng Li",
    "Weihao Lu",
    "Qian Lin"
  ],
  "comment": "61 pages, 11 figures",
  "categories": [
    "cs.LG"
  ],
  "abstract": "Motivated by the studies of neural networks (e.g.,the neural tangent kernel theory), we perform a study on the large-dimensional behavior of kernel ridge regression (KRR) where the sample size $n \\asymp d^{\\gamma}$ for some $\\gamma > 0$. Given an RKHS $\\mathcal{H}$ associated with an inner product kernel defined on the sphere $\\mathbb{S}^{d}$, we suppose that the true function $f_{\\rho}^{*} \\in [\\mathcal{H}]^{s}$, the interpolation space of $\\mathcal{H}$ with source condition $s>0$. We first determined the exact order (both upper and lower bound) of the generalization error of kernel ridge regression for the optimally chosen regularization parameter $\\lambda$. We then further showed that when $0<s\\le1$, KRR is minimax optimal; and when $s>1$, KRR is not minimax optimal (a.k.a. he saturation effect). Our results illustrate that the curves of rate varying along $\\gamma$ exhibit the periodic plateau behavior and the multiple descent behavior and show how the curves evolve with $s>0$. Interestingly, our work provides a unified viewpoint of several recent works on kernel regression in the large-dimensional setting, which correspond to $s=0$ and $s=1$ respectively.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2024-01-02T16:14:35.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "kernel ridge regression",
      "source condition",
      "optimal rates",
      "large dimensions",
      "neural tangent kernel theory"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 61,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}