{
  "id": "2106.08981",
  "version": "v1",
  "published": "2021-06-16T17:21:59.000Z",
  "updated": "2021-06-16T17:21:59.000Z",
  "title": "Nonequilibrium thermodynamics of self-supervised learning",
  "authors": [
    "Domingos S. P. Salazar"
  ],
  "comment": "6 pages, 1 figure",
  "categories": [
    "cond-mat.stat-mech",
    "cs.AI",
    "cs.LG"
  ],
  "abstract": "Self-supervised learning (SSL) of energy based models has an intuitive relation to equilibrium thermodynamics because the softmax layer, mapping energies to probabilities, is a Gibbs distribution. However, in what way SSL is a thermodynamic process? We show that some SSL paradigms behave as a thermodynamic composite system formed by representations and self-labels in contact with a nonequilibrium reservoir. Moreover, this system is subjected to usual thermodynamic cycles, such as adiabatic expansion and isochoric heating, resulting in a generalized Gibbs ensemble (GGE). In this picture, we show that learning is seen as a demon that operates in cycles using feedback measurements to extract negative work from the system. As applications, we examine some SSL algorithms using this idea.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2021-06-16T17:21:59.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "nonequilibrium thermodynamics",
      "self-supervised learning",
      "usual thermodynamic cycles",
      "ssl paradigms behave",
      "thermodynamic composite system"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 6,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}