{
  "id": "2210.11348",
  "version": "v1",
  "published": "2022-10-20T15:34:52.000Z",
  "updated": "2022-10-20T15:34:52.000Z",
  "title": "Hypernetworks in Meta-Reinforcement Learning",
  "authors": [
    "Jacob Beck",
    "Matthew Thomas Jackson",
    "Risto Vuorio",
    "Shimon Whiteson"
  ],
  "comment": "Published at CoRL 2022",
  "categories": [
    "cs.LG",
    "cs.AI",
    "cs.RO"
  ],
  "abstract": "Training a reinforcement learning (RL) agent on a real-world robotics task remains generally impractical due to sample inefficiency. Multi-task RL and meta-RL aim to improve sample efficiency by generalizing over a distribution of related tasks. However, doing so is difficult in practice: In multi-task RL, state of the art methods often fail to outperform a degenerate solution that simply learns each task separately. Hypernetworks are a promising path forward since they replicate the separate policies of the degenerate solution while also allowing for generalization across tasks, and are applicable to meta-RL. However, evidence from supervised learning suggests hypernetwork performance is highly sensitive to the initialization. In this paper, we 1) show that hypernetwork initialization is also a critical factor in meta-RL, and that naive initializations yield poor performance; 2) propose a novel hypernetwork initialization scheme that matches or exceeds the performance of a state-of-the-art approach proposed for supervised settings, as well as being simpler and more general; and 3) use this method to show that hypernetworks can improve performance in meta-RL by evaluating on multiple simulated robotics benchmarks.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2022-10-20T15:34:52.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "meta-reinforcement learning",
      "degenerate solution",
      "novel hypernetwork initialization scheme",
      "real-world robotics task remains",
      "naive initializations yield poor performance"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}