{
  "id": "2210.14721",
  "version": "v1",
  "published": "2022-10-25T17:50:36.000Z",
  "updated": "2022-10-25T17:50:36.000Z",
  "title": "Sim-to-Real via Sim-to-Seg: End-to-end Off-road Autonomous Driving Without Real Data",
  "authors": [
    "John So",
    "Amber Xie",
    "Sunggoo Jung",
    "Jeffrey Edlund",
    "Rohan Thakker",
    "Ali Agha-mohammadi",
    "Pieter Abbeel",
    "Stephen James"
  ],
  "comment": "CoRL 2022 Paper",
  "categories": [
    "cs.LG",
    "cs.AI"
  ],
  "abstract": "Autonomous driving is complex, requiring sophisticated 3D scene understanding, localization, mapping, and control. Rather than explicitly modelling and fusing each of these components, we instead consider an end-to-end approach via reinforcement learning (RL). However, collecting exploration driving data in the real world is impractical and dangerous. While training in simulation and deploying visual sim-to-real techniques has worked well for robot manipulation, deploying beyond controlled workspace viewpoints remains a challenge. In this paper, we address this challenge by presenting Sim2Seg, a re-imagining of RCAN that crosses the visual reality gap for off-road autonomous driving, without using any real-world data. This is done by learning to translate randomized simulation images into simulated segmentation and depth maps, subsequently enabling real-world images to also be translated. This allows us to train an end-to-end RL policy in simulation, and directly deploy in the real-world. Our approach, which can be trained in 48 hours on 1 GPU, can perform equally as well as a classical perception and control stack that took thousands of engineering hours over several months to build. We hope this work motivates future end-to-end autonomous driving research.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2022-10-25T17:50:36.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "end-to-end off-road autonomous driving",
      "real data",
      "sophisticated 3d scene understanding",
      "sim-to-seg",
      "end-to-end rl policy"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}