{
  "id": "2212.12952",
  "version": "v1",
  "published": "2022-12-25T19:56:21.000Z",
  "updated": "2022-12-25T19:56:21.000Z",
  "title": "Neural Shape Compiler: A Unified Framework for Transforming between Text, Point Cloud, and Program",
  "authors": [
    "Tiange Luo",
    "Honglak Lee",
    "Justin Johnson"
  ],
  "comment": "project page: https://tiangeluo.github.io/projectpages/shapecompiler.html",
  "categories": [
    "cs.CV",
    "cs.AI"
  ],
  "abstract": "3D shapes have complementary abstractions from low-level geometry to part-based hierarchies to languages, which convey different levels of information. This paper presents a unified framework to translate between pairs of shape abstractions: $\\textit{Text}$ $\\Longleftrightarrow$ $\\textit{Point Cloud}$ $\\Longleftrightarrow$ $\\textit{Program}$. We propose $\\textbf{Neural Shape Compiler}$ to model the abstraction transformation as a conditional generation process. It converts 3D shapes of three abstract types into unified discrete shape code, transforms each shape code into code of other abstract types through the proposed $\\textit{ShapeCode Transformer}$, and decodes them to output the target shape abstraction. Point Cloud code is obtained in a class-agnostic way by the proposed $\\textit{Point}$VQVAE. On Text2Shape, ShapeGlot, ABO, Genre, and Program Synthetic datasets, Neural Shape Compiler shows strengths in $\\textit{Text}$ $\\Longrightarrow$ $\\textit{Point Cloud}$, $\\textit{Point Cloud}$ $\\Longrightarrow$ $\\textit{Text}$, $\\textit{Point Cloud}$ $\\Longrightarrow$ $\\textit{Program}$, and Point Cloud Completion tasks. Additionally, Neural Shape Compiler benefits from jointly training on all heterogeneous data and tasks.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2022-12-25T19:56:21.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "unified framework",
      "neural shape compiler benefits",
      "point cloud completion tasks",
      "abstract types",
      "conditional generation process"
    ],
    "tags": [
      "github project"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}