{
  "id": "2302.01330",
  "version": "v1",
  "published": "2023-02-02T18:59:16.000Z",
  "updated": "2023-02-02T18:59:16.000Z",
  "title": "SceneDreamer: Unbounded 3D Scene Generation from 2D Image Collections",
  "authors": [
    "Zhaoxi Chen",
    "Guangcong Wang",
    "Ziwei Liu"
  ],
  "comment": "Project Page https://scene-dreamer.github.io/",
  "categories": [
    "cs.CV",
    "cs.GR"
  ],
  "abstract": "In this work, we present SceneDreamer, an unconditional generative model for unbounded 3D scenes, which synthesizes large-scale 3D landscapes from random noises. Our framework is learned from in-the-wild 2D image collections only, without any 3D annotations. At the core of SceneDreamer is a principled learning paradigm comprising 1) an efficient yet expressive 3D scene representation, 2) a generative scene parameterization, and 3) an effective renderer that can leverage the knowledge from 2D images. Our framework starts from an efficient bird's-eye-view (BEV) representation generated from simplex noise, which consists of a height field and a semantic field. The height field represents the surface elevation of 3D scenes, while the semantic field provides detailed scene semantics. This BEV scene representation enables 1) representing a 3D scene with quadratic complexity, 2) disentangled geometry and semantics, and 3) efficient training. Furthermore, we propose a novel generative neural hash grid to parameterize the latent space given 3D positions and the scene semantics, which aims to encode generalizable features across scenes. Lastly, a neural volumetric renderer, learned from 2D image collections through adversarial training, is employed to produce photorealistic images. Extensive experiments demonstrate the effectiveness of SceneDreamer and superiority over state-of-the-art methods in generating vivid yet diverse unbounded 3D worlds.",
  "revisions": [
    {
      "version": "v1",
      "updated": "2023-02-02T18:59:16.000Z"
    }
  ],
  "analyses": {
    "keywords": [
      "2d image collections",
      "unbounded 3d scene generation",
      "scenedreamer",
      "novel generative neural hash grid",
      "bev scene representation enables"
    ],
    "tags": [
      "github project"
    ],
    "note": {
      "typesetting": "TeX",
      "pages": 0,
      "language": "en",
      "license": "arXiv",
      "status": "editable"
    }
  }
}