Adding 3DGS model(s) on demand

tingyus839 commented 4 weeks ago

Hello, thanks for the great library.

I'm wondering if it is possible to add additional models on demand (via API call) instead of having to load all the models in one single function call.

Currently, it seems that calling addSplatScene(), addSplatScenes(), or addSplatBuffers() the second time will cause the model(s) added in the first time be cleared. Is there any way to add new models to the scene while retaining the model(s) added previously?

Comments of addSplatBuffersToMesh() seems promising, but maybe using it directly requires setting up the splat sorting web worker manually like what is done in addSplatBuffers() but without clearing previously added model(s)?

mkkellogg commented 3 weeks ago

Thanks for noticing this, this is definitely a bug. I've made some changes in this branch that should fix the issue: https://github.com/mkkellogg/GaussianSplats3D/tree/general_updates. Would you be willing to test it out to see if it works for you?

tingyus839 commented 3 weeks ago

Hello @mkkellogg, thank you for your timely respond and bug fixes. The updated code does indeed work for me!

However, when adding/removing splat scene(s), there seems to be a split second where the rendered scene is empty even if sceneRevealMode is set to GaussianSplats3D.SceneRevealMode.Instant. I'm wondering if it is possible to prevent the old scene(s) from disappearing for a split second right before adding/removing splat scene(s)?

I think achieving this will definitely improve the user experience, especially when the app is adding models on demand frequently (e.g. in interactive VR apps)

mkkellogg commented 3 weeks ago

Hmm, I didn't notice any momentarily empty scene on my end, can you post the code you are using to perform the scene load and maybe I can debug it?

tingyus839 commented 3 weeks ago

Hello, here is a minimum reproducible example modified from demo/dropin.html. I've also recorded a video deomnstrating the issue:

Minimum reproducible example modified from demo/dropin.html:

<!DOCTYPE html>
<html lang="en">

<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  <meta http-equiv="x-ua-compatible" content="ie=edge">
  <title>3D Gaussian Splats - Drop-in example</title>
  <script type="text/javascript" src="js/util.js"></script>
  <script type="importmap">
    {
        "imports": {
            "three": "./lib/three.module.js",
            "@mkkellogg/gaussian-splats-3d": "./lib/gaussian-splats-3d.module.js"
        }
    }
  </script>
  <style>
    body {
      background-color: #000000;
      height: 100vh;
      margin: 0px;
    }
  </style>

</head>

<body>
  <button id="add-btn">Add Another Model</button>
  <script type="module">
    import * as GaussianSplats3D from '@mkkellogg/gaussian-splats-3d';
    import * as THREE from 'three';

    function setupRenderer() {
      const renderWidth = 800;
      const renderHeight = 600;

      const rootElement = document.createElement('div');
      rootElement.style.width = renderWidth + 'px';
      rootElement.style.height = renderHeight + 'px';
      rootElement.style.position = 'relative';
      rootElement.style.left = '50%';
      rootElement.style.top = '50%';
      rootElement.style.transform = 'translate(-50%, -50%)';
      document.body.appendChild(rootElement);

      const renderer = new THREE.WebGLRenderer({
        antialias: false,
      });
      renderer.setSize(renderWidth, renderHeight);
      rootElement.appendChild(renderer.domElement);

      return {
        'renderer': renderer,
        'renderWidth': renderWidth,
        'renderHeight': renderHeight
      }
    }

    function setupCamera(renderWidth, renderHeight) {
      const camera = new THREE.PerspectiveCamera(65, renderWidth / renderHeight, 0.1, 500);
      camera.position.copy(new THREE.Vector3().fromArray([-1, -4, 6]));
      camera.lookAt(new THREE.Vector3().fromArray([0, 4, -0]));
      camera.up = new THREE.Vector3().fromArray([0, -1, -0.6]).normalize();
      return camera;
    }

    function setupThreeScene() {
      const threeScene = new THREE.Scene();
      const boxColor = 0xBBBBBB;
      const boxGeometry = new THREE.BoxGeometry(2, 2, 2);
      const boxMesh = new THREE.Mesh(boxGeometry, new THREE.MeshBasicMaterial({ 'color': boxColor }));
      threeScene.add(boxMesh);
      boxMesh.position.set(3, 2, 2);
      return threeScene;
    }

    function setupControls(camera, renderer) {
      const controls = new GaussianSplats3D.OrbitControls(camera, renderer.domElement);
      controls.rotateSpeed = 0.5;
      controls.maxPolarAngle = Math.PI * .75;
      controls.minPolarAngle = 0.1;
      controls.enableDamping = true;
      controls.dampingFactor = 0.05;
      return controls;
    }

    const { renderer, renderWidth, renderHeight } = setupRenderer();
    const camera = setupCamera(renderWidth, renderHeight);
    const threeScene = setupThreeScene();
    const controls = setupControls(camera, renderer);

    const viewer = new GaussianSplats3D.DropInViewer({
      gpuAcceleratedSort: false,
      sharedMemoryForWorkers: false,
      selfDrivenMode: false,
      dynamicScene: true,
      sceneRevealMode: GaussianSplats3D.SceneRevealMode.Instant,
      logLevel: GaussianSplats3D.LogLevel.Debug
    });
    viewer.addSplatScenes([
      {
        'path': 'assets/drums.ply',
        'splatAlphaRemovalThreshold': 20,
      },
    ], true);
    threeScene.add(viewer);

    requestAnimationFrame(update);
    function update() {
      requestAnimationFrame(update);
      controls.update();
      renderer.render(threeScene, camera);
    }

    const btn = document.querySelector("#add-btn")
    btn.addEventListener('click', e => {
      console.log("Add")
      viewer.addSplatScenes([
        {
          'path': 'assets/lego.ply',
          'position': [0, 1, 0],
          'splatAlphaRemovalThreshold': 20,
        }
      ], true);
    })

  </script>
</body>

</html>

mkkellogg commented 3 weeks ago

Ah, you're not running in selfDrivenMode, so the viewer has no control over the rendering. Whenever the splat data changes (removing or adding splat scenes), rendering should really always be paused because the splat data textures need to be updated as well as several internal data structures including the splatmesh's octree and the sort worker's data buffers. That all takes some time, and if you render during that time you'll see artifacts or a blank screen.

If you change your rendering logic to something like the example below, it should help. Also make sure the grab the latest from that branch, I had to add some tweaks to make this example work correctly :)

let renderPause = false;
requestAnimationFrame(update);
function update() {
  requestAnimationFrame(update);
  if (!renderPause) {
    controls.update();
    renderer.render(threeScene, camera);
  }
}

const btn = document.querySelector("#add-btn")
btn.addEventListener('click', e => {
  console.log("Add")
  renderPause = true;
  viewer.addSplatScenes([
    {
      'path': 'assets/data/bonsai/bonsai_trimmed.ksplat',
      'position': [1, -1, 0],
      'splatAlphaRemovalThreshold': 20,
    }
  ], true).then(() => {
    renderPause = false;
  })
})

tingyus839 commented 3 weeks ago

Thanks for your advice and updates. It is now working perfectly for me!

mkkellogg / GaussianSplats3D

Adding 3DGS model(s) on demand #243