Black Screen on Integrating the Advanced Viewer with Threejs Code

[cyborgsince90s]

Hi There,

I am trying to integrate the Advanced viewer code with my own Scene, Camera, Orbit Control and Renderer. But I am seeing only a Black screen, I tested with a Cube, The Cube gets loaded but as soon as the viewer is loaded, The Cube is covered by the Black screen.

Attaching the Scripts for your reference,

Hope to get this fixed or resolved.

-----Scene.js to setup the scene--------

import * as THREE from './threejs/src/Three.js';
import { OrbitControls } from '../threejs/controls/OrbitControls.js';

export const setCamera = () => {
   const camera = new THREE.PerspectiveCamera(65, window.innerWidth / window.innerHeight, 0.1, 500);
   camera.position.set(0,0,0);
    return camera;
}

export const setScene = () => {
    const scene = new THREE.Scene();
    return scene;
}

export const setControls = (camera, renderer) => {
    const controls = new OrbitControls(camera, renderer.domElement);

    controls.enableZoom = true;
    controls.maxPolarAngle =  Math.PI / 2;
    //controls.minPolarAngle = Math.PI / 3;
    controls.enableDamping = true;
    controls.enablePan = false;
    controls.dampingFactor = 0.1;
    controls.autoRotate = true; // Toggle  to automatically rotate
    controls.autoRotateSpeed = 0.05; // 30

    controls.minDistance = 1;
    controls.maxDistance = 100;

    controls.target.set(3, 2, 2);

    controls.update();
    return controls;
}

export const setRenderer = () => {

    let canvas = document.querySelector("#threejs-canvas");
    const renderer = new THREE.WebGLRenderer({ canvas, antialias: true });
    renderer.setSize(window.innerWidth, window.innerHeight);    

    return renderer;
}

export const setLights = (scene) => {
    const dirLight = new THREE.DirectionalLight(0xffffff, .5, 100);
    dirLight.position.set(-1, 2, 4);
    scene.add(dirLight);

}

-----Main.js where the main work happens-----

import * as THREE from './threejs/src/Three.js';
import { RGBELoader } from "./threejs/loaders/RGBELoader.js";

import * as GaussianSplats3D from './gaussian-splats-3d.module.js';

import {
    setCamera,
    setScene,
    setControls,
    setRenderer,
    setLights,
} from "./scene.js";

let clock;
let camera, scene, renderer, controls;
let viewer;

function onWindowResize() {
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
renderer.setSize(window.innerWidth, window.innerHeight);
}

function init() {
    camera = setCamera();
    camera.position.set(2, 2, 10); // Adjust camera position
    camera.up.set(0, -1, 0)
    scene = setScene();
    setLights(scene);

    renderer = setRenderer();
    controls = setControls(camera, renderer);
    window.scene = scene;
    clock = new THREE.Clock();

    window.addEventListener("resize", onWindowResize, false);   

    const boxColor = 0xBBBBBB;
    const boxGeometry = new THREE.BoxGeometry(2, 2, 2);
    const boxMesh = new THREE.Mesh(boxGeometry, new THREE.MeshBasicMaterial({'color': boxColor}));
    boxMesh.position.set(3, 2, 2);
    scene.add(boxMesh);

    viewer = new GaussianSplats3D.Viewer({
        'selfDrivenMode': false,
        'renderer': renderer,
        'camera': camera,
        'useBuiltInControls': false,
        'ignoreDevicePixelRatio': false,
        'gpuAcceleratedSort': true,
        'enableSIMDInSort': true,
        'sharedMemoryForWorkers': true,
        'integerBasedSort': true,
        'halfPrecisionCovariancesOnGPU': true,
        'dynamicScene': false,
        'webXRMode': GaussianSplats3D.WebXRMode.None,
        'renderMode': GaussianSplats3D.RenderMode.OnChange,
        'sceneRevealMode': GaussianSplats3D.SceneRevealMode.Instant,
        'antialiased': false,
        'focalAdjustment': 1.0,
        'logLevel': GaussianSplats3D.LogLevel.None,
        'sphericalHarmonicsDegree': 0,
        'enableOptionalEffects': false,
        'inMemoryCompressionLevel': 2,
        'freeIntermediateSplatData': false
    });
    viewer.addSplatScene('bike_1.ply')
        .then(() => {            
            console.log('Splat scene loaded successfully!');
            requestAnimationFrame(animate);
        })
        .catch(err => {
            console.error('Error loading splat scene:', err);
        });        

}

function animate() {
    requestAnimationFrame(animate);
    if(viewer){
        viewer.update();
        viewer.render();
    }
    else{
        console.log("Viewer not available");
    }  
    renderer.setAnimationLoop(render);
}

function render(time) {
    time *= 0.001; // Time in seconds
    renderer.render(scene, camera);
}

init();
animate();

By the way even though I am using Node, I am using the installed packages separately out of the node_modules folder.

Thanks,

[mkkellogg]

My first question is: why are you using two different animation loops? You're using both requestAnimationFrame() and renderer.setAnimationLoop. Also, in order to render your three.js scene along with the splats, the splats viewer needs to know about it and handle its rendering along with rendering the splats.

You need to pass the three.js to the viewer when you create it:

viewer = new GaussianSplats3D.Viewer({
  'threeScene': scene,
  'selfDrivenMode': false,
  'renderer': renderer,
  'camera': camera,
  'useBuiltInControls': false,
  'ignoreDevicePixelRatio': false,
  'gpuAcceleratedSort': true,
  'enableSIMDInSort': true,
  'sharedMemoryForWorkers': true,
  'integerBasedSort': true,
  'halfPrecisionCovariancesOnGPU': true,
  'dynamicScene': false,
  'webXRMode': GaussianSplats3D.WebXRMode.None,
  'renderMode': GaussianSplats3D.RenderMode.OnChange,
  'sceneRevealMode': GaussianSplats3D.SceneRevealMode.Instant,
  'antialiased': false,
  'focalAdjustment': 1.0,
  'logLevel': GaussianSplats3D.LogLevel.None,
  'sphericalHarmonicsDegree': 0,
  'enableOptionalEffects': false,
  'inMemoryCompressionLevel': 2,
  'freeIntermediateSplatData': false
});

Your animation loop should look something like:

function animate() {
    requestAnimationFrame(animate);
    if(viewer){
        viewer.update();
        viewer.render();
    }
}

Then you wouldn't even need to call renderer.render()

[cyborgsince90s]

Thank you for your response, I can now see a loading indicator for the Splat and See my cube once it reaches 100%, But I am still not able to see the Loaded splat (No matter how zoomed in/out I went). I can see the debug log for the Splat, Attaching it as an image.

Any way to identify the Position of the loaded splat?

Thanks in-advance.

[hybridherbst]

I believe I'm seeing the same issue.

[mkkellogg]

Sorry for the late reply, I was able to replicate your issue locally, and when I set gpuAcceleratedSort to false, that seemed to fix it. Looks like I need to fix that feature, but FWIW even when gpuAcceleratedSort works correctly, the viewer generally works better when gpuAcceleratedSort is disabled.

[hybridherbst]

Thanks! For me it still stays black no matter if gpuAcceleratedSort is false or true, but I'm using the DropInViewer inside an existing three.js project.

I get the same log as https://github.com/mkkellogg/GaussianSplats3D/issues/345#issuecomment-2408272415 and then nothing is displayed. In Spector.js, I can see that the appropriate draw calls are made (one instanced draw call with the right number of vertices), but they result in nothing being rendered.

Any idea how to further debug this?

[mkkellogg]

@hybridherbst Would you be willing to share the relevant code snippet? It might be easier if I try to debug from my side :)

[cyborgsince90s]

Awesome, It's working. Thank you so much

On Fri, Oct 11, 2024 at 9:45 AM Mark Kellogg @.***> wrote:

My first question is: why are you using two different animation loops? You;re using both requestAnimationFrame() and renderer.setAnimationLoop. Also, in order to render your three.js scene along with the splats, the splats viewer needs to know about it and handle its rendering along with rendering the splats.

You need to pass the three.js to the viewer when you create it:

viewer = new GaussianSplats3D.Viewer({ 'threeScene': scene, 'selfDrivenMode': false, 'renderer': renderer, 'camera': camera, 'useBuiltInControls': false, 'ignoreDevicePixelRatio': false, 'gpuAcceleratedSort': true, 'enableSIMDInSort': true, 'sharedMemoryForWorkers': true, 'integerBasedSort': true, 'halfPrecisionCovariancesOnGPU': true, 'dynamicScene': false, 'webXRMode': GaussianSplats3D.WebXRMode.None, 'renderMode': GaussianSplats3D.RenderMode.OnChange, 'sceneRevealMode': GaussianSplats3D.SceneRevealMode.Instant, 'antialiased': false, 'focalAdjustment': 1.0, 'logLevel': GaussianSplats3D.LogLevel.None, 'sphericalHarmonicsDegree': 0, 'enableOptionalEffects': false, 'inMemoryCompressionLevel': 2, 'freeIntermediateSplatData': false});

Your animation loop should look something like:

function animate() { requestAnimationFrame(animate); if(viewer){ viewer.update(); viewer.render(); }}

— Reply to this email directly, view it on GitHub https://github.com/mkkellogg/GaussianSplats3D/issues/345#issuecomment-2406522484, or unsubscribe https://github.com/notifications/unsubscribe-auth/AFSZB3KGZRX2W2XPXPI5NUDZ25GHJAVCNFSM6AAAAABPG4SU4GVHI2DSMVQWIX3LMV43OSLTON2WKQ3PNVWWK3TUHMZDIMBWGUZDENBYGQ . You are receiving this because you authored the thread.Message ID: @.***>

-- Suraj K M, 3D/XR & GenAI Developer, India

[hybridherbst]

@mkkellogg I'll try to get a repro together in the next days and let you know. Thanks!

[hybridherbst]

@mkkellogg I made a new issue with reproduction steps since it seems it's a different root cause, would be awesome if you can take a look:

• https://github.com/mkkellogg/GaussianSplats3D/issues/357