Renderer returns empty image with batch size = 3

benjiebob commented 5 years ago

Hi there, I happened across quite a scary bug (I think) with this library today. As far as I can tell, the renderer seems to return empty images when provided vertices, faces with batch dimension = 3.

To test, I cloned and built the library today (20/10/2018) and produced a demo which I hope will enable you to reproduce the problem:

import matplotlib.pyplot as plt
import numpy as np

import torch
import torch.optim as optim

import trimesh
import neural_renderer as nr

IMAGE_SIZE = 256
TEXTURE_SIZE = 4

def run_renderer_with_batch_size(verts, faces, textures, renderer, batch_size):
    # We just duplicate (using pytorch expand) the cube (i.e. duplicate verts, faces and textures) 'batch_size' times
    verts = verts.expand(batch_size, -1, 3)
    faces = faces.expand(batch_size, -1, 3)
    textures = textures.expand(batch_size, faces.shape[1], TEXTURE_SIZE, TEXTURE_SIZE, TEXTURE_SIZE, 3)

    # Run the renderer and the silhouette renderer
    rendered_images = renderer.render(verts, faces, textures)
    rendered_silhouettes = renderer.render_silhouettes(verts, faces)

    return rendered_images, rendered_silhouettes

def main():
    # Initialize the renderer
    renderer = nr.Renderer(camera_mode='look_at')
    renderer.eye = nr.get_points_from_angles(5.0, 45.0, 45.0)
    renderer.image_size = IMAGE_SIZE

    # Create a 2x2x2 cube
    box_verts = np.array([
        [-1.0, -1.0, -1.0],
        [-1.0, -1.0,  1.0],
        [-1.0,  1.0, -1.0],
        [-1.0,  1.0,  1.0],
        [ 1.0, -1.0, -1.0],
        [ 1.0, -1.0,  1.0],
        [ 1.0,  1.0, -1.0],
        [ 1.0,  1.0,  1.0]])

    box_faces = np.array([
        [1, 3, 0],
        [4, 1, 0],
        [0, 3, 2],
        [2, 4, 0],
        [1, 7, 3],
        [5, 1, 4],
        [5, 7, 1],
        [3, 7, 2],
        [6, 4, 2],
        [2, 7, 6],
        [6, 5, 4],
        [7, 5, 6]])

    # Add a batch dimension, initially = 1
    verts = torch.from_numpy(box_verts[None, :]).float().cuda()
    faces = torch.from_numpy(box_faces[None, :]).int().cuda()

    # Create some pretty textures, again with a batch dimension = 1
    textures = torch.ones(1, box_faces.shape[0], TEXTURE_SIZE, TEXTURE_SIZE, TEXTURE_SIZE, 3).float().cuda()
    textures = torch.tanh(textures)

    # Test the renderer with various batch sizes
    for batch_size in range(1, 100):
        rendered_images, rendered_silhouettes = run_renderer_with_batch_size(verts, faces, textures, renderer, batch_size)

        # Print to the terminal what the largest pixel value is over the whole batch
        # nr.render() should produce tanh(1.0) = 0.7615...
        # nr.render_silhouettes() should produce 1.0

        print ("Batch size: {0}, nr.render(): {1}, nr.render_silhouettes(.): {2}".format(
            batch_size,
            rendered_images.max(),
            rendered_silhouettes.max()))

        # Visualize the first entry in the batch
        images_vis = rendered_images.permute(0, 2, 3, 1).cpu().numpy()
        images_vis = (images_vis * 255.0).astype(np.uint8)

        silhouette_vis = rendered_silhouettes.cpu().numpy()

        plt.figure()
        plt.suptitle("Batch size: {0}, Max image value across batch: {1}".format(batch_size, rendered_images.max()))
        plt.subplot(121)
        plt.imshow(images_vis[0])
        plt.subplot(122)
        plt.imshow(silhouette_vis[0])
        plt.show()

if __name__ == '__main__':
    main()

This produces output

Batch size: 1, nr.render(): 0.761594295502, nr.render_silhouettes(.): 1.0
Batch size: 2, nr.render(): 0.761594295502, nr.render_silhouettes(.): 1.0
Batch size: 3, nr.render(): 0.0, nr.render_silhouettes(.): 0.0
Batch size: 4, nr.render(): 0.761594295502, nr.render_silhouettes(.): 1.0
Batch size: 5, nr.render(): 0.761594295502, nr.render_silhouettes(.): 1.0
Batch size: 6, nr.render(): 0.761594295502, nr.render_silhouettes(.): 1.0
Batch size: 7, nr.render(): 0.761594295502, nr.render_silhouettes(.): 1.0
Batch size: 8, nr.render(): 0.761594295502, nr.render_silhouettes(.): 1.0
Batch size: 9, nr.render(): 0.761594295502, nr.render_silhouettes(.): 1.0
...
Batch size: 100, nr.render(): 0.761594295502, nr.render_silhouettes(.): 1.0

Batch size = 3 seems to have a problem.

batch_size 2 batch_size 3 batch_size 4