Colormap optimization sanity check failed

[yd-yin]

I did a sanity check on colormap optimization both with default demo data (fountain_small) and our data. It suceeded on demo data but failed on our data. The experiment is: only one RGB-D pair, identity matrix as extrinsic -> mesh reconstruction by TSDFVolume -> colormap optimization

Notice that it is related to issue #823 @jeongjoonpark @syncle @abuzaina

I upload demo and our data here. colormap_optim.zip

import open3d as o3d
import numpy as np
import os

def fuse(rgbd, intrinsic, extrinsic):
    tsdf = o3d.integration.ScalableTSDFVolume(
        voxel_length=4.0 / 512.0,
        sdf_trunc=0.04,
        color_type=o3d.integration.TSDFVolumeColorType.RGB8
    )

    tsdf.integrate(rgbd, intrinsic, extrinsic)
    mesh = tsdf.extract_triangle_mesh()
    o3d.io.write_triangle_mesh(os.path.join(mesh_dir, f'{data}_mesh.obj'), mesh, write_ascii=True)

def colormap_optimize(rgbd, intrinsic, extrinsic):
    mesh = o3d.io.read_triangle_mesh(os.path.join(mesh_dir, f'{data}_mesh.obj'))

    with open(os.path.join(mesh_dir, 'trajectory.log'), 'w') as traj:
        traj.write(f"0 0 0\n"
                   f"{extrinsic[0, 0]} {extrinsic[0, 1]} {extrinsic[0, 2]} {extrinsic[0, 3]}\n"
                   f"{extrinsic[1, 0]} {extrinsic[1, 1]} {extrinsic[1, 2]} {extrinsic[1, 3]}\n"
                   f"{extrinsic[2, 0]} {extrinsic[2, 1]} {extrinsic[2, 2]} {extrinsic[2, 3]}\n"
                   f"{extrinsic[3, 0]} {extrinsic[3, 1]} {extrinsic[3, 2]} {extrinsic[3, 3]}\n"
                   )
    camera = o3d.io.read_pinhole_camera_trajectory(os.path.join(mesh_dir, 'trajectory.log'))

    for para in camera.parameters:
        para.intrinsic = intrinsic

    option = o3d.color_map.ColorMapOptimizationOption()
    # option.non_rigid_camera_coordinate = True
    # option.maximum_iteration = 0
    o3d.color_map.color_map_optimization(mesh, [rgbd], camera, option)
    o3d.io.write_triangle_mesh(os.path.join(mesh_dir, f'{data}_mesh_color_optim.obj'), mesh, write_ascii=True)

def main():
    ours_intrinsic = np.array([[480, 0, 320], [0, 480, 240], [0, 0, 1]])
    ours_img_h, ours_img_w = 480, 640
    colorname = os.path.join(image_dir, f'{data}_color.png')
    color = o3d.io.read_image(colorname)
    depth = o3d.io.read_image(colorname.replace('color.png', 'depth.png'))
    rgbd_parser = o3d.geometry.RGBDImage()
    rgbd = rgbd_parser.create_from_color_and_depth(color, depth, depth_scale=1000, depth_trunc=10, convert_rgb_to_intensity=False)
    if data == 'ours':
        intrinsic = o3d.camera.PinholeCameraIntrinsic(ours_img_w, ours_img_h, ours_intrinsic[0, 0], ours_intrinsic[1, 1], ours_intrinsic[0, 2], ours_intrinsic[1, 2])
    else:
        intrinsic = o3d.camera.PinholeCameraIntrinsic(o3d.camera.PinholeCameraIntrinsicParameters.PrimeSenseDefault)
    extrinsic = np.eye(4, dtype=np.float64)

    fuse(rgbd, intrinsic, extrinsic)
    colormap_optimize(rgbd, intrinsic, extrinsic)

if __name__ == '__main__':
    root = '/home/wsy/Data/ydyin/proactive/colormap_optim'
    mesh_dir = os.path.join(root, 'mesh')
    image_dir = os.path.join(root, 'images')

    target = 'colormap_optim'
    # data = 'demo'
    data = 'ours'

    main()

Left: reconstruction results, right: color optimization results

[griegler]

@yxlao can you have a look at this please

[yxlao]

It succeeded on demo data but failed on our data.

That means that the sanity check succeeds. The parameter we used is only suitable for the demo data. For a different dataset, the optimal parameters are likely to be different.

I would recommend the following:

• changing colormap optimization parameters
• modify the code to obtain intermediate results, and render it for debugging