Convert weakperspective camera to perspective camera

[Darius-H]

I'm trying to run https://github.com/open-mmlab/mmhuman3d/blob/main/demo/pymafx_estimate_smplx.py to get smplx params and perspective camera matrices. I tried to extract the weakperspective camera matrices R,T,K from the code here https://github.com/open-mmlab/mmhuman3d/blob/9431addec32f7fbeffa1786927a854c0ab79d9ea/mmhuman3d/core/visualization/visualize_smpl.py#L998C1-L1007C27 and covert them into perspective camera matrices. Those matrices works well, as I'm running the following code:

from mmhuman3d.core.renderer.torch3d_renderer.smpl_renderer import SMPLRenderer
from mmhuman3d.core.renderer.torch3d_renderer.utils import align_input_to_padded  
from mmhuman3d.core.renderer.torch3d_renderer.render_smpl_config import (
    RENDER_CONFIGS,
)
from mmhuman3d.core.cameras.builder import build_cameras
from mmhuman3d.core.renderer.torch3d_renderer import render_runner
from mmhuman3d.core.renderer.torch3d_renderer.meshes import ParametricMeshes
import copy

output_path="debug_out"

render_choice='hq'
render_param_dict = copy.deepcopy(RENDER_CONFIGS[render_choice.lower()])
render_param_dict.update(num_class=27)
projection = 'weakperspective'
# initialize the renderer.
renderer = SMPLRenderer(
    resolution=render_resolution,
    device=device,
    output_path=output_path,
    return_tensor=True,# 这个设置为True可以得到image
    alpha=1.0,
    read_img_format='%06d.png',
    render_choice=render_choice,
    frames_folder= None,
    plot_kps=False,
    vis_kp_index=False,
    final_resolution=render_resolution,
    **render_param_dict)

cameras = build_cameras(# 这边还是weakperspective camera
    dict(
        type=projection,
        in_ndc=True,
        device=device,
        K=K,
        R=R,
        T=T,
        resolution=render_resolution))

image_array = np.array(input_img)
if image_array is not None:
    image_array = torch.Tensor(image_array)
    image_array = align_input_to_padded(
        image_array, ndim=4, batch_size=num_frames, padding_mode='ones')
# prepare the render data.
meshes = ParametricMeshes(verts=vertices.squeeze(1),faces=torch.Tensor(smplx_model.faces.astype(np.int32)).to(device),model_type='smplx')
render_data = dict(
    images=image_array,
    meshes=meshes,
    cameras=cameras,
    joints=None,
    joints_gt=None,
)

results = render_runner.render(
    renderer=renderer,
    device=device,
    batch_size=1,
    output_path=output_path,
    return_tensor=True, # 这个设置为True可以得到image
    no_grad=True,
    verbose=True,
    **render_data)

plt.imshow(input_img+results.cpu()[0,...,:3].numpy())

And output a good result like this:

Then I follow the tutorial here https://github.com/open-mmlab/mmhuman3d/blob/main/docs_zh-CN/cameras.md to change intrinsic matrix K to perspective camera, i write the following code:

from mmhuman3d.core.conventions.cameras.convert_projection import (
    convert_perspective_to_weakperspective,
    convert_weakperspective_to_perspective)
print(vertices.shape)
# zmean = vertices[...,2].mean().unsqueeze(dim=0)
points_depth = cameras.compute_depth_of_points(vertices[0])
zmean = points_depth.mean().unsqueeze(dim=0)
print(zmean.shape,zmean)

K=K.to(device)
pK = convert_weakperspective_to_perspective(
    K, zmean, in_ndc=Ture, resolution=render_resolution, convention='pytorch3d')

pcameras = build_cameras(
    dict(
        type='Perspective',
        in_ndc=True,
        device=device,
        K=pK,
        R=R,
        T=T,
        resolution=render_resolution))

render_data = dict(
    images=image_array,
    meshes=meshes,
    cameras=pcameras,
    joints=None,
    joints_gt=None,
)

results = render_runner.render(
    renderer=renderer,
    device=device,
    batch_size=1,
    output_path=output_path,
    return_tensor=True, # 这个设置为True可以得到image
    no_grad=True,
    verbose=True,
    **render_data)

plt.imshow(input_img+results.cpu()[0,...,:3].numpy())

However the output image is not well aligned:

Can you give me some advice to generate perspective camera matrices that can align with the image?

[Dipankar1997161]

I have small function for myself which does the conversion. Check if it helps.

Adjust certain values accordingly, such as Focal and img_res

def convert_weak_perspective_to_perspective(
weak_perspective_camera,
focal_length=5000.,
img_res=224,
):
# Convert Weak Perspective Camera [s, tx, ty] to camera translation [tx, ty, tz]
# in 3D given the bounding box size
# This camera translation can be used in a full-perspective projection

perspective_camera = np.stack(
    [
        weak_perspective_camera[1],
        weak_perspective_camera[2],
        2 * focal_length / (img_res * weak_perspective_camera[0] + 1e-9)
    ],
    axis=-1
)

return perspective_camera