Regarding the projection of point clouds onto images

[LoveMYChen]

Hello, may I ask if you have added code to project point clouds onto corresponding images based on true and predicted values？

[martin-liao]

Please refer to code. You could modify the code according to your own needs

[LoveMYChen]

I modified the code in train.exe to project the point cloud onto the image and display it, but there were only a few points and all the projected points were concentrated in the upper left corner, which did not achieve the same effect as the legend in your paper. Do you know how to solve this problem？

[martin-liao]

Show me your projection code, please.

[LoveMYChen]

def project_and_display(pc_points, T, K, img):

将点云从齐次坐标变换到相机坐标系

pc_points_h = np.hstack((pc_points, np.ones((pc_points.shape[0], 1))))
pc_cam = np.dot(T, pc_points_h.T).T[:, :3]

# 使用相机内参进行投影
pc_img = np.dot(K, pc_cam.T).T

# 归一化齐次坐标
pc_img[:, 0] /= pc_img[:, 2]
pc_img[:, 1] /= pc_img[:, 2]

# 转换为图像坐标
img_coords = pc_img[:, :2].astype(np.int32)

# 显示图像
img_with_pc = img.copy()
for coord in img_coords:
    if 0 <= coord[0] < img.shape[1] and 0 <= coord[1] < img.shape[0]:
        cv2.circle(img_with_pc, (coord[0], coord[1]), 2, (0, 255, 0), -1)

cv2.imshow('Point Cloud Projection', img_with_pc)
cv2.waitKey(1)
This is my code!

[martin-liao]

Sorry for the late reply. I will help you project the point clouds onto the image plane now.

[martin-liao]

Since your project code is not available in this issue, I have pasted the code in the e-mail here:

import matplotlib.pyplot as plt
import torch

def test_acc(model, testdataloader, args):

t_diff_set = []
angles_diff_set = []
topk_list = torch.zeros(6, 5)
count = 0
mode = 'val'
for step, data in enumerate(testdataloader):
    if count >= 6:
        break
    model.eval()
    img = data['img'].cuda()
    pc_data_dict = data['pc_data_dict']
    for key in pc_data_dict:
        for j in range(len(pc_data_dict[key])):
            pc_data_dict[key][j] = torch.squeeze(pc_data_dict[key][j]).cuda()  
    pc_data_dict['feats'] = torch.squeeze(pc_data_dict['feats']).cuda() 
    K = torch.squeeze(data['K'].cuda())
    K_4 = torch.squeeze(data['K_4'].cuda())
    P = torch.squeeze(data['P'].cuda())  
    coarse_img_mask = torch.squeeze(data['coarse_img_mask']).cuda()      #1/4 size

    pc_kpt_idx = torch.squeeze(data['pc_kpt_idx']).cuda()  #(128)
    fine_center_kpt_coors = torch.squeeze(data['fine_center_kpt_coors']).cuda()  #[3, 128]
    fine_xy = torch.squeeze(data['fine_xy_coors']).cuda()
    fine_pc_inline_index = torch.squeeze(data['fine_pc_inline_index']).cuda()

    img_x = torch.linspace(0, coarse_img_mask.size(-1)-1, coarse_img_mask.size(-1)).view(1,-1).expand(coarse_img_mask.size(-2),coarse_img_mask.size(-1)).unsqueeze(0).cuda()
    img_y = torch.linspace(0, coarse_img_mask.size(-2)-1, coarse_img_mask.size(-2)).view(-1,1).expand(coarse_img_mask.size(-2),coarse_img_mask.size(-1)).unsqueeze(0).cuda()
    img_xy = torch.cat((img_x,img_y),dim=0)

    img_features, pc_features, coarse_img_score, coarse_pc_score, fine_img_feature_patch, fine_pc_inline_feature, fine_center_xy, coarse_pc_points = model(pc_data_dict, img, fine_center_kpt_coors, fine_xy, fine_pc_inline_index, mode)    # [128, 20, 64] ,[128, 2560]

    pc_features_inline = torch.gather(pc_features, index=pc_kpt_idx.expand(pc_features.size(0), args.num_kpt), dim=-1)
    pc_xyz_inline = torch.gather(pc_data_dict['points'][-1].T, index=pc_kpt_idx.unsqueeze(0).expand(3, args.num_kpt), dim=-1)

    img_features_flatten = img_features.contiguous().view(img_features.size(1), -1)
    img_xy_flatten = img_xy.contiguous().view(2, -1)
    img_features_flatten_inline = torch.gather(img_features_flatten, index=coarse_img_kpt_idx.unsqueeze(0).expand(img_features_flatten.size(0), args.num_kpt), dim=-1)
    img_xy_flatten_inline = torch.gather(img_xy_flatten, index=coarse_img_kpt_idx.unsqueeze(0).expand(2, args.num_kpt), dim=-1)

    pc_xyz_projection = torch.mm(K_4, (torch.mm(P[0:3, 0:3], pc_xyz_inline) + P[0:3, 3:]))
    pc_xy_projection = pc_xyz_projection[0:2, :] / pc_xyz_projection[2:,:]

    correspondence_mask = (torch.sqrt(torch.sum(torch.square(img_xy_flatten_inline.unsqueeze(-1) - pc_xy_projection.unsqueeze(-2)), dim=0)) <= args.dist_thres).float()

    dist_corr = 1 - torch.sum(img_features_flatten_inline.unsqueeze(-1) * pc_features_inline.unsqueeze(-2), dim=0)
    dist_mask = correspondence_mask * dist_corr  # only match got non-zero value
    true_index_list = torch.nonzero(dist_mask, as_tuple=False)
    true_value_list = dist_mask[true_index_list[:, 0], true_index_list[:, 1]].tolist()
    sorted_dist, indices = torch.sort(dist_corr, dim=-1, descending=False)
    topk = [1, 2, 3, 4, 5]

    for k in topk:
        candidate_values = sorted_dist[:, 0:k]
        for i in range(pc_kpt_idx.shape[0]):  # 128
            candidates = candidate_values[i, :].tolist()
            for candidate in candidates:
                if candidate in true_value_list:
                    topk_list[count, k - 1] += 1
    count += 1

    # 将图像和点云投影点从 GPU 移动到 CPU
    img_cpu = img.squeeze().permute(1, 2, 0).detach().cpu().numpy()
    pc_xy_projection_cpu = pc_xy_projection.detach().cpu().numpy()

    # 绘制图像和投影点
    plt.figure(figsize=(10, 10))
    plt.imshow(img_cpu)
    plt.scatter(pc_xy_projection_cpu[0, :], pc_xy_projection_cpu[1, :], c='r', s=10)
    plt.title(f"Projection of Point Cloud on Image - Step {step}")
    plt.show()

acc = torch.mean(topk_list / len(true_value_list), dim=0)
return acc

[martin-liao]

pc_xyz_projection = torch.mm(K_4, (torch.mm(P[0:3, 0:3], pc_xyz_inline) + P[0:3, 3:]))

K_4 is the scaled intrinsic matrix, and the above operation projects the point cloud onto a 1/4 resolution of the image plane. This is the reason why most of the points are in the upper left corner. If you want to project onto the original image plane, please use K instead of K_4. If you have any further concerns, please feel free to contact me.

[LoveMYChen]

Thank you very much for your help！

[martin-liao]

Since the problem has been resolved, I will close the issue.