Question regarding MatchLoss

[cmh1027]

def forward(self, models, gt_E, pts1, pts2, K1, K2, im_size1, im_size2, topk_flag=False, k=1):
    essential_loss = []
    for b in range(gt_E.shape[0]):
        if self.fmat:
            Es = K2[b].transpose(-1, -2) @ models[b] @ K1[b]
            pts1_1 = normalize_keypoints_tensor(denormalize_pts(pts1[b].clone(), im_size1[b]), K1[b])
            pts2_2 = normalize_keypoints_tensor(denormalize_pts(pts2[b].clone(), im_size2[b]), K2[b])
        else:
            pts1_1 = pts1[b].clone()
            pts2_2 = pts2[b].clone()
            Es = models[b]

        _, gt_R_1, gt_t_1, gt_inliers = cv2.recoverPose(
            gt_E[b].astype(np.float64),
            pts1_1.unsqueeze(1).cpu().detach().numpy(),
            pts2_2.unsqueeze(1).cpu().detach().numpy(),
            np.eye(3, dtype=gt_E.dtype)
        )

        # find the ground truth inliers
        gt_mask = np.where(gt_inliers.ravel() > 0, 1.0, 0.0).astype(np.bool)
        gt_mask = torch.from_numpy(gt_mask).to(pts1_1.device)

        # symmetric epipolar errors based on gt inliers
        geod = batch_episym(
            pts1_1[gt_mask].repeat(Es.shape[0], 1, 1),
            pts2_2[gt_mask].repeat(Es.shape[0], 1, 1),
            Es
        )
        e_l = torch.min(geod, geod.new_ones(geod.shape))
        if torch.isnan(e_l.mean()).any():
            print("nan values in pose loss")# .1*

        if topk_flag:
            topk_indices = torch.topk(e_l.mean(1), k=k, largest=False).indices
            essential_loss.append(e_l[topk_indices].mean())
        else:
            essential_loss.append(e_l.mean())
    # average
    return sum(essential_loss) / gt_E.shape[0]

def batch_episym(x1, x2, F, eps=1e-10):
    """
    Epipolar symmetric error from CLNet.
    x1, x2 : (B, N, 2)
    F: (B, 3, 3)    

    """    
    batch_size, num_pts = x1.shape[0], x1.shape[1] # (B, N, 2)
    x1 = torch.cat([x1, x1.new_ones(batch_size, num_pts, 1)], dim=-1).reshape(batch_size, num_pts, 3, 1) # (B, N, 3, 1)
    x2 = torch.cat([x2, x2.new_ones(batch_size, num_pts, 1)], dim=-1).reshape(batch_size, num_pts, 3, 1) # (B, N, 3, 1)
    F = F.reshape(-1, 1, 3, 3).repeat(1, num_pts, 1, 1) # (B, N, 3, 3)
    x2Fx1 = torch.matmul(x2.transpose(2, 3), torch.matmul(F, x1)).reshape(batch_size, num_pts) # (B, N)
    Fx1 = torch.matmul(F, x1).reshape(batch_size, num_pts, 3) # (B, N, 3)
    Ftx2 = torch.matmul(F.transpose(2, 3), x2).reshape(batch_size, num_pts, 3)
    ys = x2Fx1 ** 2 * (1.0 / (Fx1[:, :, 0] ** 2 + Fx1[:, :, 1] ** 2 + eps) + 1.0 / (Ftx2[:, :, 0] ** 2 + Ftx2[:, :, 1] ** 2 + eps))
    if torch.isnan(ys).any():
        print("ys is nan in batch_episym")
    return ys

When self.fmat is on, F of batch_episym contains Fundamental matrix so x1,x2 of batch_episym must be points in a pixel space. However, according to your code, they will be on the normalized image plane by the code below.

pts1_1 = normalize_keypoints_tensor(denormalize_pts(pts1[b].clone(), im_size1[b]), K1[b])
pts2_2 = normalize_keypoints_tensor(denormalize_pts(pts2[b].clone(), im_size2[b]), K2[b])

Is this intended?

[weitong8591]

Yes, I normalized the coordinates becase in the fundamental matrix case, the estimated F is transformed to E matrix. Feel free to use this or directly compute the symmetric epipolar errors on F matrices.

[cmh1027]

@weitong8591 Could you let me know in which part F is transformed to E? It seems Es matrix fed to batch_episym is fundamental matrix due to the following code. Es = K2[b].transpose(-1, -2) @ models[b] @ K1[b]

[cmh1027]

In addition, is there any plan to release the checkpoint & code for SuperGlue? I've struggled with it by myself, but NaN problem hassles me.

[weitong8591]

In addition, is there any plan to release the checkpoint & code for SuperGlue? I've struggled with it by myself, but NaN problem hassles me.

I don't have anything trained with SuperGlue that can be released.

[weitong8591]

@weitong8591 Could you let me know in which part F is transformed to E? It seems Es matrix fed to batch_episym is fundamental matrix due to the following code. Es = K2[b].transpose(-1, -2) @ models[b] @ K1[b]

In F case, models[b] are supposed to be F matrices, which are transformed to Es and passed to batch_episym.

[weitong8591]

hi, I am closing this issue, pls reopen it if anything is unclear.