Help Regarding Skeleton Plotting on Image with transformation

Hello @Yzmblog,

Thanks for your previous help with "perspective camera matrix".

This is a big request, so I hope you can check these. Now, I want to plot the 3d smpl joints on the output images that we get. I used the "extrinsic" matrix from the function https://github.com/Yzmblog/MonoHuman/blob/6429fdbf8c3eb7f8f77326a969e6519e91dad6cc/core/data/monohuman/freeview.py#L277

I tried to plot it, but apparently, I am getting upside down skeleton, so I think there is something issue with the matrix that I am using . Can you verify it once. It would be really helpful

Note: Even if I just get the joints on the image without the skeleton, thats more than enough. If you have any files or function of your own, feel free to let me know

Data used

  skeleton_tree = {
      'color': [
          'k', 'r', 'r', 'r', 'b', 'b', 'b', 'k', 'r', 'r', 'r', 'b', 'b', 'b',
          'y', 'y', 'y', 'y', 'b', 'b', 'b', 'r', 'r', 'r'
      ],
      'smpl_tree': [[ 0, 1 ],[ 0, 2 ],[ 0, 3 ],[ 1, 4 ],[ 2, 5 ],[ 3, 6 ],[ 4, 7 ],[ 5, 8 ],[ 6, 9 ],[ 7, 10],
          [ 8, 11],[ 9, 12],[ 9, 13],[ 9, 14],[12, 15],[13, 16],[14, 17],[16, 18],[17, 19],[18, 20],[19, 21],[20, 22],[21, 23]]
  }
  with open('/home/ndip/humannerf/dataset/zju_mocap/390/mesh_infos.pkl','rb') as f:
      data = pickle.load(f, encoding='latin1')
      frame120_data= data['frame_000120']
  joints = frame120_data['joints'].astype(np.float32)
  joints = joints.reshape(24,3)
  joints3d = joints[:, [0,2,1]]

  E = np.array([[-0.21479376,  0.97530604, -0.05139818, -0.58367276],
         [-0.28440348, -0.01211552,  0.95862813,  1.02107571],
         [ 0.93433307,  0.22052516,  0.27998276,  2.72222895],
         [ 0.        ,  0.        ,  0.        ,  1.        ]])

  K  = np.array([[537.1407 ,   0.     , 271.4171 ],
         [  0.     , 537.7115, 242.44179],
         [  0.     ,   0.     ,   1.     ]])

  R = E[:3, :3]
  t = E[:3, 3]

Transformation from 3d to 2d

  P = np.matmul(K, np.hstack((R, t.reshape(-1, 1))))
  N_poses = joints3d.shape[0]
  homogeneous_coords = np.concatenate((joints3d, np.ones((N_poses, 1))), axis=1)
  points_new = np.matmul(P, homogeneous_coords.T).T
  points_new /= points_new[:, 2:]  # Normalize the homogeneous coordinates
  points_new = points_new[:, :2]

For plotting the skeleton, I used the following function

  def plotSkel2D(pts,
                 config=skeleton_tree,
                 ax=None,
                 linewidth=2,
                 alpha=1,
                 max_range=1,
                 imgshape=None,
                 thres=0.1):
      if len(pts.shape) == 2:
          pts = pts[None, :, :]  #(nP, nJ, 2/3)
      elif len(pts.shape) == 3:
          pass
      else:
          raise RuntimeError('The dimension of the points is wrong!')
      if torch.is_tensor(pts):
          pts = pts.detach().cpu().numpy()
      if pts.shape[2] == 3 or pts.shape[2] == 2:
          pts = pts.transpose((0, 2, 1))
      # pts : bn, 2/3, NumOfPoints or (2/3, N)
      if ax is None:
          fig = plt.figure(figsize=[5, 5])
          ax = fig.add_subplot(111)
      if 'color' in config.keys():
          colors = config['color']
      else:
          colors = ['b' for _ in range(len(config['smpl_tree']))]

      def inrange(imgshape, pts):
          if pts[0] < 5 or \
             pts[0] > imgshape[1] - 5 or \
             pts[1] < 5 or \
             pts[1] > imgshape[0] - 5:
              return False
          else:
              return True

      for nP in range(pts.shape[0]):
          for idx, (i, j) in enumerate(config['smpl_tree']):
              if pts.shape[1] == 3:  # with confidence
                  if np.min(pts[nP][2][[i, j]]) < thres:
                      continue
                  lw = linewidth * 2 * np.min(pts[nP][2][[i, j]])
              else:
                  lw = linewidth
              if imgshape is not None:
                  if inrange(imgshape, pts[nP, :, i]) and \
                      inrange(imgshape, pts[nP, :, j]):
                      pass
                  else:
                      continue
              ax.plot([pts[nP][0][i], pts[nP][0][j]],
                      [pts[nP][1][i], pts[nP][1][j]],
                      lw=lw,
                      color=colors[idx],
                      alpha=1)
          # if pts.shape[1] > 2:
          ax.scatter(pts[nP][0], pts[nP][1], c='r')
      if False:
          ax.axis('equal')
          plt.xlabel('x')
          plt.ylabel('y')
      else:
          ax.axis('off')
      return ax

LASTLY

  def vis_skeleton_single_image(image_path, keypoints):
      img = cv2.imread(image_path)
      kpts2d = np.array(keypoints)

      _, ax = plt.subplots(1, 1)
      ax.imshow(img[..., ::-1])
      H, W = img.shape[:2]
  #    plotSkel2D(kpts2d, ax = ax)
      plotSkel2D(kpts2d, skeleton_tree, ax = ax,  linewidth = 2, alpha = 1, max_range = 1, thres = 0.5 )
      plt.show()
  img_path = "/home/ndip/humannerf/experiments/human_nerf/zju_mocap/p390/adventure/latest/freeview_120/000000.png"
  vis_skeleton_single_image(img_path, points_new)

Result is this: Screenshot 2023-08-06 131214

I am unsure, why it flipped, because when I just used the PlotSkel2D function, I got this

Screenshot 2023-08-06 131539

Could this be the transformation of points or the plotting is an issue??? Kindly help. Thank you @Yzmblog

Yzmblog / MonoHuman