smpl data correction - Githubissues

I note in zju_mocap.py, you correct the smpl data like this.
        # Use camera extrinsic to rotate the simple to each camera coordinate frame!

        # the R,t is used like this, stored in cam
        # i.e. the T stored in cam is actually p_c = T_cw @ p_w
        # def get_rays(H, W, K, R, T):
        #     # calculate the camera origin
        #     rays_o = -np.dot(R.T, T).ravel()
        #     # calculate the world coodinates of pixels
        #     i, j = np.meshgrid(
        #         np.arange(W, dtype=np.float32), np.arange(H, dtype=np.float32), indexing="xy"
        #     )
        #     xy1 = np.stack([i, j, np.ones_like(i)], axis=2)
        #     pixel_camera = np.dot(xy1, np.linalg.inv(K).T)
        #     pixel_world = np.dot(pixel_camera - T.ravel(), R)
        #     # calculate the ray direction
        #     rays_d = pixel_world - rays_o[None, None]
        #     rays_d = rays_d / np.linalg.norm(rays_d, axis=2, keepdims=True)
        #     rays_o = np.broadcast_to(rays_o, rays_d.shape)
        #     return rays_o, rays_d

        # ! the cams R is in a very low precision, have use SVD to project back to SO(3)
        for cid in range(num_cams):
            _R = self.cams["R"][cid]
            u, s, vh = np.linalg.svd(_R)
            new_R = u @ vh
            self.cams["R"][cid] = new_R

        # this is copied
        smpl_layer = SMPLLayer(osp.join(osp.dirname(__file__), "../data/smpl-meta/SMPL_NEUTRAL.pkl"))

        # * Load smpl to camera frame
        self.smpl_theta_list, self.smpl_trans_list, smpl_beta_list = [], [], []
        self.meta = []
        for img_fn in self.ims:
            cam_ind = int(img_fn.split("/")[-2])
            frame_idx = int(img_fn.split("/")[-1].split(".")[0])
            self.meta.append({"cam_ind": cam_ind, "frame_idx": frame_idx})
            smpl_fn = osp.join(root, "smpl_params", f"{frame_idx}.npy")
            smpl_data = np.load(smpl_fn, allow_pickle=True).item()
            T_cw = np.eye(4)
            T_cw[:3, :3], T_cw[:3, 3] = (
                np.array(self.cams["R"][cam_ind]),
                np.array(self.cams["T"][cam_ind]).squeeze(-1) / 1000.0,
            )

            smpl_theta = smpl_data["poses"].reshape((24, 3))
            assert np.allclose(smpl_theta[0], 0)
            smpl_rot, smpl_trans = smpl_data["Rh"][0], smpl_data["Th"]
            smpl_R = axangle2mat(
                smpl_rot / (np.linalg.norm(smpl_rot) + 1e-6), np.linalg.norm(smpl_rot)
            )

            T_wh = np.eye(4)
            T_wh[:3, :3], T_wh[:3, 3] = smpl_R.copy(), smpl_trans.squeeze(0).copy()

            T_ch = T_cw.astype(np.float64) @ T_wh.astype(np.float64)

            smpl_global_rot_d, smpl_global_rot_a = mat2axangle(T_ch[:3, :3])
            smpl_global_rot = smpl_global_rot_d * smpl_global_rot_a
            smpl_trans = T_ch[:3, 3]  # 3
            smpl_theta[0] = smpl_global_rot
            beta = smpl_data["shapes"][0][:10]

            # ! Because SMPL global rot is rot around joint-0, have to correct this in the global translation!!
            _pose = axis_angle_to_matrix(torch.from_numpy(smpl_theta)[None])
            so = smpl_layer(
                torch.from_numpy(beta)[None],
                body_pose=_pose[:, 1:],
            )
            j0 = (so.joints[0, 0]).numpy()
            t_correction = (_pose[0, 0].numpy() - np.eye(3)) @ j0
            smpl_trans = smpl_trans + t_correction

            self.smpl_theta_list.append(smpl_theta)
            smpl_beta_list.append(beta)
            self.smpl_trans_list.append(smpl_trans)
Do this correction has any reference? Since during the correction, camera pose is used, it means one frame has different smpl data in different views. It feels quiet strange...
JiahuiLei / GART

smpl data correction #7
