yhd-ai commented 1 week ago

Thanks for your great work! I want to evaluate the geometry metrics but I found there were no code to compute CD and NC.

wenj commented 4 days ago

import trimesh
import torch
import json
import pytorch3d
import pytorch3d.loss
import pytorch3d.ops
import numpy as np

valid_frames = {
        '377': [30, 90, 120],
        '386': [150, 180, 270],
        '387': [150],
        '392': [30],
        '393': [0, 60, 120, 150, 180, 210, 240],
        '394': [0, 30, 60, 90, 120, 150, 180, 210, 240, 270],
        }

def normal_consistency_vertex(pred_trimesh, gt_trimesh):
    """
    :param pred: predicted trimesh
    :param gt trimesh: GT mesh trimesh
    """
    pred_vertices = np.array(pred_trimesh.vertices)
    pred_normals = np.array(pred_trimesh.vertex_normals)

    gt_vertices = np.array(gt_trimesh.vertices)
    gt_normals = np.array(gt_trimesh.vertex_normals)

    pred_verts_torch = torch.from_numpy(pred_vertices).double().unsqueeze(0).cuda()
    gt_verts_torch = torch.from_numpy(gt_vertices).double().unsqueeze(0).cuda()

    knn_ret = pytorch3d.ops.knn_points(gt_verts_torch, pred_verts_torch)
    p_idx = knn_ret.idx.squeeze(-1).detach().cpu().numpy()

    pred_normals = pred_normals[p_idx, :]

    consistency = 1 - np.linalg.norm(pred_normals - gt_normals, axis=-1).mean()

    return consistency

def filter_mesh(mesh, a, b, d, subject):
    # Filter out potential floating blobs
    labels = trimesh.graph.connected_component_labels(mesh.face_adjacency)
    components, cnt = np.unique(labels, return_counts=True)

    face_mask = (labels == components[np.argmax(cnt)])
    valid_faces = np.array(mesh.faces)[face_mask, ...]
    n_vertices = len(mesh.vertices)
    vertex_mask = np.isin(np.arange(n_vertices), valid_faces)
    mesh.update_faces(face_mask)
    mesh.update_vertices(vertex_mask)

    if subject in ['313', '315']:
        mesh = mesh.slice_plane([0, 0, d], [a, b, 1.0])
    else:
        mesh = mesh.slice_plane([0, 0, d], [-a, -b, -1.0])

    return mesh

our_losses = []

our_nc = []

with open('neus/ground_planes.json', 'r') as f:
    ground_planes = json.load(f)

for subject in valid_frames.keys():
    a, b, d = ground_planes[subject]
    for idx in valid_frames[subject]:
        print(subject, idx)
        gt = trimesh.load('YOUR GT PATH HERE')
        gt = filter_mesh(gt, a, b, d, subject)

        ours = trimesh.load('YOUR PREDICTED PATH HERE')
        ours = filter_mesh(nb, a, b, d, subject)

        # gt.export('tmp/gt.ply')
        # ours.export('tmp/ours.ply')

        # Normal consistency
        our_nc.append(normal_consistency_vertex(ours, gt))

        gt_verts = torch.from_numpy(gt.vertices * 100).double().unsqueeze(0).cuda()
        our_verts = torch.from_numpy(ours.vertices * 100).double().unsqueeze(0).cuda()

        our_loss = pytorch3d.loss.chamfer_distance(our_verts, gt_verts)
        our_losses.append(our_loss[0])

print ('CD: {:.4f}, NC: {:.4f}'.format((torch.stack(our_losses, dim=0).mean() / 2.0).item(), np.mean(our_nc)))

The script above is modified from ARAH. It should work with minimal adaptation. You could download the ground-truth and gound_planes.json from ARAH repository.

yhd-ai commented 2 days ago

Thanks for your reply! And I wonder how to export the mesh. I export mesh by adding these code in model.py ` mesh_canonical = Meshes(vertices_canonical.permute(1, 0).unsqueeze(0), self.faces.unsqueeze(0)) mesh_observation = Meshes(vertices_observation.permute(1, 0).unsqueeze(0), self.faces.unsqueeze(0)) xyz_observation = vertices_observation.permute(1, 0)[self.faces.reshape(-1)].reshape(F, 3, -1).mean(dim=1)

export mesh

ver = vertices_observation.permute(1, 0).detach().cpu().numpy() faces = self.faces.double().detach().cpu().numpy() mesh = trimesh.Trimesh(ver,faces) path = "/data0/lg2/GoMAvatar_ori/log/zju-mocap_377/mesh/" + str(idx)[2:-2] + ".ply" mesh.export(path)

` But I found the XYZ orientations and scale of exported mesh are different from the gt obtained by Neus, leading to extremely high error CD: 9003.5615, NC: -0.3155

wenj / GoMAvatar

Question about geometry metric？ #3

export mesh