NoW benchmark public validation set results

[AyushP123]

Hi,

I was checking MICA's performance on validation and test set released by NoW benchmark to the public and wanted to verify if the numbers I am getting are indeed correct:

Validation Set: median: 1.267735, mean: 2.329532, std: 3.083015

I wanted to know if you have run the model on the public validation dataset and if the numbers match as they are far away from the leaderboard results.

[AyushP123]

Here is the code for my inferencer which is a slight modification of demo.py

import argparse
import os
import random
from glob import glob
from pathlib import Path

import cv2
import numpy as np
import torch
import torch.backends.cudnn as cudnn
from insightface.app import FaceAnalysis
from insightface.app.common import Face
from insightface.utils import face_align
from loguru import logger
from pytorch3d.io import save_ply, save_obj
from skimage.io import imread
from tqdm import tqdm

from configs.config import get_cfg_defaults
from datasets.creation.util import get_arcface_input, get_center
from utils import util

def deterministic(rank):
    torch.manual_seed(rank)
    torch.cuda.manual_seed(rank)
    np.random.seed(rank)
    random.seed(rank)

    cudnn.deterministic = True
    cudnn.benchmark = False

def process(image_path, app, image_size=224):
    name = Path(image_path).stem
    img = cv2.imread(image_path)
    bboxes, kpss = app.det_model.detect(img, max_num=0, metric='default')
    if bboxes.shape[0] == 0:
        return None
    i = get_center(bboxes, img)
    bbox = bboxes[i, 0:4]
    det_score = bboxes[i, 4]
    kps = None
    if kpss is not None:
        kps = kpss[i]
    face = Face(bbox=bbox, kps=kps, det_score=det_score)
    blob, aimg = get_arcface_input(face, img)

    return blob, face_align.norm_crop(img, landmark=face.kps, image_size=image_size)

def to_batch(image_path, app):
    arcface, image = process(image_path, app)

    image = image / 255.
    image = cv2.resize(image, (224, 224)).transpose(2, 0, 1)
    image = torch.tensor(image).cuda()[None]

    arcface = torch.tensor(arcface).cuda()[None]

    return image, arcface

def load_checkpoint(args, mica):
    checkpoint = torch.load(args.m)
    if 'arcface' in checkpoint:
        mica.arcface.load_state_dict(checkpoint['arcface'])
    if 'flameModel' in checkpoint:
        mica.flameModel.load_state_dict(checkpoint['flameModel'])

def main(cfg, args, src_folder, paths_file, dst_folder):
    device = 'cuda:0'
    cfg.model.testing = True
    mica = util.find_model_using_name(model_dir='micalib.models', model_name=cfg.model.name)(cfg, device)
    load_checkpoint(args, mica)
    mica.eval()

    faces = mica.render.faces[0].cpu()

    app = FaceAnalysis(name='antelopev2', providers=['CUDAExecutionProvider'])
    app.prepare(ctx_id=0, det_size=(224, 224))

    with torch.no_grad():
        logger.info(f'Processing has started...')
        image_paths = open(paths_file).readlines()
        for path in tqdm(image_paths):
            dir_name = os.path.dirname(path)
            path = os.path.join(src_folder, path.strip())
            name = Path(path).stem
            images, arcface = to_batch(path, app)
            codedict = mica.encode(images, arcface)
            opdict = mica.decode(codedict)
            meshes = opdict['pred_canonical_shape_vertices']
            code = opdict['pred_shape_code']
            lmk = mica.flame.compute_landmarks(meshes)

            mesh = meshes[0]
            landmark_51 = lmk[0, 17:]
            landmark_7 = landmark_51[[19, 22, 25, 28, 16, 31, 37]]
            rendering = mica.render.render_mesh(mesh[None])
            image = (rendering[0].cpu().numpy().transpose(1, 2, 0).copy() * 255)[:, :, [2, 1, 0]]
            image = np.minimum(np.maximum(image, 0), 255).astype(np.uint8)

            dst = os.path.join(dst_folder, dir_name)
            os.makedirs(dst, exist_ok=True)
            save_obj(f'{dst}/{name}.obj', verts=mesh.cpu() * 1000.0, faces=faces)
            np.savetxt(f'{dst}/{name}.txt', landmark_7.cpu().numpy(), delimiter=' ')

        logger.info(f'Processing finished. Results has been saved')

if __name__ == '__main__':
    parser = argparse.ArgumentParser(description='MICA - Towards Metrical Reconstruction of Human Faces')
    parser.add_argument('-s', default='/home/ayushp/Capstone/NoW_dataset/NoW_Dataset/final_release_version/iphone_pictures/',
                        type=str, help='Path to the folder with iphone pictures')
    parser.add_argument('-p', default='/home/ayushp/Capstone/NoW_dataset/NoW_Dataset/final_release_version/iphone_pictures/imagepathsvalidation.txt',
                         type=str, help='Path to the file with image paths')
    parser.add_argument('-d', default='/home/ayushp/Capstone/mica_now',
                         type=str, help='Output directory')
    parser.add_argument('-m', default='data/pretrained/mica.tar', type=str, help='Pretrained model path')

    args = parser.parse_args()
    cfg = get_cfg_defaults()

    deterministic(42)

    main(cfg, args, args.s, args.p, args.d)

[Zielon]

Your landmarks and mesh are in different scale. FLAME mesh is in meters. When you multiply it by 1000 you will get millimeters. So you have landmarks in meters and mesh in millimeters.

Take a look here: https://github.com/Zielon/MICA/blob/master/demo.py#L137

[AyushP123]

Thank you for your response, after fixing the issue I got the following values which are more consistent

median: 0.909922, mean: 1.125202, std: 0.939006