inaccurate mesh positioning

[p30arena]

@patrikhuber Hi!

I can't position the mesh correctly and I don't know what the problem is:

# -*- coding: utf-8 -*-
"""3d-face.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/1z8PRHUfyhgmHKiakjcduHJCkPKJk1t7Z
"""

!pip install eos-py
#!pip uninstall eos

import dlib
import eos

!python --version

!pip install pillow

import matplotlib.pyplot as plt
import numpy as np
import dlib
import eos

!wget -O face.jpg "https://img.freepik.com/free-photo/portrait-caucasian-woman-smiling_53876-24998.jpg?size=626&ext=jpg"

import cv2
img = cv2.imread("face.jpg")
b,g,r = cv2.split(img)
img = cv2.merge([r,g,b])

from io import BytesIO
import PIL
import IPython.display
from skimage.draw import circle

def show_array(a, fmt='JPEG'):
    #a = np.uint8(np.clip(a, 0, 255))
    image_data = BytesIO()
    PIL.Image.fromarray(a).save(image_data, fmt)
    IPython.display.display(IPython.display.Image(data=image_data.getvalue()))

def draw_circle(canvas, x, y, r=1):
    rr,cc = circle(x, y, r, shape=canvas.shape)
    canvas[rr,cc] = (255, 255, 255)

!wget -O shape_predictor_68_face_landmarks.dat "https://github.com/italojs/facial-landmarks-recognition-/raw/master/shape_predictor_68_face_landmarks.dat"

detector = dlib.get_frontal_face_detector()
shape_predictor = dlib.shape_predictor('shape_predictor_68_face_landmarks.dat')
boxes = detector(img)

shapes = [shape_predictor(img, box) for box in boxes]
landmarks = [[[part.x, part.y] for part in shape.parts()] for shape in shapes][0]
canvas = np.copy(img)
for point in landmarks:
    draw_circle(canvas, point[1], point[0], r=2)
show_array(canvas)

!wget -O sfm_shape_3448.bin "https://github.com/patrikhuber/eos/raw/master/share/sfm_shape_3448.bin"
!wget -O expression_blendshapes_3448.bin "https://github.com/patrikhuber/eos/raw/master/share/expression_blendshapes_3448.bin"
!wget -O ibug_to_sfm.txt "https://github.com/patrikhuber/eos/raw/master/share/ibug_to_sfm.txt"
!wget -O sfm_3448_edge_topology.json "https://github.com/patrikhuber/eos/raw/master/share/sfm_3448_edge_topology.json"
!wget -O sfm_model_contours.json "https://github.com/patrikhuber/eos/raw/master/share/sfm_model_contours.json"

def read_pts(data):
  landmarks = []
  ibug_index = 1
  for coords in data:
    landmarks.append(eos.core.Landmark(str(ibug_index), [float(coords[0]), float(coords[1])]))
    ibug_index = ibug_index + 1

  return landmarks

image_width = img.shape[1]
image_height = img.shape[0]

model = eos.morphablemodel.load_model("sfm_shape_3448.bin")
blendshapes = eos.morphablemodel.load_blendshapes("expression_blendshapes_3448.bin")
landmark_mapper = eos.core.LandmarkMapper('ibug_to_sfm.txt')
edge_topology = eos.morphablemodel.load_edge_topology('sfm_3448_edge_topology.json')
contour_landmarks = eos.fitting.ContourLandmarks.load('ibug_to_sfm.txt')
model_contour = eos.fitting.ModelContour.load('sfm_model_contours.json')
eos_landmarks = [landmarks]
morphablemodel_with_expressions = eos.morphablemodel.MorphableModel(model.get_shape_model(), blendshapes,
                                                                        color_model=eos.morphablemodel.PcaModel(),
                                                                        vertex_definitions=None,
                                                                        texture_coordinates=model.get_texture_coordinates())

(mesh, pose, shape_coeffs, blendshape_coeffs) = eos.fitting.fit_shape_and_pose(morphablemodel_with_expressions, read_pts(landmarks), landmark_mapper, image_width, image_height, edge_topology, contour_landmarks, model_contour)

projection = pose.get_projection()
modelview = pose.get_modelview()

v = np.asarray(mesh.vertices)
plt.figure(figsize=(5,5))
plt.axis('equal')
plt.scatter(v[:,0], v[:,1], s=0.5)
plt.show()

# based on https://github.com/g-truc/glm/blob/master/glm/gtc/matrix_transform.inl#L438-L462
def project_points(points, modelview, projection, width, height):
    # project through pose and projection
    points = np.hstack((points, np.zeros((points.shape[0], 1), dtype=points.dtype))).dot(modelview).dot(projection)
#     points /= points[:,3].reshape(-1, 1)

    # convert from homogenous to screen coordinates
    points[:,0] = (points[:,0] + 1) * (width / 2)
    points[:,1] = (points[:,1] + 1) * (height / 2)

    # drop the w coordinate
    return points[:,:3]

points_2d = project_points(np.copy(v), modelview, projection, image_width, image_height)

v = np.asarray(mesh.vertices)
plt.figure(figsize=(5,5))
plt.axis('equal')
plt.gca().invert_xaxis()
plt.scatter(points_2d[:,0], points_2d[:,1], s=0.5)
plt.show()

canvas = np.copy(img)
for point in points_2d:
  draw_circle(canvas, image_height - point[1], image_width - point[0], r=1)
show_array(canvas)

center = boxes[0].center()
x,y = center.x-image_width/2, center.y-image_height/2

canvas = np.copy(img)
for point in points_2d:
    draw_circle(canvas, (image_height - point[1]) + y, (image_width - point[0]) + x, r=1)
show_array(canvas)

[gigadeplex]

Hi, this was referenced in #140 I posted a demo on how to do this here https://github.com/gigadeplex/dlib-eos

[p30arena]

@gigadeplex

Hi! That's Brilliant! Thank you!

How can I contact you for further collaborations?

[patrikhuber]

Hi,

Thanks @gigadeplex for the reply and for linking #140.

I'm going to close this, as it's not an issue with the eos library.