Landmarks are completely incorrect

[ricglz]

I'm trying to create the 2D landmarks of an image, but they're completely wrong as can be seen in the following images:

Original image

Annotated image with bbox and landmarks

Code

from face_alignment import FaceAlignment, LandmarksType
from facenet_pytorch import MTCNN
from PIL import Image
from torch.cuda import is_available as is_cuda_available
import cv2
import numpy

mtcnn = MTCNN()
aligner = FaceAlignment(
    LandmarksType._2D,
    device='cuda' if is_cuda_available() else 'cpu',
    face_detector='blazeface',
)

def annotate_landmarks(im, landmarks):
    im = im.copy()
    for idx, point in enumerate(landmarks):
        pos = (point[0, 0], point[0, 1])
        cv2.putText(im, str(idx), pos,
                    fontFace=cv2.FONT_HERSHEY_SCRIPT_SIMPLEX,
                    fontScale=0.4,
                    color=(0, 0, 255))
        cv2.circle(im, pos, 3, color=(0, 255, 255))
    return im

def annotate_bboxes(im, bboxes):
    im = im.copy()
    for _, box in enumerate(bboxes):
        pos_1, pos_2 = (box[0], box[1]), (box[2], box[3])
        cv2.rectangle(im, pos_1, pos_2, color=(255, 0, 0))
    return im

def parse_bbox(bbox):
    np_box = numpy.array(bbox)
    np_box[:2] -= 25
    np_box[2:] += 25
    return np_box.astype(int)

def get_bboxes(img):
    boxes = mtcnn.detect(Image.fromarray(img))[0]
    return [parse_bbox(bbox) for bbox in boxes]

def get_landmarks(img, boxes):
    points = aligner.get_landmarks_from_image(img, detected_faces=boxes)
    points = points[0].astype(numpy.uint8)

    landmarks = numpy.matrix([[p[0], p[1]] for p in points])

    return landmarks

def main():
    img = cv2.imread('./avatars/opened_eyes.jpg')
    img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)

    bboxes = get_bboxes(img)
    landmarks = get_landmarks(img, bboxes)

    img = cv2.cvtColor(img, cv2.COLOR_RGB2BGR)
    annotated_img = annotate_bboxes(img, bboxes)
    annotated_img = annotate_landmarks(annotated_img, landmarks)

    cv2.imwrite('annotated_img.jpg', annotated_img)

if __name__ == "__main__":
    main()

Environment

MacbookPro 2012, MacOS

[1adrianb]

I tried running the library using your image and it worked fine. I assume it's an issue with the format of your bounding box. Please see the implementation from https://github.com/1adrianb/face-alignment/tree/master/face_alignment/detection

For an example of how to run the code see: https://github.com/1adrianb/face-alignment/blob/master/examples/detect_landmarks_in_image.py

Running the code linked on your image produces the following result:

[ricglz]

Ok, just found the actual error. When doing the points = points[0].astype(numpy.uint8) statement, attempting to cast float32 to uint8 reduces significantly the value of the numbers. This can be fixed by changing the line to either of the following:

points = points[0].astype(numpy.uint32)
points = points[0].astype(numpy.int32)
points = points[0].astype(int)