How do a server face recognition with Node.js?

[ArcSoul]

Hello everyone, well I have a question about face-api.js and I do not understand how a nodej.js server would read for example base64 or a file, and transfer it to a string where it says it was recognized or something , I have done it before with Angular and Typescript, but the thing now is to pass it as a service and for a server to take care of the heavy load of detecting the face and not the browser, since it is currently somewhat slow, I did not find solutions in java , more than javacv or opencv, and those solutions did not work for me when it is validation (from 1 to 1 not from 1 an) with only 3 or 12 images of the same person (probe with 3 and 12), and I have not found any tutorial of how to work in node.js (since in case I gave up on java) and now I am looking to do the recognition service only in node.js, but as I said, I only found the previous api that was face-recognition.js and I read that the same author made him obsolete, so I chose to look for this one but as I said before, I can't find a tutorial, more than the mini guide of the same github, which I did monkey.env which I don't know how I would do if it was just a server in node.js and that works in CanvasHtml I think, help me please: 'v

in other words a web services in node.js

[msvargas]

If you need real time, check detecting expression in #467, its same with recognition, using node-webRTC, if you only recognion a snapshot, you upload image from client and convert buffer image to Tensor and response with JSON

Code from stack overflow NOTE: consumption of CPU and RAM is very High

"use strict";
require("@tensorflow/tfjs-node");
const tf = require("@tensorflow/tfjs");
const nodeFetch = require("node-fetch");
const fapi = require("face-api.js");
const path = require("path");
const { createCanvas, createImageData } = require("canvas");
const {
  RTCVideoSink,
  RTCVideoSource,
  i420ToRgba,
  rgbaToI420
} = require("wrtc").nonstandard;

fapi.env.monkeyPatch({ fetch: nodeFetch });
const MODELS_URL = path.join(__dirname, "/weights");

const width = 640;
const height = 480;

Promise.all([
  fapi.nets.tinyFaceDetector.loadFromDisk(MODELS_URL),
  fapi.nets.faceLandmark68Net.loadFromDisk(MODELS_URL),
  fapi.nets.faceRecognitionNet.loadFromDisk(MODELS_URL),
  fapi.nets.faceExpressionNet.loadFromDisk(MODELS_URL)
]);

function beforeOffer(peerConnection) {
  const source = new RTCVideoSource();
  const track = source.createTrack();
  const transceiver = peerConnection.addTransceiver(track);
  const sink = new RTCVideoSink(transceiver.receiver.track);

  let lastFrame = null;

  function onFrame({ frame }) {
    lastFrame = frame;
  }

  sink.addEventListener("frame", onFrame);

  // TODO(mroberts): Is pixelFormat really necessary?
  const canvas = createCanvas(width, height);
  const context = canvas.getContext("2d", { pixelFormat: "RGBA24" });
  context.fillStyle = "white";
  context.fillRect(0, 0, width, height);
  const emotionsArr = {
    0: "neutral",
    1: "happy",
    2: "sad",
    3: "angry",
    4: "fearful",
    5: "disgusted",
    6: "surprised"
  };
  async function detectEmotion(lastFrameCanvas) {
    const frameTensor3D = tf.browser.fromPixels(lastFrameCanvas);
    const face = await fapi
      .detectSingleFace(
        frameTensor3D,
        new fapi.TinyFaceDetectorOptions({ inputSize: 160 })
      )
      .withFaceExpressions();
    //console.log(face);
    const emo = getEmotion(face);
    frameTensor3D.dispose();
    return emo;
  }
  function getEmotion(face) {
    try {
      let mostLikelyEmotion = emotionsArr[0];
      let predictionArruracy = face.expressions[emotionsArr[0]];

      for (let i = 0; i < Object.keys(face.expressions).length; i++) {
        if (
          face.expressions[emotionsArr[i]] > predictionArruracy &&
          face.expressions[emotionsArr[i]] < 1
        ) {
          mostLikelyEmotion = emotionsArr[i];
          predictionArruracy = face.expressions[emotionsArr[i]];
        }
      }
      //console.log(mostLikelyEmotion);
      return mostLikelyEmotion;
    } catch (e) {
      return "";
    }
  }
  let emotion = "";
  const interval = setInterval(() => {
    if (lastFrame) {
      const lastFrameCanvas = createCanvas(lastFrame.width, lastFrame.height);
      const lastFrameContext = lastFrameCanvas.getContext("2d", {
        pixelFormat: "RGBA24"
      });

      const rgba = new Uint8ClampedArray(
        lastFrame.width * lastFrame.height * 4
      );
      const rgbaFrame = createImageData(
        rgba,
        lastFrame.width,
        lastFrame.height
      );
      i420ToRgba(lastFrame, rgbaFrame);

      lastFrameContext.putImageData(rgbaFrame, 0, 0);
      context.drawImage(lastFrameCanvas, 0, 0);

      detectEmotion(lastFrameCanvas).then(function(res) {
        emotion = res;
      });
    } else {
      context.fillStyle = "rgba(255, 255, 255, 0.025)";
      context.fillRect(0, 0, width, height);
    }

    if (emotion != "") {
      context.font = "60px Sans-serif";
      context.strokeStyle = "black";
      context.lineWidth = 1;
      context.fillStyle = `rgba(${Math.round(255)}, ${Math.round(
        255
      )}, ${Math.round(255)}, 1)`;
      context.textAlign = "center";
      context.save();
      context.translate(width / 2, height);
      context.strokeText(emotion, 0, 0);
      context.fillText(emotion, 0, 0);
      context.restore();
    }

    const rgbaFrame = context.getImageData(0, 0, width, height);
    const i420Frame = {
      width,
      height,
      data: new Uint8ClampedArray(1.5 * width * height)
    };
    rgbaToI420(rgbaFrame, i420Frame);
    source.onFrame(i420Frame);
  });

  const { close } = peerConnection;
  peerConnection.close = function() {
    clearInterval(interval);
    sink.stop();
    track.stop();
    return close.apply(this, arguments);
  };
}

module.exports = { beforeOffer };

[whyboris]

I found a solution to a related problem that may help here.

If you have an image on the hard disk, there is no problem:

    const img = await canvas.loadImage('./images/myImage.jpg');            
    const detections = await faceapi.detectAllFaces(img);

But if you don't want to write to disk first, you can just convert the image to base64, create a fake canvas, and you're done!

For example:

  const data = new Image();

  data.src = new Buffer(myImageAsBase64String, 'base64');
  data.width = 480;
  data.height = 360;

  const detections = await faceapi.detectAllFaces(data);

ps: my imports require node-canvas

const canvas = require('canvas');
const { Canvas, Image } = canvas;
faceapi.env.monkeyPatch({ Canvas, Image });