flatsiedatsie
commented
4 years ago Here is my webworker:
importScripts('faceEnvWorkerPatch.js');
importScripts('js/face-api.min.js');
let model;
var detector_options;
let selected_face_detector = 'tiny_face_detector'; //'ssd_mobilenetv1';
let inputSize = 512
let scoreThreshold = 0.5
const minConfidence = 0.05; // expression
var loaded_models = [];
var selected_use = 'face'; // can be changed with a command from the main script.
faceapi_worker_loaded = false;
beauty_worker_loaded = false;
ethnicity_worker_loaded = false;
async function load_mobile_net() {
try{
await faceapi.nets.ssdMobilenetv1.load('models/');
console.log("loaded mobilenet face detection");
detector_options = getFaceDetectorOptions();
}
catch(e){console.log(e);}
}
async function load_tiny_net() {
try{
await faceapi.nets.tinyFaceDetector.load('models/');
console.log("loaded tiny face detection");
detector_options = getFaceDetectorOptions();
}
catch(e){console.log(e);}
postMessage({'loaded':'tiny'});
}
async function load_age_and_gender() {
try{
await faceapi.nets.ageGenderNet.load('models/'); // gender
console.log("loaded age and gender detection model");
}
catch(e){console.log(e);}
}
async function load_landmarks() {
try{
await faceapi.loadFaceLandmarkModel('models/'); // landmarks
console.log("loaded landmark detection model");
}
catch(e){console.log(e);}
}
async function load_expression() {
try{
await faceapi.loadFaceExpressionModel('models/'); // expression
console.log("loaded expression detection model");
}
catch(e){console.log(e);}
}
async function load_recognition() {
try{
await faceapi.loadFaceRecognitionModel('models/') // face vectors
console.log("loaded face recognition detection model");
}
catch(e){console.log(e);}
}
function getCurrentFaceDetectionNet() {
if (selected_face_detector === SSD_MOBILENETV1) {
return faceapi.nets.ssdMobilenetv1
}
if (selected_face_detector === TINY_FACE_DETECTOR) {
return faceapi.nets.tinyFaceDetector
}
}
function getFaceDetectorOptions() {
//new faceapi.SsdMobilenetv1Options({ minConfidence });
if(selected_face_detector == 'tiny_face_detector'){
return new faceapi.TinyFaceDetectorOptions({ inputSize, scoreThreshold });
}
else{
return new faceapi.SsdMobilenetv1Options({ minConfidence });
}
}
onmessage = function(incoming) {
//console.log('Message received from main script:');
//console.log(incoming.data);
var return_message = '';
if(incoming.data.type == 'frame'){
console.log("worker: incoming frame");
use(selected_use, incoming.data)
}
//else if(Array.isArray(incoming.data)){
else {
console.log("worker: incoming init message");
if(incoming.data.net !== undefined){
if(incoming.data.net == 'tiny'){
selected_face_detector = 'tiny_face_detector';
load_tiny_net();
console.log("worker: loading tiny face net");
}
else if(incoming.data.net == 'mobilenet'){
selected_face_detector = 'ssd_mobilenetv1';
load_mobile_net();
console.log("worker: loading mobile face net");
}
//postMessage('net loaded');
}
if(incoming.data.load !== undefined){
incoming.data.load.forEach(function (item, index) {
//console.log("loading model:");
//console.log(index, item);
if(item == 'age_and_gender'){
load_age_and_gender();
console.log('loading age and gender model');
}
if(item == 'expression'){
load_expression();
console.log('loading expression model');
}
if(item == 'recognition'){
load_recognition();
console.log('loading face recognition model');
}
if(item == 'landmarks'){
load_landmarks();
console.log('loading landmarks model');
}
//console.log('loading ' + item);
//postMessage("loaded");
});
}
if(incoming.data.use !== undefined){
selected_use = incoming.data.use;
console.log("selected_use is now set to " + selected_use);
}
}
}
async function use(use_case, image_data){
console.log("in worker USE. use_case = " + use_case + ", imagedata:");
console.log(image_data);
// props is the message from the main thread
const imgData = new ImageData(
new Uint8ClampedArray(image_data.data),
image_data.width,
image_data.height
);
console.log(imgData);
// Create a canvas from our rgbaBuffer
const img = faceapi.createCanvasFromMedia(imgData);
//const results = await faceapi.detectAllFaces(img, this.faceDetectorOptions)
var result = {};
if(use_case == 'face'){
result = await faceapi.detectSingleFace(img, detector_options);
}
else if(use_case == 'landmarks'){
result = await faceapi.detectSingleFace(img, detector_options).withFaceLandmarks();
try {
const jawOutline = result.landmarks.getJawOutline();
const nose = result.landmarks.getNose();
const mouth = result.landmarks.getMouth();
const leftEye = result.landmarks.getLeftEye();
const rightEye = result.landmarks.getRightEye();
const leftEyeBrow = result.landmarks.getLeftEyeBrow();
const rightEyeBrow = result.landmarks.getRightEyeBrow();
postMessage({
"landmarks":result.landmarks.positions,
"jawOutline":jawOutline,
"nose":nose,
"mouth":mouth,
"leftEye":leftEye,
"rightEye":rightEye,
"leftEyeBrow":leftEyeBrow,
"rightEyeBrow":rightEyeBrow,
"faceparts":true
});
}
catch (e) {
postMessage({"faceparts":false});
}
}
else if(use_case == 'age_and_gender'){
result = await faceapi.detectSingleFace(img, detector_options).withFaceLandmarks().withAgeAndGender();
}
else if(use_case == 'expression'){
result = await faceapi.detectSingleFace(img, detector_options).withFaceLandmarks().withFaceExpressions()
}
else if(use_case == 'recognition'){
result = await faceapi.detectSingleFace(img, detector_options).withFaceLandmarks().withFaceDescriptor();
}
else if(use_case == 'all'){
result = await faceapi.detectSingleFace(img, detector_options).withFaceLandmarks().withFaceExpressions().withAgeAndGender().withFaceDescriptor();
console.log("use case all result was: ");
console.log(result);
}
else{
console.log("Error: no fitting use case provided");
postMessage({"error":"selected use does not exist"});
}
if( result === undefined){
result = {'error':'result undefined'};
}
postMessage(result);
}That file also loads second patch file called faceEnvWorkerPatch.js to make faceapijs work as a webworker. It contains:
// From: https://github.com/justadudewhohacks/face-api.js/issues/47
self.Canvas = self.HTMLCanvasElement = OffscreenCanvas;
self.HTMLCanvasElement.name = 'HTMLCanvasElement';
self.Canvas.name = 'Canvas';
self.CanvasRenderingContext2D = OffscreenCanvasRenderingContext2D;
function HTMLImageElement(){}
function HTMLVideoElement(){}
self.Image = HTMLImageElement;
self.Video = HTMLVideoElement;
// Canvas.prototype = Object.create(OffscreenCanvas.prototype);
function Storage () {
let _data = {};
this.clear = function(){ return _data = {}; };
this.getItem = function(id){ return _data.hasOwnProperty(id) ? _data[id] : undefined; };
this.removeItem = function(id){ return delete _data[id]; };
this.setItem = function(id, val){ return _data[id] = String(val); };
}
class Document extends EventTarget {}
let window, document = new Document();
window = self.Window = self;
self.localStorage = new Storage();
function createElement(element) {
// console.log('FAKE ELELEMT instance', createElement, 'type', createElement, '(', createElement.arguments, ')');
switch(element) {
case 'canvas':
// console.log('creating canvas');
let canvas = new Canvas(1,1);
canvas.localName = 'canvas';
canvas.nodeName = 'CANVAS';
canvas.tagName = 'CANVAS';
canvas.nodeType = 1;
canvas.innerHTML = '';
canvas.remove = () => { console.log('nope'); };
// console.log('returning canvas', canvas);
return canvas;
default:
console.log('arg', element);
break;
}
}
document.createElement = createElement;
document.location = self.location;
//console.log('*faked* Document object for the worker', document);
if(!typeof window == 'object') {
console.warn("Check failed: window");
}
if(typeof document === 'undefined') {
console.warn("Check failed: document");
}
if(typeof HTMLImageElement === 'undefined') {
console.warn("Check failed: HTMLImageElement");
}
if(typeof HTMLCanvasElement === 'undefined') {
console.warn("Check failed: HTMLCanvasElement");
}
if(typeof HTMLVideoElement === 'undefined') {
console.warn("Check failed: HTMLVideoElement");
}
if(typeof ImageData === 'undefined') {
console.warn("Check failed: ImageData");
}
if(typeof CanvasRenderingContext2D === 'undefined') {
console.warn("Check failed: CanvasRenderingContext2D");
}
self.window = window;
self.document = document;
self.HTMLImageElement = HTMLImageElement;
self.HTMLVideoElement = HTMLVideoElement;
const isBrowserCheck = typeof window === 'object'
&& typeof document !== 'undefined'
&& typeof HTMLImageElement !== 'undefined'
&& typeof HTMLCanvasElement !== 'undefined'
&& typeof HTMLVideoElement !== 'undefined'
&& typeof ImageData !== 'undefined'
&& typeof CanvasRenderingContext2D !== 'undefined';
;
// console.warn("++ Check passed locally:", isBrowserCheck);
if(!isBrowserCheck) {
throw new Error("Failed to monkey patch for face-api, face-api will fail");
}
The webworker is interacted with from the main script. Here is some of that code:
// If the browser supports web workers...
if (typeof(Worker) !== "undefined") {
window.faceapi_worker = new Worker('faceapi-worker.js');
window.faceapi_worker.onmessage = function(incoming) {
console.log('Message received from faceapi worker .data: ' + JSON.stringify(incoming.data,null,2));
if( 'data' in incoming ){
if(incoming.data === undefined){
console.log("incoming.data was undefined?")
}
else if('loaded' in incoming.data ){
console.log("incoming message that ML was loaded");
}
else if('detection' in incoming.data ){
console.log(incoming.data); // Do something with the data here.
const face_box = incoming.data.detection['_box']; // example
}
else{
console.log("No detection data in incoming data..");
}
}
else{
console.log("undefined result from worker.");
}
}
window.faceapi_worker.postMessage({'net':'tiny','load':['landmarks','expression','age_and_gender','recognition'],'use':['all']});
}And here's the part of the main code that sends a webcam frame to the webworker:
canvy_context.drawImage(webcam_video,0,0,640,480);
var imgData2 = canvy_context.getImageData(0, 0, 640, 480);
console.log(imgData2);
const { height, width, data } = imgData2;
console.log("height:" + height);
// Transfer the buffer via a transferlist. See https://stackoverflow.com/questions/41497124/allowable-format-for-webworker-data-transfer
window.faceapi_worker.postMessage({
type: "frame",
height,
width,
data,
}, [ data.buffer ]);
rlaurente
sprengerst
remipassmoilesel
josiahbryan
cindyloo
jeffreytgilbert
ScottDellinger
aendra-rininsland
zy308718320
Waishnav
Hi there! Great work on this plugin! Has anybody managed to run this in a webworker?