search

ioBroker / AdapterRequests

This Place is used to track the status of new Adapter-Requests.
248 stars 36 forks source link

Face detection adapter #131

Open GermanBluefox opened 5 years ago

GermanBluefox commented 5 years ago

With https://github.com/justadudewhohacks/face-api.js

const path = require('path');
const trainedModelFile = 'model_08.json';
//const trainedModelDirPath = '/opt/iobroker/node_modules/iobroker.javascript/node_modules/';
//const dataPath = path.resolve('/opt/iobroker/iobroker-data/files/camera/input');
//const facepath =  '/opt/iobroker/iobroker-data/files/camera/faces/';

const dataDir = '/opt/iobroker/iobroker-data/files';
const facepath =  dataDir+'/camera/faces/';
const trainedModelDirPath = dataDir+'/camera/conf/';
const dataPath = dataDir+'/camera/input';

const percent_prob = 10;
const days_history = 60;  //Количество дней хранения рапознанных лиц в БД и на диске (кроме обученных)
const fs = require('fs');
const fr = require('face-recognition');
const detector = fr.FaceDetector();
const recognizer = fr.FaceRecognizer();
const recognizer_new = fr.FaceRecognizer();
var db_faces;
var array_checked = [];
var model = [];
var model_new = [];

const init_user = {"className":"inituser","faceDescriptors":[[
0.05138051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634
],[
0.05838051210860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,
0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634,0.05838051810860634
]]};

var init_array = [];
init_array.push(init_user);

fs.stat(trainedModelDirPath+trainedModelFile, function(err, stat) {
    if(err === null) {
        model= JSON.parse(fs.readFileSync(trainedModelDirPath+trainedModelFile));
        recognizer.load(model);
        console.log('face_db='+JSON.stringify(recognizer.getDescriptorState()));
        db_faces = recognizer.getDescriptorState();
        setState('javascript.0.fr_model',JSON.stringify(model));

    }
    else if(err.code == 'ENOENT') {
        const modelinit = init_array;
        console.log('Create '+trainedModelFile +' with init_db ='+JSON.stringify(recognizer.getDescriptorState()));
        fs.writeFileSync(trainedModelDirPath+trainedModelFile, JSON.stringify(modelinit));
        model= JSON.parse(fs.readFileSync(trainedModelDirPath+trainedModelFile));
        recognizer.load(model);
        console.log('new_face_db='+JSON.stringify(recognizer.getDescriptorState()));
        db_faces = recognizer.getDescriptorState();
    }

    var arr3 = {};

    model.forEach(function(element_U,index_U,array_U) {
        var user = model[index_U]['className'];
        var arr2 = (model[index_U]['faceDescriptors']);
        arr2.forEach(function(element_F,index_F,array_F) {
            var new_index=user+ '__'+index_F;
            arr3 = {"className":new_index,"faceDescriptors":[element_F,element_F]};
            model_new.push(arr3);
        });  
    });
    setState('javascript.0.fr_model_alt',JSON.stringify(model_new));
    recognizer_new.load(model_new);
    var query2='select id, filename_face, face_desc from iobroker.faces where checked=1';
    sendTo('sql.0', 'query',query2 , function (result) {
                                if (result.error) {
                                    console.error(result.error);
                                    }
                                else
                                    {
                                         var obj = JSON.stringify(result.result);
                                         obj = JSON.parse(obj );  
                                         if (obj.length>0)
                                             {
                                                array_checked=obj;
                                                //console.log('array_checked[0]='+JSON.stringify(array_checked[0]));
                                                setState('javascript.0.fr_array_checked',JSON.stringify(array_checked));
                                             }
                                    }
                            });

});

var FileArray;
var InputImgs;
var Timestamp;  

on({id: "javascript.0.fr_model", change: 'any'}, function (objsp) {
    console.log('Update model');
    model= JSON.parse(getState("javascript.0.fr_model").val);
    recognizer.load(model);
    model_new= JSON.parse(getState('javascript.0.fr_model_alt').val);
    recognizer_new.load(model_new);
    array_checked = JSON.parse(getState("javascript.0.fr_array_checked").val);

});

schedule("0 * * * *", function () {

    var query2 =  'SELECT filename_face FROM  iobroker.faces  where checked<>1 and datetime < DATE_SUB(now(), INTERVAL '+days_history+' DAY)';
    console.log(query2);    
    sendTo("sql.0", "query", query2, function (result){
    if (result.error) {
        console.error(result.error);
        }
    else
        {
            var obj = JSON.stringify(result.result);
            obj = JSON.parse(obj );
            if (obj.length>0)
                {
                    for(var i=0;i<obj.length;i++)
                        {
                          fs.unlink(obj["filename_face"], function(err){
                            if (err) throw err;
                            console.log(obj["filename_face"] + "deleted");

                            });
                        }
                    var query3 =  'DELETE FROM  iobroker.faces  where checked<>1 and datetime < DATE_SUB(now(), INTERVAL '+days_history+' DAY)';
                    //console.log(query3);    
                    sendTo("sql.0", "query", query3, function (result){
                        if (result.error) {
                            console.error(result.error);
                            }
                    });

                }

        }
    });
});

function processFiles()
{  
    //console.log('face_db='+JSON.stringify(recognizer.getDescriptorState()));
   // console.log('face_db_new='+JSON.stringify(recognizer_new.getDescriptorState()));
    FileArray = fs.readdirSync(dataPath)
    var SmallArray =[];
    var max_load = 1 ;
    var method_alt = getState("javascript.0.fr_method").val;

    if (FileArray.length >0)
        {
            SmallArray=FileArray.slice(0,max_load);
            InputImgs = SmallArray
            .map(f => path.join(dataPath, f))
            .map(fp => fr.loadImage(fp));
            Timestamp = SmallArray
            .map(f => fs.statSync(dataPath +'/'+ f).mtime.toISOString()   );
            for (const [i, _img] of InputImgs.entries() )    {
                var start_rec = new Date();
                let img = _img;
                var start_detect = new Date();
                const faceRects = detector.locateFaces(img).map(res => res.rect);
                var filename_src = dataPath+'/'+FileArray[i];
                const unknownThreshold = (100-percent_prob)/100;
                console.log('faceRects.length='+faceRects.length);
                faceRects.forEach((rect, t) => {
                    console.log('Face detected!');
                    //const faceImages = detector.detectFaces(img);
                    const faces = detector.getFacesFromLocations(img, faceRects, 150);
                    console.log('End detectFaces='+new Date());
                    var event_date_sql;
                    var face_name;
                    var prob;
                    var calc_time;
                    var filename_rec='';
                    var end_detect;

                    if (method_alt === true)
                        {
                            console.log('Use alt model');
                           // const prediction = recognizer_new.predictBest( faceImages[t], unknownThreshold);
                           const prediction = recognizer_new.predictBest( faces[t], unknownThreshold);
                            console.log('face_name alt ='+prediction.className);
                            event_date_sql = formatDate(Timestamp[i], "JJJJ-MM-TT SS:mm:ss.sss");
                            face_name = prediction.className;
                            prob = Math.round((1-prediction.distance)*100,0);
                            filename_face = facepath+Timestamp[i]+'_'+t+'_face.jpg';
                           // fr.saveImage(filename_face,faceImages[t]);
                            fr.saveImage(filename_face,faces[t]);
                            end_detect = new Date();
                            calc_time = (end_detect-start_detect)/1000;
                            var new_name= face_name;
                            var arr_2;
                            var fase_desc_rec;
                            // переместил вниз. строка 225 // console.log("Обнаружено лицо "+face_name);
                            // оригинальная строка, видимо с процентами console.log("Обнаружено лицо "+'face_name='+face_name);
                            model_new.forEach(function(element_U,index_U,array_U) {
                                var user = model_new[index_U]['className'];
                                if (user == face_name)
                                    {
                                    arr_2 = model_new[index_U]['faceDescriptors'];
                                    fase_desc_rec = '[['+ arr_2[0] + ']]';
                                    }
                            });
                            face_name=face_name.substring( 0, face_name.indexOf( "__" ) );
                            prob = Math.round((1-prediction.distance)*100,0);
                            var kk ='';    
                            console.log("Обнаружено лицо "+face_name);
                            if (face_name === 'Саида')
                                        {
                                            console.log("Отправил телегу Саиде");
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'Привет дорогая, отлично выглядишь!',
                                            caption: 'Добро пожаловать!',
                                            user: 'Саида',
                                            // chatId: '-186628872'
                                                                });
                                        }
                            else if (face_name === 'Руслан')
                                        {
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'О! Руслан приехал.',
                                            caption: 'Добро пожаловать!',
                                            // user: 'Anzor',
                                            chatId: '-186628872'
                                                                });
                                        }

                            else if (face_name === 'Валя')
                                        {
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'Валя пришла.',
                                            caption: 'Добро пожаловать!',
                                            // user: 'Anzor',
                                            chatId: '-186628872'
                                                                });
                                        }

                            else if (face_name === 'Женя_тварь')
                                        {
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'Женя.',
                                            caption: 'Добро пожаловать!',
                                            // user: 'Anzor',
                                            chatId: '-186628872'
                                                                });
                                        }

                            else if (face_name === 'Тимур')
                                        {
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'О! Тимур приехал.',
                                            caption: 'Добро пожаловать!',
                                            // user: 'Anzor',
                                            chatId: '-186628872'
                                                                });
                                        }
                            else if (face_name === 'Аня')
                                        {
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'Кажется, я узнал Аню!',
                                            caption: 'Добро пожаловать!',
                                            // user: 'Anzor',
                                            chatId: '-186628872'
                                                                });
                                        }

                            else if (face_name === 'Марат')
                                        {
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'О! Марат приехал.',
                                            caption: 'Добро пожаловать!',
                                            // user: 'Anzor',
                                            chatId: '-186628872'
                                                                });
                                        }

                            else if (face_name === 'Света')
                                        {
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'О! Света.',
                                            caption: 'Добро пожаловать!',
                                            // user: 'Anzor',
                                            chatId: '-186628872'
                                                                });
                                        }

                            else if (face_name === 'Раиса')
                                        {
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'О! Теща! Бегом стол накрывать!',
                                            caption: 'Добро пожаловать!',
                                            // user: 'Anzor',
                                            chatId: '-186628872'
                                                                });
                                        }
                            else if (face_name === 'Анзор')
                                        {
                                            console.log("Отправил в телегу месагу Анзору");
                                            setState("mqtt.0.homebridge.vto_TargetState"/*vto_lock_TargetState*/, 0);
                                            sendTo('telegram.0', {
                                            text: 'Хозяин, мое почтение!',
                                            caption: 'Добро пожаловать!',
                                            user: 'Anzor',
                                            //chatId: '-186628872'
                                                                });
                                        }
                            if (face_name !== 'inituser' & face_name !== 'unknown' & face_name !== '')
                                        {
                                            array_checked.forEach(function(el,ind,array) {

                                                if ( array_checked[ind]['face_desc'] == fase_desc_rec )
                                                    kk = ind;
                                             });
                                            if (kk)
                                                filename_rec = array_checked[kk]['filename_face'];
                                        }

                            var query1 =  'INSERT INTO  iobroker.faces (datetime, filename_src, face_name, filename_face,prob,calc_time,filename_rec) VALUES (\'' + String(event_date_sql)+ '\',\'' + filename_src + '\',\'' + face_name + '\',\'' + filename_face+ '\',' + prob + ','+calc_time+',\'' + filename_rec + '\')';
                            console.log(query1);                
                            sendTo("sql.0", "query", query1, function (result){
                                if (result.error) {
                                    console.error(result.error);
                                }
                                else
                                    {
                                    if (face_name == 'unknown')
                                        {
                                        }
                                    else
                                        {
                                        //var saytext = "Обнаружено лицо " + face_name;
                                        // setState("sayit.0.tts.text",'ru;60;'+saytext);
                                        }

                                    }

                            });

                        } //if method_alt
                    else
                        {
                        const prediction = recognizer.predictBest(faces[t], unknownThreshold);
                        console.log('Use classic model');
                        event_date_sql = formatDate(Timestamp[i], "JJJJ-MM-TT SS:mm:ss.sss");
                        face_name = prediction.className;
                        prob = Math.round((1-prediction.distance)*100,0);
                        filename_face = facepath+Timestamp[i]+'_'+t+'_face.jpg';
                        fr.saveImage(filename_face,faces[t]);
                        end_detect = new Date();
                        calc_time = (end_detect-start_detect)/1000;
                        var query2 =  'INSERT INTO  iobroker.faces (datetime, filename_src, face_name, filename_face,prob,calc_time,filename_rec) VALUES (\'' + String(event_date_sql)+ '\',\'' + filename_src + '\',\'' + face_name + '\',\'' + filename_face+ '\',' + prob + ','+calc_time+',\'' + filename_rec + '\')';
                        console.log(query2);    
                        sendTo("sql.0", "query", query2, function (result){
                                if (result.error) {
                                    console.error(result.error);
                                }
                                else
                                    {
                                    if (face_name == 'unknown')
                                        {
                                        }
                                    else
                                        {
                                        console.log("Обнаружено лицо2 " + face_nam);
                                        //var saytext = "Обнаружено лицо " + face_name;
                                        // setState("sayit.0.tts.text",'ru;60;'+saytext);
                                        }
                                    }
                            });
                    }   //classic method

                }); //faceRects.forEach
                fs.unlink(filename_src, function(err){
                    if (err) throw err;
                   // console.log(filename_src + " parsed and deleted");
                });
            } //for
        } //if
    setTimeout(processFiles, 1*100);
} //processFiles

setTimeout(processFiles, 1*100);
Camouflagge commented 4 years ago

Hay, is there a manual or similar for this script to integrate it into iobroker.

uwe72 commented 3 years ago

@GermanBluefox @DutchmanNL

Is there a more up-to-date example available? i want to use faceDetection within a iobroker script