Feature: Padding Input image to square improves face detection

[vladmandic]

Discussed in https://github.com/vladmandic/human/discussions/468

^{Originally posted by **StillTravelling** May 20, 2024} I'm not sure if this is a bug or not but in order for face detection to work consistently I'm having to pad out a HD image. The incoming video is 25 fps. Using human 3.2.2 on node. Here's my code: ``` const Human = require('C:\\TestNode\\node_modules\\@vladmandic\\human\\dist\\human.node-gpu.js').default; const humanConfig = { modelBasePath: 'file://hmodels', debug: false, async: true, filter: { enabled: false }, cacheSensitivity : 0.9, //skipAllowed: true, //skipFrames: 200, face: { enabled: true, detector: { enabled: true, maxDetected: 1, rotation: false, minConfidence: 0.8 }, attention: { enabled: false }, mesh: { enabled: false }, iris: { enabled: false }, description: { enabled: false }, emotion: { enabled: false }, antispoof: { enabled: false }, liveness: { enabled: false }, }, gesture: { enabled: false }, hand: { enabled: false }, body: { enabled: false }, object: { enabled: false }, segmentation: { enabled: false} }; const human = new Human(humanConfig); ... async function tensorToBuffer(tensor) { const data = await tensor.data(); const buffer = Buffer.from(data); return buffer; } function resizeImage(imageTensor, width, height) { return human.tf.image.resizeBilinear(imageTensor, [height, width]); } // Function to extract a region from an image tensor function extractImage(imageTensor, left, top, width, height) { return imageTensor.slice([top, left, 0], [height, width, -1]); } function padImage(imageTensor, targetWidth, targetHeight) { const [height, width, channels] = imageTensor.shape; const top = Math.floor((targetHeight - height) / 2); const bottom = targetHeight - height - top; const left = Math.floor((targetWidth - width) / 2); const right = targetWidth - width - left; return human.tf.pad(imageTensor, [[top, bottom], [left, right], [0, 0]]); } let lastbox; let facedetect = false; async function processImage2(frameToSend, arW, arH, fW, fH){ let extractedImage; const tensor = await human.tf.node.decodeJpeg(frameToSend, 3); // decode jpeg buffer to raw tensor let current_f = img_dim2; let current_ar = use_aspect_ratio; //set it here because use_aspect_ratio might change mid processing when received as msg if(current_ar == 'cover'){ extractedImage = tensor; } else if (['32', '64', '128', '256','512'].includes(current_ar)) //select the center 32 pixels of the image { let nv = parseInt(current_ar); extractedImage = await human.tf.tidy(() => extractImage(tensor, (arW-nv) / 2, (arH-nv) / 2, nv, nv)); } else if (['32t', '64t', '128t', '256t','512t'].includes(current_ar)) //select the center 32 pixels offset height of the image { let nv = parseInt(current_ar.replace('t','')); let nvt = nv*2; extractedImage = await human.tf.tidy(() => extractImage(tensor, (arW-nv) / 2, (arH-nvt) / 2, nv, nv)); } else if(current_ar.includes('face')){ if(busy ) { human.tf.dispose(tensor); console.log("Busy"); return; } busy = true; let nW = arW / current_f; //arW = 1920 current_f = 4 nW = 480 let nH = arH / current_f; //arH = 1080 current_f = 4 nH = 270 const tensor_r = await human.tf.tidy(() => resizeImage(tensor, nW,nH)); //resize to improve performance? const tensor_b = await human.tf.tidy(() => padImage(tensor_r, nW,nW)); //pad to a square to improve face rec -- this definitely improve detection human.tf.dispose(tensor_r); let res; if(current_ar == 'faceInterpolated'){ const res1 = await human.detect(tensor_b); res = await human.next(res1); } else{ res = await human.detect(tensor_b); } //console.log(human.tf.engine().memory()); //console.log(res.performance); if (res?.face?.[0]){ facedetect = true; thebox = res.face[0].box let left = thebox[0]; let top = thebox[1]; let width2 = thebox[2] ; let height2 = thebox[3]; lastbox = thebox; // Ensure the coordinates and dimensions are within the bounds of the original image if (left < 0) left = 0; if (top < 0) top = 0; if (left + thebox[2] > nW) width2 = nW - left; if (top + thebox[3] > nW) height2 = nW - top; extractedImage = await human.tf.tidy(() => extractImage(tensor_b, left, top, width2, height2)); human.tf.dispose(tensor_b); } else{ facedetect = false; extractedImage = tensor; } busy = false; } else{ //contain extractedImage = tensor; } const exBuffer = await tensorToBuffer(extractedImage); //set to buffer so can be used by sharp sharpImage = sharp(exBuffer, { raw: { width: extractedImage.shape[1], height: extractedImage.shape[0], channels: 3 }}); //export as sharp to resize later as tf resize bilinear seems to be horrible human.tf.dispose(tensor); human.tf.dispose(extractedImage); return sharpImage; } ``` Is this is a bug? I can't find anywhere in the documentation the input image should be cropped or padded into a square? Finally performance is great when a face is found, but when a face isn't found, performance drops from 25fps to about 17fps.

[vladmandic]

added in 3.3 alpha release.