classify pose using mediapipe handpose model and ml5js

[kurnyaannn]

Hi.

i'm using ml5js to classify handpose from Mediapipe handpose model, which detect 21 keypoints, so if there's only one hand to detect, the ml5.neuralNetwork(options) should be :

let options = {
    inputs: 42, //(for each x, and y)
    ....
}

so if i wanted to detect 2 hands the inputs should be doubled to 84. The data collection, and data classification works as expected.

but at data training section i'm having an issue here, so when i collect the data for single hand (either left or right hand) the data training was successfull and classification was fine, but when i collect the data for both hands (left and right at the same time) the data training didn't go as expected, it's just go flat and never going down. I checked the collected data, and i don't find anything suspicious, and i couldn't tell any different between single hand and multihands either except the input length between them.

even if i set the options inputs to 84, i'm still able to collect, train, and classify for single hand with that options, but why the same things didn't happen for multiple hands ?.

I'm opening an issue because i'm curious, maybe i missed something that the Mediapipe has not documented it yet. Maybe there's something i didn't know about the output, or maybe something else. I've been trying to fix this for several days now, so i'm hoping someone here can give me some clue or anything. Here's the main function to collect the data.

// declare variable
let options = {
  inputs: 84,
  output: 4,
  task: 'classification',
  debug: true
};
let neural = ml5.neuralNetwork(options);
...

// main function
function onResults(results) {
  document.body.classList.add('loaded');

  fpsControl.tick();

  canvasCtx.save();
  canvasCtx.clearRect(0, 0, canvasElement.width, canvasElement.height);
  canvasCtx.drawImage(results.image, 0, 0, canvasElement.width, canvasElement.height);
  if (results.multiHandLandmarks && results.multiHandedness) {
    for (let i = 0; i < results.multiHandLandmarks.length; i++) {
      const classification = results.multiHandedness[i];
      const isRightHand = classification.label === 'Right';
      const landmarks = results.multiHandLandmarks[i];
      drawConnectors(canvasCtx, landmarks, HAND_CONNECTIONS, {
        color: isRightHand ? '#00FF00' : '#FF0000'
      }), drawLandmarks(canvasCtx, landmarks, {
        color: isRightHand ? '#00FF00' : '#FF0000',
        fillColor: isRightHand ? '#FF0000' : '#00FF00',
        radius: (x) => {
          return lerp(x.from.z, -0.15, .1, 10, 1);
        }
      });
    }
    // data collecting process
    if (state == 'collecting') {
      let inputs = [];
      for (let i = 0; i < results.multiHandLandmarks.length; i++) {
        for (let j = 0; j < results.multiHandLandmarks[i].length; j++) {
          let x = results.multiHandLandmarks[i][j].x;
          let y = results.multiHandLandmarks[i][j].y;
          inputs.push(x);
          inputs.push(y);
          console.log(x, y);
        }
      }
      let label = [targetLabel];
      console.log(inputs);
      neural.addData(inputs, label);
    }
  }
  canvasCtx.restore();
  return results;
}
...

[kurnyaannn]

UPDATE

Because i couldn't find any difference between single and multiple hand data. So i'm trying to normalize the data both of them first using normalizeData() function from ml5js, even though i didn't need to because mediapipe already done that, but because i still haven't found any clue so i did it just to make sure, and as i thought, there's something.

So when i normalize the single hand data it works as expected just like before. No error, no warning. But when i normalize the multihands data, there's an error on the console says : Uncaught (in promise) Error: Error: error in inputArray of normalizeArray() function

I tried to googled it, but can't find anything helpfull so far.

[kurnyaannn]

UPDATE_2

Nevermind guys, i fixed it.

   if (results.multiHandLandmarks.length > 1) {
      if (state == 'collecting') {
        let inputs = [];
        for (let i = 0; i < results.multiHandLandmarks.length; i++) {
          for (let j = 0; j < results.multiHandLandmarks[i].length; j++) {
            let x = results.multiHandLandmarks[i][j].x;
            let y = results.multiHandLandmarks[i][j].y;
            inputs.push(x);
            inputs.push(y);
          }
        }
        let label = [targetLabel];
        console.log(inputs);
        neural.addData(inputs, label);
      }
    } else {
      if (state == 'collecting') {
        let inputs = [];
        for (let i = 0; i < results.multiHandLandmarks.length; i++) {
          for (let j = 0; j < results.multiHandLandmarks[i].length; j++) {
            let x = results.multiHandLandmarks[i][j].x;
            let y = results.multiHandLandmarks[i][j].y;
            inputs.push(x);
            inputs.push(y);
          }
        }
        let label = [targetLabel];
        console.log(inputs);
        neural.addData(inputs, label);
      }
    }

i don't know why this is working, but yeah this is javascript anyway.