[featureExtractor] featureExtractor for images from file

jcranney commented 6 years ago

I'm running into an error when trying to train the ml5.featureExtractor('MobileNet') on images from file. The working example in this repository uses createCapture(VIDEO), but I get the following when trying to add an image from createImg("bird.jpg"):

Model ready!         sketch.js:17
Image ready!         sketch.js:22
backend_webgl.js:124 Uncaught (in promise) Error: MathBackendWebGL.writePixels(): pixels can not be null
    at e.fromPixels (backend_webgl.js:124)
    at e.fromPixels (engine.js:293)
    at e.fromPixels (array_ops.js:280)
    at operation.js:9
    at e.tidy (tracking.js:34)
    at Object.n.value (operation.js:9)
    at imageUtilities.js:67
    at Object.e.tidy (tracking.js:34)
    at t.imgToTensor (imageUtilities.js:66)
    at Mobilenet.js:124

The toy example used to generate this error is this:

var featureExtractor;
var regressor;
var img;

function setup() {
    createCanvas(340,280);
    img = createImg("bird.jpg");
    img.hide();
    featureExtractor = ml5.featureExtractor('MobileNet', modelReady);
}

function draw(){
    image(img,0,0,340,280);
}

function modelReady() {
    console.log("Model ready!");
    regressor = featureExtractor.regression(img, imageReady);
}

function imageReady() {
    console.log("Image ready!");
    regressor.addImage(1);
}

I'm running locally on a live-server and the exact same code works fine when you replace img = createImg("bird.jpg") with img = createCapture(VIDEO). The bird image loads and displays fine. The following is the index.html file for completeness:

<html>
<head>
  <meta charset="UTF-8">
  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.6.0/p5.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/p5.js/0.6.0/addons/p5.dom.min.js"></script>
  <script src="https://unpkg.com/ml5@0.1.1/dist/ml5.min.js" type="text/javascript"></script>
  <script src="sketch.js"></script>
</head>
</html>

cvalenzuela commented 6 years ago

There's a bug when using p5 images: https://github.com/ml5js/ml5-library/issues/208 If you create a DOM image, instead of a p5 Image, that should work for now.

poud commented 5 years ago

I have the same problem. I tried with DOM image instead but it did not work, any solution yet?

jcranney commented 5 years ago

I couldn't get it working with a DOM image either. Supposedly fixed in #191 but I haven't tested.

samfultonjr commented 5 years ago

Seems to not work with a DOM image or p5 image. Has anyone come up with another solution?

ComputerCarl commented 5 years ago

Please note this is something I hacked together, but it does work (slowly);

/*
@article{Xie2015SCUTFBPAB,
  title={SCUT-FBP: A Benchmark Dataset for Facial Beauty Perception},
  author={Duorui Xie and Lingyu Liang and Lianwen Jin and Jie Xu and Mengru Li},
  journal={2015 IEEE International Conference on Systems, Man, and Cybernetics},
  year={2015},
  pages={1821-1826}
}
*/

const imgNamer = i => `./Data_Collection/SCUT-FBP-${i}.jpg`;
const promises = [];
const totalImages = 500;
const trainingImages = 450;

function setup() {
    // don't even know what this does, but not using P5
    noCanvas();
    useFeatures();
}

async function useFeatures() {
    const featureExtractor = await ml5.featureExtractor('MobileNet');
    // https://ml5js.org/docs/custom-regression
    const classifier = await featureExtractor.regression();
    // const classifier = await featureExtractor.classification();

    // there are 500 images in the set. We will train on some. Very slow.
    for (var i = 1; i < trainingImages; i++) {
        const label = attrMeta[i].attr;

        //using vanilla js
        promises[i] = new Promise(resolve => {
            var img = new Image();
            img.src = imgNamer(i);
            // the secret is that addImage is async
            classifier.addImage(img, label, () => {
                resolve();
                img = null;
            });
        }); // end promises

        // Does not work with P5
        // Error: pixels passed to tf.fromPixels() can not be null
        // promises[i] = new Promise(resolve => {
        //     loadImage(imgNamer(i), img => {
        //         classifier.addImage(img, label, () => {
        //             resolve();
        //         });
        //     });
        // }); // end promises
    }

    // when all images have been added to the regressor, train and predict
    Promise.all(promises)
        .then(() => {
            console.log('Training.');
            classifier.train(lossValue => {
                if (lossValue) {
                    // training
                    console.log(lossValue);
                } else {
                    // done
                    var img = new Image();
                    const makeNum = () => Math.round(Math.random() * (totalImages - trainingImages )) + trainingImages;
                    img.src = imgNamer(makeNum());
                    const el = document.querySelector('#pic');
                    el.src = img.src;
                    classifier.predict(img, (err, result) => {
                        if (err) return console.warn(err);
                        // this should be labels
                        console.log('rating: ' + result);
                    });
                }
            });
        });
}

rnlewis commented 4 years ago

Is this still an issue? I have been trying to do classifier.addImage(img, label) where img is a p5 Image, and receive the errors Mobilenet.js:260 Uncaught (in promise) Error: Add some examples before training! and Uncaught (in promise) Error: pixels passed to tf.browser.fromPixels() can not be null.

ishback commented 4 years ago

I still have this issue. I've tried to pass an p5 Image and also a pg = createGraphics(100, 100) where I record doodles, but get the same error. Any suggestions?

BeetorBot commented 4 years ago

I have the same issue. I'm trying to add a series of images from a JSON file but I get the need to add examples error. I looked everywhere online but no one was able to add images to and train them only live video.

function modelReady(){
  console.log('ready');
  classifier = mobilenet.classification();
   for (let n = 0; n < 2; n++){// 4 directiories
    for (let i = 0; i < 6; i++){ // with 9 images each
     //Load the image
     imgPath = data.child[n].children[i].path;
     PaintingType = data.child[n].children[i].type;
     img = createImg(imgPath);
     img.hide();
     image(img,0,0);
     classifier.addImage(PaintingType);
   //  classifier.addExample(img, PaintingType);
    }

joeyklee commented 4 years ago

Hi @BeetorBot - Sorry to hear you're having this issue. Here's a few suggestions.

Load your images using the preload() function

const imageList = ['img1.png', 'img2.png'];
let myImages = [];
function preload(){
 myImages = imageList.map(img => loadImage(img));
}

function setup(){
classifier = mobilenet.classification();
// ... the rest of your code
}

createImg is not synchronous as far as I understand it so you'd likely have to add a callback to handle when the image has completed loading.

Hope this helps.

BeetorBot commented 4 years ago

Hey thanks for the replying but I still couldn't get it to work. I'm not sure if that's on me or ml5. I tried a couple of different variations of your code but I guess it was never meant to able to add your own dataset. Here is my full code if want to read.

///<reference path="./p5.global-mode.d.ts" />

let mobilenet;
let classifier;
var data;
let imgPath;
let PaintingType;
let img;
const imageList = ['Van Gogh/0.jpg', 'Van Gogh/1.jpg', 'Van Gogh/2.jpg', 'Van Gogh/3.jpg', 'Van Gogh/4.jpg', 'Van Gogh/5.jpg', 'Van Gogh/6.jpg', ];
let myImages = [];

function preload(){
//data = loadJSON("Painting.JSON");
myImages = imageList.map(img => loadImage(img));
}

function setup() {
  createCanvas(400, 400);
  background(0);
  mobilenet = ml5.featureExtractor('mobilenet', modelReady);

}

function modelReady(){
  console.log('ready');
  classifier = mobilenet.classification();
   //for (let n = 0; n < 1; n++){// 4 directiories
    for (let i = 0; i < 6; i++){ // with 9 images each
     //Load the image
    // imgPath = data.child[n].children[i].path;
     //PaintingType = data.child[0].children[i].type;
    //img = createImg(myImages[i], '');
    // img.hide();
  //image(myImages[i],0,0);
     classifier.addImage(myImages[i], 'Van Gogh');
   //  classifier.addExample(img, PaintingType);
    }
   //}
const trainer = classifier.train(function(lossValue) {
  console.log('Loss is', lossValue)
});
}

joeyklee commented 4 years ago

Hi @BeetorBot, A few things to ensure before continuing:

Can you confirm that your images are loaded?
The spaces in the file names is concerning. Can you also ensure that this isn't causing issues?

BeetorBot commented 4 years ago

Yes they are loading. I did a bunch of trail ones where it just draws the image to the canvas with none of the machine learning. It works just fine. But just for the heck of it, I change the file names so they won't have spaces, and its the same error. Anyways thanks for your help I didn't think anyone would have replied.

Here the errors I get.

Mobilenet.js:271 Uncaught (in promise) Error: Add some examples before training!

tf-core.esm.js:17 Uncaught (in promise) Error: pixels passed to tf.browser.fromPixels() can not be null

On Fri, May 22, 2020 at 12:07 PM Joey Lee notifications@github.com wrote:

Hi @BeetorBot https://github.com/BeetorBot, A few things to ensure before continuing:

Can you confirm that your images are loaded?

The spaces in the file names is concerning. Can you also ensure that this isn't causing issues?

— You are receiving this because you were mentioned. Reply to this email directly, view it on GitHub https://github.com/ml5js/ml5-library/issues/887#issuecomment-632864037, or unsubscribe https://github.com/notifications/unsubscribe-auth/APHMIK23NX7OFHW4XJ7NUWLRS3ENZANCNFSM4MBFZ4HQ .

joeyklee commented 4 years ago

@BeetorBot - Thanks for all your effort in trying to debug this. I will try to make an example to figure out what is going on and share back what I learn. Likely will get to this sometime this coming week. Sorry for the inconvenience!

As an interim solution, you might try to use Google Teachable Machine https://teachablemachine.withgoogle.com/ to generate your custom image classification model.

CarolCPR commented 4 years ago

Hi, friends

I'm also trying to use an featureExtractor to train with my images, below my code

let mobilenet;
let classifier;
const imageList = ['cats/cat.jpg','cats/cat2.jpg','cats/cat3.jpg','cats/cat4.png','cats/cat5.jpg','cats/cat6.jpg','cats/cat7.jpeg','cats/cat8.jpg','cats/cat9.jpg','cats/cat10.jpg','cats/cat11.jpg','cats/cat12.jpg','cats/cat13.jpg','cats/cat14.jpg','cats/cat15.jpg','cats/cat16.jpg'];
let myImages = [];
let img;
let cat;

function preload(){
  console.log('Adicionando exemplos!');
  myImages = imageList.map(img => loadImage(img));
}

function whileTraining(loss){
  for(let i=0; i<16; i++){
    classifier.addImages(myImages[i], 'cat');
    console.log('imagem treinada')
  }
  console.log('Treinando...');
  /*classifier.train(function(lossValue){
    console.log('Loss is', lossValue);
  })*/
  classifier.predict(gotResults);
}
function setup() {
  noCanvas();
  cat = createImg('cats/promisecat.jpg');
  cat.hide();
  background(0);
  mobilenet = ml5.featureExtractor('MobileNet');
  classifier = mobilenet.regression(cat, imageReady);
  cat.elt.crossOrigin = "Anonymous";
}

function imageReady(){
  console.log('Imagem pronta!');
  image(cat, 0, 0, width, height);
}

function gotResults(error, results){
  console.log("fale algo")
  if(error){
    console.error(error);
  }else {
    console.log(results);
    console.log("AAAAAAAAAAAAA")
    /*let label = results[0].className;
    let prob = results[0].probability;
    fill(0);
    textSize(64);
    text(label, 10, height - 100);
    createP(label);
    createP(prob);*/
    classifier.predict(gotResults);
  }
}

BeetorBot commented 4 years ago

@CarolCPR You're on the right track but There are a couple of things wrong with your code. 1) It is classifier.addImage() not classifier.addImages() 2)When you display your cat image you give it the width and the height as parameters however you called noCanvas() so those variables are null. 3) You also add your images in the function called while training. That function was never called so the images are never added. Also, I suggest you name the function differently or at least put that code in ImageReady function because you aren't training yet you're just adding images. Then you train it. 4)createImg() needs two-parameter. For the second parameter just put an empty string.

Warning this code still does not work because of a possible bug in ml5 or p5

`let mobilenet; let classifier; const imageList = ['cats/cat.jpg','cats/cat2.jpg','cats/cat3.jpg','cats/cat4.png','cats/cat5.jpg','cats/cat6.jpg','cats/cat7.jpeg','cats/cat8.jpg','cats/cat9.jpg','cats/cat10.jpg','cats/cat11.jpg','cats/cat12.jpg','cats/cat13.jpg','cats/cat14.jpg','cats/cat15.jpg','cats/cat16.jpg']; let myImages = []; let img; let cat;

function preload(){ console.log('Adicionando exemplos!'); myImages = imageList.map(img => loadImage(img)); }

function whileTraining(loss) { if(loss == null){ console.log('training complete'); classifier.classify(gotResults); } else console.log(loss); }

function setup() { createCanvas(400,400); cat = createImg('cats/promisecat.jpg',''); cat.hide(); background(0); mobilenet = ml5.featureExtractor('mobilenet', modelReady); //cat.elt.crossOrigin = "Anonymous"; }

function modelReady(){ classifier = mobilenet.classification();

console.log('Imagem pronta!'); image(cat, 0, 0, width, height);

for(let i=0; i<16; i++){ classifier.addImage(myImages[i], 'cat'); console.log('imagem treinada') } console.log('Treinando...'); classifier.train(whileTraining); }

I'm not sure what cat.elt.crossOrigin = "Anonymous"; is supposed to do. Anyways I'm not even sure that it is possible for us to add our own images to the ml5 dataset. We can add video elements but for some reason, p5 images don't work, or at least I couldn't get it to work. I'm just waiting for @joeyklee to come back with an example. I'm hoping he is able to get it to work.

bomanimc commented 3 years ago

Closing this for now. Thank you @BeetorBot and @joeyklee for helping to debug issues here!

ml5js / ml5-library

[featureExtractor] featureExtractor for images from file #887