n-riesco / ijavascript

IJavascript is a javascript kernel for the Jupyter notebook
Other
2.19k stars 185 forks source link

Continuous canvas graphic update using p5js #131

Closed abargnesi closed 6 years ago

abargnesi commented 6 years ago

Hello! I have been trying out ijavascript and cannot draw to a canvas using p5js.

This codepen shows what I am trying to accomplish in a ijavascript notebook.

Here is what I have tried in a notebook using the asynchronous output pattern.

package.json

{
  "name": "processing.notebook",
  "version": "1.0.0",
  "dependencies": {
    "canvas": "1.6.7",
    "ijavascript": "5.0.20",
    "jsdom": "11.4.0",
    "nel": "latest",
    "p5": "0.5.16",
    "twitter-stream-api": "0.5.2",
    "usertiming": "0.1.8"
  }
}

notebook

// Set up jsdom
"use strict"

const jsdom     = require("jsdom");
const { JSDOM } = jsdom;

// polyfill for window.performance.now()
require('usertiming')

// create div container for p5 using jsdom
var dom = new JSDOM(`<!DOCTYPE html><div id="main"></div>`);
var window = dom.window;
console.info(window);
var document = dom.window.document;
var screen = dom.window.screen;

// load p5 library
var p5 = require("p5");

$$.async();

{
    let console = global.console;
    let $$ = global.$$;

    // Create p5 example    
    var instance = function( p ) {
        p.setup = function() {
            p.createCanvas(400, 400);
        };

        p.draw = function() {
            p.background('red');
            p.rect(30, 20, 55, 55, 5);
            // send the canvas HTML result each time it's drawn
            $$.sendResult($$.html(dom.window.document.getElementById("main").outerHTML));
        };
    };
    new p5(instance, 'main');
}

The output is blank although the canvas object is present in the outputted result within the <div id="main"></div> container.

Maybe for a canvas object there is a different mechanism to update the result in the notebook?

n-riesco commented 6 years ago

The reason why you see a white canvas is that the text representation of a canvas tag doesn't contain any info about the state of the canvas. In this case dom.window.document.getElementById("main").outerHTML yields <canvas id="defaultCanvas0" style="width: 400px; height: 400px;" width="400" height="400"></canvas>.

The solution is to use a library that outputs SVG.


There are other minor issues with the code above. I'm sure $$.sendResult($$.html(dom.window.document.getElementById("main").outerHTML)) doesn't do what you want. $$.html(...) sends an html result to the frontend and it returns undefined. $$.sendResult(undefined) sends undefined to the frontend. Thus $$sendResult($$.html(...)) sends two results to the frontend (an html result and undefined). Most frontends expect only one execution result.


To update an output, Jupyter defines the concept of display. I recently introduced the function $$.display() for this purpose, but I haven't documented it yet (nor it's tested very thoroughly; I've just found issue #132). Here's how your code adapted to use $$.display()

// Set up jsdom
var JSDOM = require("jsdom").JSDOM;

// polyfill for window.performance.now()
require('usertiming')

// create div container for p5 using jsdom
var dom = new JSDOM('<div id="myp5-main">No canvas yet!</div>');
var window = dom.window;
var document = dom.window.document;
var screen = dom.window.screen;

// load p5 library
var p5 = require("p5");

// setup main display
var main = $$.display('myp5-main');
main.html(dom.window.document.getElementById("myp5-main").outerHTML);

var myp5 = new p5(function(p) {
    p.setup = function() {
        p.createCanvas(400, 400);
    };

    p.draw = function() {
        p.background('red');
        p.rect(30, 20, 55, 55, 5);

        // send the canvas HTML result each time it's drawn
        main.html(dom.window.document.getElementById("myp5-main").outerHTML);
    };
}, 'myp5-main');

Again, for this code to work:


One last thing. It isn't a good idea to use const in a notebook (for the same reasons, it isn't a good idea to use it in a REPL).

abargnesi commented 6 years ago

@n-riesco thank you very much for the detailed information. This project and your efforts are valuable to the open source community and I personally appreciate it.


I updated my notebook by using SVG rendering for p5js by including another library.

Now it seems like I can asynchronous update the output and call $$.done(); to stop the output.

package.json

{
  "name": "processing.notebook",
  "version": "1.0.0",
  "dependencies": {
    "canvas": "1.6.7",
    "ijavascript": "5.0.20",
    "jsdom": "11.4.0",
    "nel": "latest",
    "p5": "0.4.13",
    "p5.js-svg": "0.5.2",
    "twitter-stream-api": "0.5.2",
    "usertiming": "0.1.8"
  }
}

notebook

// first cell

// Setup
var JSDOM = require("jsdom").JSDOM;
require("usertiming");

var dom = new JSDOM(`<!DOCTYPE html><div id="main"></div>`);
var window = dom.window;
console.info(window);
var document = dom.window.document;
var screen = dom.window.screen;
var navigator = dom.window.navigator;

// second cell

// load p5 libraries
var p5 = require("p5");
require("p5.js-svg")(p5);

// setup main display
$$.async();
var main = $$.display('main');
main.html(dom.window.document.getElementById("main").outerHTML);

{
    let console = global.console;
    let $$ = global.$$;

    var myp5 = new p5(function(p) {
        let rounded = 10;
        p.setup = function() {
            p.createCanvas(400, 400, 'svg');
            p.background(255);
            p.fill(150);
        };

        p.draw = function() {
            if (Math.random() > 0.90) {
                let red = Math.round(Math.random() * 255);
                let grn = Math.round(Math.random() * 255);
                let blu = Math.round(Math.random() * 255);
                p.background(red, grn, blu);
            }
            if (Math.random() > 0.999) {
                main.text("!!!!!!!!");
                $$.done();
//                 var mainEl = dom.window.document.getElementById('main');|
//                 if (mainEl) {
//                     mainEl.parent.removeChild(mainEl);
//                 }
                myp5.remove();
            }
            p.rect(30, 20, 55, 55, rounded);

            // send the canvas HTML result each time it's drawn
            main.html(dom.window.document.getElementById("main").outerHTML);
        };
    }, 'main');
}

Unfortunately re-running the cell results in this exception in the p5js SVG library:

/home/tony/projects/processing.notebook/node_modules/p5.js-svg/src/p5.RendererSVG.js:20
        parent.replaceChild(svgCanvas.getElement(), elt);
               ^

TypeError: parent.replaceChild is not a function
    at new RendererSVG (/home/tony/projects/processing.notebook/node_modules/p5.js-svg/src/p5.RendererSVG.js:20:16)
    at p5.createCanvas (/home/tony/projects/processing.notebook/node_modules/p5.js-svg/src/rendering.js:86:44)
    at p5.createCanvas (/home/tony/projects/processing.notebook/node_modules/p5.js-svg/src/rendering.js:83:38)
    at p5.createCanvas (/home/tony/projects/processing.notebook/node_modules/p5.js-svg/src/rendering.js:83:38)
    at p5.p.setup (evalmachine.<anonymous>:18:15)
    at p5.<anonymous> (/home/tony/projects/processing.notebook/node_modules/p5/lib/p5.js:11613:15)
    at p5.<anonymous> (/home/tony/projects/processing.notebook/node_modules/p5/lib/p5.js:11564:12)
    at new p5 (/home/tony/projects/processing.notebook/node_modules/p5/lib/p5.js:11816:12)
    at evalmachine.<anonymous>:15:16
    at ContextifyScript.Script.runInThisContext (vm.js:50:33)

This may be related to reusing the same jsdom context, but I'm not sure. Is there a good way to debug libraries within the notebook execution (e.g. those in node_modules/)?

n-riesco commented 6 years ago

What versions of jupyter are you using? Last night I found behaviour changed with the frontend version. Could you post the output of jupyter notebook --version and ipython --version?

There is no support for debugging implemented in IJavascript. What I usually do is:

In your case, I'd try to add a console.log(dom.window.document.getElementById("main").outerHTML) just above p.createCanvas(400, 400, 'svg') to check whether the div#main still exists when you rerun the cell.

n-riesco commented 6 years ago

OK, I managed to fix the example, but you need to install the latest version of the jupyter notebook. Here's the code:

// In[1]:
// Setup
var JSDOM = require("jsdom").JSDOM;
require("usertiming");

// setup DOM
var dom = new JSDOM();
var window = dom.window;
var document = dom.window.document;
var screen = dom.window.screen;
var navigator = dom.window.navigator;

// load p5 libraries
var p5 = require("p5");
require("p5.js-svg")(p5);

// In[2]:
// setup div#main
document.body.innerHTML = "<div id='main'></div>";

// setup main display
var main = $$.display('main');
main.html("Display hasn't been updated yet");

// start p5 job
$$.async();

{
    let console = global.console;
    let $$ = global.$$;

    var myp5 = new p5(function(p) {
        let count = 0;
        let rounded = 10;

        p.setup = function() {
            //console.log("\n before setup", dom.window.document.body.innerHTML);
            p.createCanvas(400, 400, 'svg');
            p.background(255);
            p.fill(150);
            //console.log("\n after setup", dom.window.document.body.innerHTML);
        };

        p.draw = function() {
            if (count++ > 50) {
                //console.log("\nbefore remove", dom.window.document.body.innerHTML);
                myp5.remove();
                //console.log("\nafter remove", dom.window.document.body.innerHTML);

                main.text("!!!!!!!!");
                $$.done();

                return;
            }

            let red = Math.round(Math.random() * 255);
            let grn = Math.round(Math.random() * 255);
            let blu = Math.round(Math.random() * 255);
            p.background(red, grn, blu);
            p.rect(30, 20, 55, 55, rounded);
            main.html(dom.window.document.getElementById("main").innerHTML);
        };
    }, 'main');
}

Rerunning the cell was failing because p.createCanvas(400, 400, 'svg'); expected new p5(...); to insert a <canvas> tag (but running new p5(...); after myp5.remove(); doesn't insert a new canvas). the issue can be workaround by setting document.body.innerHTML = "<div id='main'></div>"; before calling new p5(...);.


Keep in mind that this example works, because the code that creates and updates the cell is inside the same cell, I still need to fix #132.

n-riesco commented 6 years ago

I've just released NEL@0.5.8 that fixes the issue with updating displays from different cells.

Here's the updated example:

// In[1]:
// Setup
var JSDOM = require("jsdom").JSDOM;
require("usertiming");

// setup DOM
var dom = new JSDOM();
var window = dom.window;
var document = dom.window.document;
var screen = dom.window.screen;
var navigator = dom.window.navigator;

// load p5 libraries
var p5 = require("p5");
require("p5.js-svg")(p5);

// In[2]:
// setup div#main
document.body.innerHTML = "<div id='main'></div>";

// setup main display
var main = $$.display('main');
main.html("Display hasn't been updated yet");

// In[3]:
// start p5 job
{
    let console = global.console;
    let $$ = global.$$;

    var myp5 = new p5(function(p) {
        let count = 0;
        let rounded = 10;

        p.setup = function() {
            p.createCanvas(400, 400, 'svg');
            p.background(255);
            p.fill(150);
        };

        p.draw = function() {
            if (count++ > 50) {
                myp5.remove();
                main.text("The End!");
                return;
            }

            let red = Math.round(Math.random() * 255);
            let grn = Math.round(Math.random() * 255);
            let blu = Math.round(Math.random() * 255);
            p.background(red, grn, blu);
            p.rect(30, 20, 55, 55, rounded);
            main.html(dom.window.document.getElementById("main").innerHTML);
        };
    }, 'main');
}

peek 2018-01-25 23-20

westurner commented 6 years ago

When I:

$ jupyter-notebook --version
4.4.1  # doesn't work
$ conda update -y notebook
$ jupyter-notebook --version
5.4.1  # it works!
  "dependencies": {
    "canvas": "^1.6.10",
    "ijavascript": "^5.0.20",
    "jsdom": "^11.8.0",
    "mathjs": "^4.1.2",
    "nel": "^1.0.0",
    "p5": "^0.6.0",
    "p5.js-svg": "^0.5.2",
    "usertiming": "^0.1.8"
  }

Thanks!

n-riesco commented 6 years ago

@westurner Thank you for investigating this issue.

As a workaround, one could also run delete window.performance before requesting p5:

// In[1]:
// Setup
var JSDOM = require("jsdom").JSDOM;
require("usertiming");

// setup DOM
var dom = new JSDOM();
var window = dom.window;
var document = dom.window.document;
var screen = dom.window.screen;
var navigator = dom.window.navigator;

// workaround for https://github.com/processing/p5.js/issues/2797
delete window.performance;

// load p5 libraries
var p5 = require("p5");
require("p5.js-svg")(p5);

// In[2]:
// setup div#main
document.body.innerHTML = "<div id='main'></div>";

// setup main display
var main = $$.display('main');
main.html("Display hasn't been updated yet");

// In[3]:
// start p5 job
{
    let console = global.console;
    let $$ = global.$$;

    var myp5 = new p5(function(p) {
        let count = 0;
        let rounded = 10;

        p.setup = function() {
            p.createCanvas(400, 400, 'svg');
            p.background(255);
            p.fill(150);
        };

        p.draw = function() {
            if (count++ > 50) {
                myp5.remove();
                main.text("The End!");
                return;
            }

            let red = Math.round(Math.random() * 255);
            let grn = Math.round(Math.random() * 255);
            let blu = Math.round(Math.random() * 255);
            p.background(red, grn, blu);
            p.rect(30, 20, 55, 55, rounded);
            main.html(dom.window.document.getElementById("main").innerHTML);
        };
    }, 'main');
}