Add ImJoy RPC functionality to the UMD interface

[thewtex]

As suggested by @oeway, something like:

    <script>
      var container = document.querySelector('.content');             async function setupViewerForImJoy(container){
      var viewer = itkVtkViewer.processURLParameters(container, { fullscreen: true });              const imjoyRPC = await imjoyLoader.loadImJoyRPC({'api_version': '0.2.0'})
      if (!viewer) {                const api = await imjoyRPC.setupRPC()
        itkVtkViewer.createViewerFromLocalFiles(container);             api.export({
          setup(){ api.log("itk-vtk-viewer loaded successfully.")},
          async run(ctx) {
            if(ctx.data && ctx.data.image_array){
              this.imshow(ctx.data.image_array)
            }
            else{
              itkVtkViewer.createViewerFromLocalFiles(container)
            }
          },
          imshow(image_array){
            const vtkImage = convertVtkImage(image_array)
            const extent_3d = vtkImage.getExtent();
            const extent =  [extent_3d[0], extent_3d[2], extent_3d[1], extent_3d[3]];
            itkVtkViewer.createViewer(container, {
              image: vtkImage,
              pointSets: null,
              geometries: null,
              use2D: true,
              rotate: false
            })
          }
        })
        const numpy2vtkType = {
          'int8': {componentType: 'int8_t', arrayType: Int8Array },
          'uint8': {componentType: 'uint8_t', arrayType: Uint8Array },
          'int16': {componentType: 'int16_t', arrayType: Int16Array },
          'uint16': {componentType: 'uint16_t', arrayType: Uint16Array },
          'int32': {componentType: 'int32_t', arrayType: Int32Array },
          'uint32': {componentType: 'uint32_t', arrayType: Uint32Array },
          'float32': {componentType: 'float', arrayType: Float32Array },
          'float64': {componentType: 'double', arrayType: Float64Array }
        }

        function convertVtkImage(data){
          if(data.__jailed_type__ === 'ndarray'){
            if(data.__shape__.length === 2 || (data.__shape__.length==3 && data.__shape__[2]<=3)){
              const dtype = numpy2vtkType[data.__dtype__];
              const dshape = data.__shape__.length === 2? data.__shape__ : data.__shape__.slice(0, 2)
              const channels =  data.__shape__.length === 3 ?  data.__shape__[2] : 1;
              return itkVtkViewer.utils.vtkITKHelper.convertItkToVtkImage({
                imageType: { dimension: 2, pixelType: 1, componentType: dtype.componentType, components:channels},
                name: 'Image',
                origin: [0,0],
                spacing: [1,1],
                direction: {data: [1,0,0,1]},
                size: [dshape[1], dshape[0]],
                data: new dtype.arrayType(data.__value__.buffer)
              })
            }
            else if(data.__shape__.length === 3){
              return itkVtkViewer.utils.vtkITKHelper.convertItkToVtkImage({
                imageType: { dimension: 3, pixelType: 1, componentType: dtype.componentType, components: 1},
                name: 'image',
                origin: [0,0,0],
                spacing: [1,1,1],
                direction: {data: [1,0,0,0,1,0,0,0,1]},
                size: [dshape[2], dshape[1], dshape[0]],
                data: new dtype.arrayType(data.__value__.buffer)
              })
            }
            else{
              throw new Error(`Unsupported shape: ${arr.shape}`)
            }
          }
          return data
        }
      }
      const container = document.querySelector('.content');
      const urlParams = new URLSearchParams(window.location.search)
      // inside an iframe
      if(window.self !== window.top && urlParams.get('imjoy')){
        setupViewerForImJoy(container)
      }
      else{
        let viewer = itkVtkViewer.processURLParameters(container, { fullscreen: true });
        if (!viewer) {
          itkVtkViewer.createViewerFromLocalFiles(container)
        }

[thewtex]

See also:

• https://github.com/imjoy-team/imjoy-rpc
• https://github.com/imjoy-team/imjoy-core

[oeway]

Thanks @thewtex !

Yes, I think it would be cool, I will paste a gif here to see what can we do with this additional imjoy-rpc support.

In the example below, we start a jupyter notebook inside ImJoy (with RPC supported), then it can use the rpc to call other plugins, including the itk-vtk-viewer (my forked version with the above modifications).

As you can see, I can call viewer= await api.showDialog(src="https://oeway.github.io/itk-vtk-viewer/?imjoy=1") and viewer.imshow(imageio.imread(MY_IMAGE_FILE)) in Python, and I got a dialog with the numpy image loaded to the itk-vtk-viewer:

[oeway]

Update: we have been refining the internal encoding for the ImJoy RPC library, so the above code won't work with the latest imjoy app, a working version can be find here: https://github.com/oeway/itk-vtk-viewer/blob/master/dist/index.html