Cannot run selfie segmentation in web worker - React.js

[DebasisDa]

Have I written custom code (as opposed to using a stock example script provided in MediaPipe)

None

OS Platform and Distribution

Windows 11 Enterprise

Mobile device if the issue happens on mobile device

No response

Browser and version if the issue happens on browser

Google chrome - Version 125.0.6422.78 (Official Build) (32-bit)

Programming Language and version

Javascript ( React.js )

MediaPipe version

@mediapipe/selfie_segmentation: "0.1.1632777926"

Bazel version

No response

Solution

Selfie Segmentation

Android Studio, NDK, SDK versions (if issue is related to building in Android environment)

No response

Xcode & Tulsi version (if issue is related to building for iOS)

No response

Describe the actual behavior

mediapipe/selfie_segmentation should works the same way as in main thread

Describe the expected behaviour

Brower Error : (anonymous) @ selfie_segmentation.js:51 ta @ selfie_segmentation.js:14 next @ selfie_segmentation.js:15 h @ selfie_segmentation.js:40 Promise.then (async) g @ selfie_segmentation.js:40 (anonymous) @ selfie_segmentation.js:40 Y @ selfie_segmentation.js:40 Pb @ selfie_segmentation.js:49 (anonymous) @ selfie_segmentation.js:58 ta @ selfie_segmentation.js:14 next @ selfie_segmentation.js:15 (anonymous) @ selfie_segmentation.js:40 Y @ selfie_segmentation.js:40 push../node_modules/@mediapipe/selfie_segmentation/selfie_segmentation.js.D.initialize @ selfie_segmentation.js:58 (anonymous) @ selfie_segmentation.js:59 ta @ selfie_segmentation.js:14 next @ selfie_segmentation.js:15 h @ selfie_segmentation.js:40 Promise.then (async) g @ selfie_segmentation.js:40 (anonymous) @ selfie_segmentation.js:40 Y @ selfie_segmentation.js:40 push../node_modules/@mediapipe/selfie_segmentation/selfie_segmentation.js.D.send @ selfie_segmentation.js:59 (anonymous) @ selfie_segmentation.js:75 ta @ selfie_segmentation.js:14 next @ selfie_segmentation.js:15 (anonymous) @ selfie_segmentation.js:40 Y @ selfie_segmentation.js:40 push../node_modules/@mediapipe/selfie_segmentation/selfie_segmentation.js.D.send @ selfie_segmentation.js:75 (anonymous) @ worker.js:27 selfie_segmentation.js:51 Refused to execute script from 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1.1675465747/selfie_segmentation_landscape.tflite' because its MIME type ('application/octet-stream') is not executable, and strict MIME type checking is enabled.

Standalone code/steps you may have used to try to get what you need

//Worker.js
/* eslint-disable no-restricted-globals */

import { SelfieSegmentation } from "@mediapipe/selfie_segmentation";
let selfieSegmentation;
self.addEventListener("message", async (event) => {  //eslint-enable no-restricted-globals 
  const { type, data } = event.data;
  switch (type) {
    case "initialize":
      console.log("initialize");
      selfieSegmentation = new SelfieSegmentation({
        locateFile: (file) =>
          `https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1.1675465747/${file}`,
      });
      selfieSegmentation.setOptions({
        modelSelection: 1,
        selfieMode: false,
      });
      break;
    case "processFrame":
      if (selfieSegmentation) {
        await selfieSegmentation.send({ image: data });
        const results = selfieSegmentation.getResults();
        self.postMessage({ type: "results", data: results });  //eslint-enable no-restricted-globals
      }
      break;
    default:
      break;
  }
});
/* eslint-disable no-restricted-globals */

//BlurBackgroundWebWorker.js
import React, { useEffect, useRef, useState } from "react";

function BlurBackgroundWebWorker() {
  const inputVideoRef = useRef();
  const canvasRef = useRef();
  const workerRef = useRef();

  const [workerr, setWorker] = useState(null);
  let offscreenCanvas = null;

  useEffect(() => {
    const constraints = {
      video: { width: { min: 1280 }, height: { min: 720 } },
    };

    // Get user media
    navigator.mediaDevices.getUserMedia(constraints).then((stream) => {
      inputVideoRef.current.srcObject = stream;
      initializeWorker();
    });
  }, []);

  useEffect(() => {
    if (workerr != null) {
      workerr.postMessage({ type: "initialize" });

      workerr.addEventListener("message", handleWorkerMessage);

      // Process each frame
      const processFrame = () => {
        if (inputVideoRef.current.readyState === inputVideoRef.current.HAVE_ENOUGH_DATA) {
          createImageBitmap(inputVideoRef.current).then((videoBitmap) => {
            workerr.postMessage({ type: "processFrame", videoBitmap }, [videoBitmap]);
          });
        }
        requestAnimationFrame(processFrame);
      };
      processFrame();

      return () => {
        if (workerr) {
          workerr.terminate();
        }
      };
    }
  }, [workerr]);

  const initializeWorker = () => {
    const worker = new Worker(new URL('./worker.js', import.meta.url), { type: 'module' });
    setWorker(worker);

    // Process each frame
    if (offscreenCanvas == null)
      offscreenCanvas = canvasRef.current.transferControlToOffscreen();

  };

  const handleWorkerMessage = (event) => {
    const { type, data } = event.data;
    if (type === "results") {
      renderResults(data);
    }
  };

  const renderResults = (results) => {
    const context = canvasRef.current.getContext("2d");
    context.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height);
    context.drawImage(results.segmentationMask, 0, 0);
    context.globalCompositeOperation = "source-in";
    context.drawImage(results.image, 0, 0);
    context.globalCompositeOperation = "destination-atop";
    context.filter = "blur(10px)";
    context.drawImage(results.image, 0, 0);
  };

  return (
    <div className="App">
      <video autoPlay ref={inputVideoRef} style={{ position: "absolute" }} width={800} height={500} />
      <canvas ref={canvasRef} width={800} height={500} style={{ position: "absolute" }} />
    </div>
  );
}

export { BlurBackgroundWebWorker };

Other info / Complete Logs

Brower Error : 

 (anonymous) @ selfie_segmentation.js:51 ta @ selfie_segmentation.js:14 next @ selfie_segmentation.js:15 h @ selfie_segmentation.js:40 Promise.then (async) g @ selfie_segmentation.js:40 (anonymous) @ selfie_segmentation.js:40 Y @ selfie_segmentation.js:40 Pb @ selfie_segmentation.js:49 (anonymous) @ selfie_segmentation.js:58 ta @ selfie_segmentation.js:14 next @ selfie_segmentation.js:15 (anonymous) @ selfie_segmentation.js:40 Y @ selfie_segmentation.js:40 push../node_modules/@mediapipe/selfie_segmentation/selfie_segmentation.js.D.initialize @ selfie_segmentation.js:58 (anonymous) @ selfie_segmentation.js:59 ta @ selfie_segmentation.js:14 next @ selfie_segmentation.js:15 h @ selfie_segmentation.js:40 Promise.then (async) g @ selfie_segmentation.js:40 (anonymous) @ selfie_segmentation.js:40 Y @ selfie_segmentation.js:40 push../node_modules/@mediapipe/selfie_segmentation/selfie_segmentation.js.D.send @ selfie_segmentation.js:59 (anonymous) @ selfie_segmentation.js:75 ta @ selfie_segmentation.js:14 next @ selfie_segmentation.js:15 (anonymous) @ selfie_segmentation.js:40 Y @ selfie_segmentation.js:40 push../node_modules/@mediapipe/selfie_segmentation/selfie_segmentation.js.D.send @ selfie_segmentation.js:75 (anonymous) @ worker.js:27 selfie_segmentation.js:51 Refused to execute script from 'https://cdn.jsdelivr.net/npm/@mediapipe/selfie_segmentation@0.1.1675465747/selfie_segmentation_landscape.tflite' because its MIME type ('application/octet-stream') is not executable, and strict MIME type checking is enabled.

[kuaashish]

Hi @DebasisDa,

You are currently using the legacy Selfie Segmentation solution, which has undergone a significant upgrade. The corresponding model and implementation have been modified, and it is now integrated into Image Segmentation Task API. Detailed information about this upgrade is available in the documentation, and the legacy solution is no longer supported from our end or maintained.

For the updated solution, please refer to the Image Segmentation overview page first. Follow the provided instructions for setting up the solution on the web. Additionally, you can find the Selfie Segmentation model on the same page and utilize it for segmenting selfie faces.

If you encounter any similar behavior in the new model, we kindly request you to report it. Your feedback is valuable as we continue to enhance and improve our solutions.

Thank you!!

[DebasisEasy]

ok. Let me check Image Segmentation. Will update here my update. Thanks @kuaashish

[kuaashish]

Hi @DebasisDa,

Could you please provide an update on the status of this? If so, sharing an example could assist other community members with our new Task API. If this issue is resolved on your end, we can proceed to close it and mark it as resolved internally.

Thank you!!