Getting 'fs is not defined' when trying the latest "background removal" functionality in the browser?

[lancejpollard]

Question

I copied the code from https://github.com/xenova/transformers.js/blob/main/examples/remove-background-client/main.js to here, but I'm getting this error with v2.15.0 of @xenova/transformers.js:

Uncaught ReferenceError: fs is not defined
    at env.js:36:31
    at [project]/node_modules/.pnpm/@xenova+transformers@2.15.0/node_modules/@xenova/transformers/src/env.js [app-client] (ecmascript) (http://localhost:3001/_next/static/chunks/8484b_%40xenova_transformers_src_5fe153._.js:258:3)
    at runtime-base.ts:322:21
    at runModuleExecutionHooks (runtime-base.ts:376:5)
    at instantiateModule (runtime-base.ts:321:5)
    at getOrInstantiateModuleFromParent (runtime-base.ts:424:10)
    at esmImport (runtime-utils.ts:205:18)
    at hub.js:6:2
    at [project]/node_modules/.pnpm/@xenova+transformers@2.15.0/node_modules/@xenova/transformers/src/utils/hub.js [app-client] (ecmascript) (http://localhost:3001/_next/static/chunks/8484b_%40xenova_transformers_src_5fe153._.js:783:3)
    at runtime-base.ts:322:21
    at runModuleExecutionHooks (runtime-base.ts:376:5)
    at instantiateModule (runtime-base.ts:321:5)
    at getOrInstantiateModuleFromParent (runtime-base.ts:424:10)
    at esmImport (runtime-utils.ts:205:18)
    at tokenizers.js:21:2
    at [project]/node_modules/.pnpm/@xenova+transformers@2.15.0/node_modules/@xenova/transformers/src/tokenizers.js [app-client] (ecmascript) (http://localhost:3001/_next/static/chunks/8484b_%40xenova_transformers_src_5fe153._.js:6729:3)
    at runtime-base.ts:322:21
    at runModuleExecutionHooks (runtime-base.ts:376:5)
    at instantiateModule (runtime-base.ts:321:5)
    at getOrInstantiateModuleFromParent (runtime-base.ts:424:10)
    at esmImport (runtime-utils.ts:205:18)
    at pipelines.js:14:2
    at [project]/node_modules/.pnpm/@xenova+transformers@2.15.0/node_modules/@xenova/transformers/src/pipelines.js [app-client] (ecmascript) (http://localhost:3001/_next/static/chunks/8484b_%40xenova_transformers_src_5fe153._.js:17183:3)
    at runtime-base.ts:322:21
    at runModuleExecutionHooks (runtime-base.ts:376:5)
    at instantiateModule (runtime-base.ts:321:5)
    at getOrInstantiateModuleFromParent (runtime-base.ts:424:10)
    at esmImport (runtime-utils.ts:205:18)
    at 8484b_@xenova_transformers_src_5fe153._.js:17215:237
    at [project]/node_modules/.pnpm/@xenova+transformers@2.15.0/node_modules/@xenova/transformers/src/transformers.js [app-client] (ecmascript) {module evaluation} (http://localhost:3001/_next/static/chunks/8484b_%40xenova_transformers_src_5fe153._.js:17228:3)
    at runtime-base.ts:322:21
    at runModuleExecutionHooks (runtime-base.ts:376:5)
    at instantiateModule (runtime-base.ts:321:5)
    at getOrInstantiateModuleFromParent (runtime-base.ts:424:10)
    at esmImport (runtime-utils.ts:205:18)
    at _b29e97._.js:19146:268
    at [project]/app/remove/background/page.tsx [app-client] (ecmascript) (http://localhost:3001/_next/static/chunks/_b29e97._.js:19389:3)
    at runtime-base.ts:322:21
    at runModuleExecutionHooks (runtime-base.ts:376:5)
    at instantiateModule (runtime-base.ts:321:5)
    at getOrInstantiateModuleFromParent (runtime-base.ts:424:10)
    at commonJsRequire (runtime-utils.ts:230:18)
    at requireModule (react-server-dom-turbopack-client.browser.development.js:154:23)
    at initializeModuleChunk (react-server-dom-turbopack-client.browser.development.js:1336:17)
    at readChunk (react-server-dom-turbopack-client.browser.development.js:1146:7)
    at mountLazyComponent (react-dom.development.js:16652:19)
    at beginWork$1 (react-dom.development.js:18388:16)
    at beginWork (react-dom.development.js:26791:14)
    at performUnitOfWork (react-dom.development.js:25637:12)
    at workLoopSync (react-dom.development.js:25353:5)

Any idea what is wrong and how to fix it? Here is my code, which basically a direct React.js port of the background removal example you all shared:

'use client'

import {
  AutoModel,
  AutoProcessor,
  env,
  PreTrainedModel,
  Processor,
  RawImage,
} from '@xenova/transformers'
import React, {
  MouseEvent,
  useCallback,
  useEffect,
  useRef,
  useState,
} from 'react'
import _ from 'lodash'
import FileDropzone from '~/components/FileDropzone'

// Since we will download the model from the Hugging Face Hub, we can skip the local model check
env.allowLocalModels = false

// Proxy the WASM backend to prevent the UI from freezing
env.backends.onnx.wasm.proxy = true

function useModel(): {
  model?: PreTrainedModel
  processor?: Processor
} {
  const [model, setModel] = useState<PreTrainedModel>()
  const [processor, setProcessor] = useState<Processor>()

  useEffect(() => {
    AutoModel.from_pretrained('briaai/RMBG-1.4', {
      config: { model_type: 'custom' },
    }).then(m => {
      setModel(m)
    })

    AutoProcessor.from_pretrained('briaai/RMBG-1.4', {
      config: {
        do_normalize: true,
        do_pad: false,
        do_rescale: true,
        do_resize: true,
        image_mean: [0.5, 0.5, 0.5],
        feature_extractor_type: 'ImageFeatureExtractor',
        image_std: [1, 1, 1],
        resample: 2,
        rescale_factor: 0.00392156862745098,
        size: { width: 1024, height: 1024 },
      },
    }).then(p => {
      setProcessor(p)
    })
  }, [])

  if (!model || !processor) {
    return {}
  }

  return { model, processor }
}

function Body() {
  const { model, processor } = useModel()
  const [status, setStatus] = useState('Ready')
  const isLoading = !model || !processor
  const canvasRef = useRef<HTMLCanvasElement>(null)

  // Predict foreground of the given image
  async function predict(file: File) {
    if (isLoading) {
      return
    }

    const canvas = canvasRef.current

    if (!canvas) {
      return
    }

    const ctx = canvas.getContext('2d')

    if (!ctx) {
      return
    }

    setStatus('Processing...')

    // Read image
    const image = await RawImage.fromBlob(file)

    // Preprocess image
    const { pixel_values } = await processor(image)

    // Predict alpha matte
    const { output } = await model({ input: pixel_values })

    // Resize mask back to original size
    const mask = await RawImage.fromTensor(
      // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
      output[0].mul(255).to('uint8'),
    ).resize(image.width, image.height)

    // Create new canvas
    canvas.width = image.width
    canvas.height = image.height

    // Draw original image output to canvas
    ctx.drawImage(image.toCanvas() as CanvasImageSource, 0, 0)

    // Update alpha channel
    const pixelData = ctx.getImageData(0, 0, image.width, image.height)
    for (let i = 0; i < mask.data.length; ++i) {
      pixelData.data[4 * i + 3] = mask.data[i]
    }
    ctx.putImageData(pixelData, 0, 0)

    setStatus('Done!')
  }

  return (
    <>
      <div
        id="tool"
        className="relative p-16 flex flex-col gap-16"
      >
        <FileDropzone
          topped
          className="1-3-screen-minus-nav flex items-center bg-blue-50 hover:bg-blue-100"
          borderClassName="border-blue-300"
          onDrop={files => predict(files[0])}
          // bottomed={Boolean(inputError)}
        >
          <P className="text-center">Drop file here</P>
        </FileDropzone>
        <div className="text-center">{status}</div>
        <canvas ref={canvasRef} />
      </div>
    </>
  )
}

[lancejpollard]

Also, when trying to build with Next.js pnpm build, I am getting another error, if you have any ideas here, will post to the Next.js group:

Error: Expected to use Webpack bindings (react-server-dom-webpack/server.edge) for React but the current process is referencing 'createClientModuleProxy' from the Turbopack bindings (react-server-dom-turbopack/server.edge). This is likely a bug in our integration of the Next.js server runtime.
    at Object.get (/app/node_modules/.pnpm/next@14.1.1-canary.27_@babel+core@7.22.10_react-dom@18.2.0_react@18.2.0/node_modules/next/dist/compiled/next-server/app-page.runtime.dev.js:35:366642)
    at Object.<anonymous> (/app/.next/server/chunks/568bf_next_dist_78c2c7._.js:71:9)
    at [project]/node_modules/.pnpm/next@14.1.1-canary.27_@babel+core@7.22.10_react-dom@18.2.0_react@18.2.0/node_modules/next/dist/client/components/error-boundary.js (client proxy) (/app/.next/server/chunks/568bf_next_dist_78c2c7._.js:74:3)
    at instantiateModule (/app/.next/server/chunks/[turbopack]_runtime.js:488:23)
    at getOrInstantiateModuleFromParent (/app/.next/server/chunks/[turbopack]_runtime.js:539:12)
    at esmImport (/app/.next/server/chunks/[turbopack]_runtime.js:113:20)
    at /app/.next/server/chunks/568bf_next_dist_78c2c7._.js:78:312
    at [project]/node_modules/.pnpm/next@14.1.1-canary.27_@babel+core@7.22.10_react-dom@18.2.0_react@18.2.0/node_modules/next/dist/client/components/error-boundary.js [app-rsc] (ecmascript) (/app/.next/server/chunks/568bf_next_dist_78c2c7._.js:84:3)
    at instantiateModule (/app/.next/server/chunks/[turbopack]_runtime.js:488:23)
    at getOrInstantiateModuleFromParent (/app/.next/server/chunks/[turbopack]_runtime.js:539:12)

> Build error occurred
Error: Failed to collect page data for /
    at /app/node_modules/.pnpm/next@14.1.1-canary.27_@babel+core@7.22.10_react-dom@18.2.0_react@18.2.0/node_modules/next/dist/build/utils.js:1268:15
    at process.processTicksAndRejections (node:internal/process/task_queues:95:5) {
  type: 'Error'
}

[xenova]

Could you provide your webpack config? Also, seeing that you are using next.js, you might find our next.js demo app/tutorial helpful.

[lancejpollard]

My next.config.js which contains my webpack config is like this:

/** @type {import('next').NextConfig} */
import nextTranspileModules from 'next-transpile-modules'
import createMDXPlugin from '@next/mdx'
import withYaml from 'next-plugin-yaml'
import path from 'path'
import { fileURLToPath } from 'url'
import remarkGfm from 'remark-gfm'
import remarkPrism from 'remark-prism'
import rehypeSlug from 'rehype-slug'
import remarkMath from 'remark-math'
import rehypeKatex from 'rehype-katex'
import autoprefixer from 'autoprefixer'

const __dirname = path.dirname(fileURLToPath(import.meta.url))

if (
  process.env.LD_LIBRARY_PATH == null ||
  !process.env.LD_LIBRARY_PATH.includes(
    `${process.env.PWD}/node_modules/canvas/build/Release:`,
  )
) {
  process.env.LD_LIBRARY_PATH = `${
    process.env.PWD
  }/node_modules/canvas/build/Release:${
    process.env.LD_LIBRARY_PATH || ''
  }`
}

const mdxConfig = {
  extension: /\.mdx?$/,
  options: {
    providerImportSource: '@mdx-js/react',
    rehypePlugins: [[rehypeSlug], [rehypeKatex]],
    remarkPlugins: [[remarkPrism], [remarkMath], [remarkGfm]],
  },
}

const withMDX = createMDXPlugin(mdxConfig)

const nextConfig = withYaml(
  withMDX({
    swcMinify: true,
    optimizeFonts: false,
    transpilePackages: [
      'react-syntax-highlighter',
      'pdfjs-dist',
    ],
    pageExtensions: ['mdx', 'md', 'tsx', 'ts'],
    poweredByHeader: false,
    trailingSlash: false,
    webpack: (config, options) => {
      config.experiments = {
        asyncWebAssembly: true,
        syncWebAssembly: true,
        layers: true,
        topLevelAwait: true,
      }
      config.resolve.fallback = {
        fs: false,
        path: false,
        dns: false,
        net: false,
        tls: false,
      }

      config.resolve = {
        ...config.resolve,
        extensions: ['.ts', '.tsx', '.js', '.json', '.mjs'],
      }

      config.plugins ??= []

      config.module.rules.push({
        test: /\.mdx?$/,
        use: [
          // Note that Webpack runs right-to-left: `@mdx-js/loader` is used first, then
          // `babel-loader`.
          { loader: 'babel-loader', options: {} },
          {
            loader: '@mdx-js/loader',
            /** @type {import('@mdx-js/loader').Options} */
            options: {},
          },
        ],
      })

      config.resolve.alias.canvas = false

      config.module.rules.push({
        test: /\.(glsl|vs|fs|vert|frag)$/,
        type: 'asset/source',
      })

      config.module.rules.push({
        test: /\.(txt|node)$/,
        loader: 'raw-loader',
      })

      return config
    },
  }),
)

export default nextConfig

@xenova the only thing I see in there might be:

      config.resolve.fallback = {
        fs: false,
        path: false,
        dns: false,
        net: false,
        tls: false,
      }

[lancejpollard]

I tried adding this, but still same error:

      config.resolve = {
        ...config.resolve,
        sharp$: false,
        'onnxruntime-node$': false,
        extensions: ['.ts', '.tsx', '.js', '.json', '.mjs'],
      }