andywer / threads.js

🧵 Make web workers & worker threads as simple as a function call.
https://threads.js.org/
MIT License
3.04k stars 161 forks source link

Trying to use Transfer() to no avail... #472

Closed digable1 closed 1 year ago

digable1 commented 1 year ago

Hi,

This could be operator error - definitely a newbian writing (or it could be an actual issue?):

Firefox gives this: Uncaught (in promise) DOMException: The object could not be cloned..

Chrome gives this: Uncaught (in promise) DOMException: Failed to execute 'postMessage' on 'Worker': Value at index 0 does not have a transferable type.

This is my snippet that exhibits the problem (I couldn't get it to work in a ts-node command line argument, so I went full webpack and console.logs):

demo.ts

import { spawn, Transfer, TransferDescriptor } from "threads";
import { Testworker1 } from "./webworker1";

function demo(): void {
    console.log(`In demo....`);
    spawn<Testworker1>(new Worker(new URL("./webworker1.ts", import.meta.url))).then(
        async (webworker1) => {
            const request = 'Paging webworker1';

            const plainResponse = await webworker1.plainWebworker1(request);
            console.log(`demo - webworker1 plain: ${plainResponse}`);
            console.log();

            const encodedPing = encodeToWebworker(request);
            const encodedResponse = await webworker1.webworker1(encodedPing);
            console.log(`demo - webworker1 no transfer - ${decodeFromWebworker(encodedResponse)}`);
            console.log();

            const transferredPing = Transfer(encodedPing);
            console.log(`demo - webworker1 about to transfer - ${JSON.stringify(transferredPing, null, 4)}`);
            const transferEncodedResponse = await webworker1.transferWebworker(transferredPing);
            console.log(`demo webworker1 transferEncodedResponse: ${JSON.stringify(transferEncodedResponse, null, 4)}`);
            console.log(`demo - webworker1 transfer - ${decodeFromWebworker(transferEncodedResponse)}`);
            console.log();
        }
    )
}

let buffer = new Uint8Array(0);
export function encodeToWebworker(stringified: string): Uint8Array {
    buffer = new Uint8Array(stringified.length);
    new TextEncoder().encodeInto(stringified, buffer);
    return buffer;
}

export function decodeFromWebworker(buffer: Uint8Array): string {
    const clonedBuffer = new Uint8Array(buffer);
    const decodedBuffer = new TextDecoder().decode(clonedBuffer);
    return decodedBuffer;
}

demo();

webworker1.ts

import { TransferDescriptor } from "threads";
import { expose, Transfer } from "threads/worker"
// import { encodeToWebworker, decodeFromWebworker } from './demo';

const testworker1 = {
    webworker1(messageParm: Uint8Array): Uint8Array {
        console.log(`webworker1 - received message: ${decodeFromWebworker(messageParm)}`);
        const response = "This is webworker1";
        console.log(`webworker1 - sending response: ${response}`);
        const responseBuffer = encodeToWebworker(response);
        return responseBuffer;
    },
    plainWebworker1(message: string): string {
        console.log(`webworker1 - received plain message: ${message}`);
        const response = "This is plain webworker1";
        console.log(`webworker1 - sending plain response: ${response}`)
        return response;
    },
    transferWebworker(messageParm: TransferDescriptor<Uint8Array>): TransferDescriptor<Uint8Array> {
        console.log(`webworker1.transferWebworker - received transferred message: ${JSON.stringify(messageParm, null, 4)}`);
        const messageClone = new Uint8Array(messageParm.send);
        const transferredMessage = Transfer(messageParm.send) as TransferDescriptor<Uint8Array>;
        console.log(`webworker1.transferWebworker - transferred message: ${JSON.stringify(transferredMessage, null, 4)}`);
        const message = decodeFromWebworker(messageClone);
        console.log(`webworker1.transferWebworker - message: ${message}`);
        const response = "This is webworker1.transferWebworker response";
        console.log(`webworker1.transferWebworker - sending transferred response: ${response}`)
        const encodedResponse = encodeToWebworker(response);
        return Transfer(encodedResponse);
    }
}

let buffer = new Uint8Array(0);
export function encodeToWebworker(stringified: string): Uint8Array {
    buffer = new Uint8Array(stringified.length);
    new TextEncoder().encodeInto(stringified, buffer);
    return buffer;
}

export function decodeFromWebworker(buffer: Uint8Array): string {
    const clonedBuffer = new Uint8Array(buffer);
    return new TextDecoder().decode(clonedBuffer);
}
export type Testworker1 = typeof testworker1;
expose(testworker1);

tsconfig.json

{
  "files": ["demo.ts"],
  "compilerOptions": {
    "lib": ["webworker"],
    "module": "ESNext",
    "moduleResolution": "node"
  },
  "include": [
    "*.ts"
  ],
  "exclude": [
    "node_modules"
  ],
  "ts-node": {
    "esm": true
  }
}

package.json

{
    "name": "webworker_demo",
    "version": "0.0.1",
    "description": "A quick demo for multiple webworkers using threads.js",
    "main": "demo.js",
    "scripts": {
        "start": "webpack serve"
    },
    "dependencies": {
        "threads": "^1.7.0",
        "ts-node": "^10.9.1"
    },
    "devDependencies": {
        "@tsconfig/node-lts-strictest-esm": "^18.12.1",
        "clean-webpack-plugin": "^4.0.0",
        "eslint": "^8.42.0",
        "html-webpack-plugin": "^5.5.3",
        "typescript": "^5.1.3",
        "tsconfig-paths-webpack-plugin": "^4.0.1",
        "ts-loader": "^9.4.3",
        "webpack": "^5.86.0",
        "webpack-bundle-analyzer": "^4.9.0",
        "webpack-cli": "^5.1.4",
        "webpack-dev-server": "^4.15.1"
    }
}

webpack.config.js

const path = require('path');
const webpack = require('webpack');
// const ThreadsPlugin = require('threads-plugin');
const HtmlWebpackPlugin = require('html-webpack-plugin')
const CleanWebpackPlugin = require('clean-webpack-plugin');
const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;

module.exports = {
    stats: 'normal',
    entry: [
        './demo.ts'
    ],
    mode: 'production',
    devtool: 'source-map',
    devServer: {
        // contentBase: path.join(__dirname, '.'),
        static: path.join(__dirname, '.'),
        client: {
            overlay: {
                errors: true,
                warnings: false
            }
        },
        devMiddleware: {
            mimeTypes: { ts: 'text/javascript', js: 'text/javascript' }
        },
        compress: true,
        port: 8999,
        open: true
    },

    output: {
        library: 'demo',
        libraryTarget: 'umd',
        path: path.resolve(__dirname, "dist"),
    },

    resolve: {
        // Add `.ts` and `.tsx` as a resolvable extension.
        extensions: [".ts", ".tsx", ".js"]
    },
    module: {
        rules: [
            // all files with a `.ts` or `.tsx` extension will be handled by `ts-loader`
            {
                test: /\.tsx?$/,
                loader: "ts-loader",
                exclude: /node_modules/,
                options: {
                    compilerOptions: {
                        module: "esnext"
                    }
                }
            }
        ]
    },

    plugins: [
        // new ThreadsPlugin(),
        new webpack.ProgressPlugin({}),
        new CleanWebpackPlugin.CleanWebpackPlugin(),
        new webpack.HotModuleReplacementPlugin(),
        new HtmlWebpackPlugin({
            template: path.resolve(__dirname, "./index.html")
        }),
        new BundleAnalyzerPlugin({
            analyzerPort: 8887
        })
    ]
}

It blows up at my first use of Transfer() to get from the main thread to the worker thread 'const transferEncodedResponse = await webworker1.transferWebworker(transferredPing);'. I never get to the worker thread to see the first console.log().

The Chrome error looks more descriptive saying I don't have a transferable type. However, I'm using a Uint8Array, which is a view for ArrayBuffer - which should be a supported type.

I'm obviously clueless. Anybody have a clue they can spare?

digable1 commented 1 year ago

Update: Operator error! Turns out I was trying to transfer a Uint8Array object rather than uint8ArrayObject.buffer, the underlying ArrayBuffer. All better now!