worker.js: onmessage() captured an uncaught exception: ReferenceError: __register_pthread_ptr is not defined

[Zzzen]

I was trying to build a multithreaded ffmpeg. But the generated worker.js doesn't work. Apparently, __register_pthread_ptr is not defined in worker.js.

you can download https://github.com/Zzzen/videoconverter.js/tree/test and try

docker build -f "Dockerfile" -t build-img --build-arg EMCC_OPTIONS="-s ERROR_ON_UNDEFINED_SYMBOLS=0 -s USE_PTHREADS=1 -s PTHREAD_POOL_SIZE=4 -s TOTAL_MEMORY=256MB -o ffmpeg-all-codecs-wasm.js" --build-arg FFMPEG_OPTIONS='--enable-libvpx --enable-libx264 --enable-gpl --enable-nonfree --extra-cflags="-I../libvpx -I../x264 -I../zlib" --extra-libs="../libvpx/libvpx.a ../x264/libx264.a ../zlib/libz.a"'  .
docker create --name build-content build-img
docker cp build-content:/app/build/ffmpeg-all-codecs-wasm.js dist/ffmpeg-all-codecs-wasm.js
docker cp build-content:/app/build/ffmpeg-all-codecs-wasm.wasm dist/ffmpeg-all-codecs-wasm.wasm
docker cp build-content:/app/build/ffmpeg-all-codecs-wasm.js.mem dist/ffmpeg-all-codecs-wasm.js.mem
docker cp build-content:/app/build/ffmpeg-all-codecs-wasm.worker.js dist/ffmpeg-all-codecs-wasm.worker.js

generated worker.js

// Copyright 2015 The Emscripten Authors.  All rights reserved.
// Emscripten is available under two separate licenses, the MIT license and the
// University of Illinois/NCSA Open Source License.  Both these licenses can be
// found in the LICENSE file.

// Pthread Web Worker startup routine:
// This is the entry point file that is loaded first by each Web Worker
// that executes pthreads on the Emscripten application.

// Thread-local:
var threadInfoStruct = 0; // Info area for this thread in Emscripten HEAP (shared). If zero, this worker is not currently hosting an executing pthread.
var selfThreadId = 0; // The ID of this thread. 0 if not hosting a pthread.
var parentThreadId = 0; // The ID of the parent pthread that launched this thread.
var tempDoublePtr = 0; // A temporary memory area for global float and double marshalling operations.

// Thread-local: Each thread has its own allocated stack space.
var STACK_BASE = 0;
var STACKTOP = 0;
var STACK_MAX = 0;

// These are system-wide memory area parameters that are set at main runtime startup in main thread, and stay constant throughout the application.
var buffer; // All pthreads share the same Emscripten HEAP as SharedArrayBuffer with the main execution thread.
var DYNAMICTOP_PTR = 0;
var DYNAMIC_BASE = 0;

var ENVIRONMENT_IS_PTHREAD = true;
var PthreadWorkerInit = {};

// performance.now() is specced to return a wallclock time in msecs since that Web Worker/main thread launched. However for pthreads this can cause
// subtle problems in emscripten_get_now() as this essentially would measure time from pthread_create(), meaning that the clocks between each threads
// would be wildly out of sync. Therefore sync all pthreads to the clock on the main browser thread, so that different threads see a somewhat
// coherent clock across each of them (+/- 0.1msecs in testing)
var __performance_now_clock_drift = 0;

// Cannot use console.log or console.error in a web worker, since that would risk a browser deadlock! https://bugzilla.mozilla.org/show_bug.cgi?id=1049091
// Therefore implement custom logging facility for threads running in a worker, which queue the messages to main thread to print.
var Module = {};

// When error objects propagate from Web Worker to main thread, they lose helpful call stack and thread ID information, so print out errors early here,
// before that happens.
this.addEventListener('error', function(e) {
  if (e.message.indexOf('SimulateInfiniteLoop') != -1) return e.preventDefault();

  var errorSource = ' in ' + e.filename + ':' + e.lineno + ':' + e.colno;
  console.error('Pthread ' + selfThreadId + ' uncaught exception' + (e.filename || e.lineno || e.colno ? errorSource : "") + ': ' + e.message + '. Error object:');
  console.error(e.error);
});

function threadPrint() {
  var text = Array.prototype.slice.call(arguments).join(' ');
  console.log(text);
}
function threadPrintErr() {
  var text = Array.prototype.slice.call(arguments).join(' ');
  console.error(text);
  console.error(new Error().stack);
}
function threadAlert() {
  var text = Array.prototype.slice.call(arguments).join(' ');
  postMessage({cmd: 'alert', text: text, threadId: selfThreadId});
}
out = threadPrint;
err = threadPrintErr;
this.alert = threadAlert;

Module['instantiateWasm'] = function(info, receiveInstance) {
  // Instantiate from the module posted from the main thread.
  // We can just use sync instantiation in the worker.
  instance = new WebAssembly.Instance(wasmModule, info);
  // We don't need the module anymore; new threads will be spawned from the main thread.
  wasmModule = null;
  receiveInstance(instance); // The second 'module' parameter is intentionally null here, we don't need to keep a ref to the Module object from here.
  return instance.exports;
}

var wasmModule;
var wasmMemory;

this.onmessage = function(e) {
  try {
    if (e.data.cmd === 'load') { // Preload command that is called once per worker to parse and load the Emscripten code.
      // Initialize the thread-local field(s):
      tempDoublePtr = e.data.tempDoublePtr;

      // Initialize the global "process"-wide fields:
      DYNAMIC_BASE = e.data.DYNAMIC_BASE;
      DYNAMICTOP_PTR = e.data.DYNAMICTOP_PTR;

      // The Wasm module will have import fields for STACKTOP and STACK_MAX. At 'load' stage of Worker startup, we are just
      // spawning this Web Worker to act as a host for future created pthreads, i.e. we do not have a pthread to start up here yet.
      // (A single Worker can also host multiple pthreads throughout its lifetime, shutting down a pthread will not shut down its hosting Worker,
      // but the Worker is reused for later spawned pthreads). The 'run' stage below will actually start running a pthread.
      // The stack space for a pthread is allocated and deallocated when a pthread is actually run, not yet at Worker 'load' stage.
      // However, the WebAssembly module we are loading up here has import fields for STACKTOP and STACK_MAX, which it needs to get filled in
      // immediately at Wasm Module instantiation time. The values of these will not get used until pthread is actually running some code, so
      // we'll proceed to set up temporary invalid values for these fields for import purposes. Then whenever a pthread is launched at 'run' stage
      // below, these values are rewritten to establish proper stack area for the particular pthread.
      Module['STACK_MAX'] = Module['STACKTOP']  = 0x7FFFFFFF;

      // Module and memory were sent from main thread
      wasmModule = e.data.wasmModule;
      wasmMemory = e.data.wasmMemory;
      buffer = wasmMemory.buffer;

      PthreadWorkerInit = e.data.PthreadWorkerInit;

      if (typeof e.data.urlOrBlob === 'string') {
        importScripts(e.data.urlOrBlob);
      } else {
        var objectUrl = URL.createObjectURL(e.data.urlOrBlob);
        importScripts(objectUrl);
        URL.revokeObjectURL(objectUrl);
      }

      if (typeof FS !== 'undefined' && typeof FS.createStandardStreams === 'function') FS.createStandardStreams();
      postMessage({ cmd: 'loaded' });
    } else if (e.data.cmd === 'objectTransfer') {
      PThread.receiveObjectTransfer(e.data);
    } else if (e.data.cmd === 'run') { // This worker was idle, and now should start executing its pthread entry point.
      __performance_now_clock_drift = performance.now() - e.data.time; // Sync up to the clock of the main thread.
      threadInfoStruct = e.data.threadInfoStruct;
      __register_pthread_ptr(threadInfoStruct, /*isMainBrowserThread=*/0, /*isMainRuntimeThread=*/0); // Pass the thread address inside the asm.js scope to store it for fast access that avoids the need for a FFI out.
      selfThreadId = e.data.selfThreadId;
      parentThreadId = e.data.parentThreadId;
      // Establish the stack frame for this thread in global scope
      STACK_BASE = STACKTOP = e.data.stackBase;
      STACK_MAX = STACK_BASE + e.data.stackSize;
      // Call inside asm.js/wasm module to set up the stack frame for this pthread in asm.js/wasm module scope
      Module['establishStackSpace'](e.data.stackBase, e.data.stackBase + e.data.stackSize);

      PThread.receiveObjectTransfer(e.data);
      PThread.setThreadStatus(_pthread_self(), 1/*EM_THREAD_STATUS_RUNNING*/);

      try {
        // pthread entry points are always of signature 'void *ThreadMain(void *arg)'
        // Native codebases sometimes spawn threads with other thread entry point signatures,
        // such as void ThreadMain(void *arg), void *ThreadMain(), or void ThreadMain().
        // That is not acceptable per C/C++ specification, but x86 compiler ABI extensions
        // enable that to work. If you find the following line to crash, either change the signature
        // to "proper" void *ThreadMain(void *arg) form, or try linking with the Emscripten linker
        // flag -s EMULATE_FUNCTION_POINTER_CASTS=1 to add in emulation for this x86 ABI extension.
        var result = Module['dynCall_ii'](e.data.start_routine, e.data.arg);

      } catch(e) {
        if (e === 'Canceled!') {
          PThread.threadCancel();
          return;
        } else if (e === 'SimulateInfiniteLoop' || e === 'pthread_exit') {
          return;
        } else {
          Atomics.store(HEAPU32, (threadInfoStruct + 4 /*C_STRUCTS.pthread.threadExitCode*/ ) >> 2, (e instanceof ExitStatus) ? e.status : -2 /*A custom entry specific to Emscripten denoting that the thread crashed.*/);
          Atomics.store(HEAPU32, (threadInfoStruct + 0 /*C_STRUCTS.pthread.threadStatus*/ ) >> 2, 1); // Mark the thread as no longer running.
          _emscripten_futex_wake(threadInfoStruct + 0 /*C_STRUCTS.pthread.threadStatus*/, 0x7FFFFFFF/*INT_MAX*/); // Wake all threads waiting on this thread to finish.
          if (!(e instanceof ExitStatus)) throw e;
        }
      }
      // The thread might have finished without calling pthread_exit(). If so, then perform the exit operation ourselves.
      // (This is a no-op if explicit pthread_exit() had been called prior.)
      if (!Module['noExitRuntime']) PThread.threadExit(result);
    } else if (e.data.cmd === 'cancel') { // Main thread is asking for a pthread_cancel() on this thread.
      if (threadInfoStruct && PThread.thisThreadCancelState == 0/*PTHREAD_CANCEL_ENABLE*/) {
        PThread.threadCancel();
      }
    } else if (e.data.target === 'setimmediate') {
      // no-op
    } else if (e.data.cmd === 'processThreadQueue') {
      if (threadInfoStruct) { // If this thread is actually running?
        _emscripten_current_thread_process_queued_calls();
      }
    } else {
      err('worker.js received unknown command ' + e.data.cmd);
      console.error(e.data);
    }
  } catch(e) {
    console.error('worker.js onmessage() captured an uncaught exception: ' + e);
    console.error(e.stack);
    throw e;
  }
}

[kripken]

That function should arrive from loading the main JS code, by that worker.js file. Not sure offhand what could break that, but I'd start by checking if there are warnings in the devtools (did a file fail to load, etc.). If that's not it, building a hello world that creates a pthread can give you a working testcase (can look in the test suite for test_pthread* for examples, and compare that to what you have to look for differences.

[zhangxin92]

__register_pthread_ptr is not defined in worker.js

Hi @Zzzen, have you resolved the problem about '__register_pthread_ptr is not defined in worker.js'?

[kleisauke]

I get the same exception when using -s MODULARIZE_INSTANCE=1 in combination with -s EXPORT_NAME=MyModule. Here is a test case that reproduces this problem:

diff --git a/tests/test_browser.py b/tests/test_browser.py
index 1111111..2222222 100644
--- a/tests/test_browser.py
+++ b/tests/test_browser.py
@@ -3701,6 +3701,7 @@ window.close = function() {
     test([])
     test(['-O3'])
     test(['-s', 'MODULARIZE_INSTANCE=1'])
+    test(['-s', 'MODULARIZE_INSTANCE=1', '-s', 'EXPORT_NAME=MyModule', '--shell-file', path_from_root('tests', 'shell_that_launches_modularize.html')])

   # Test that preallocating worker threads work.
   @requires_threads

https://github.com/emscripten-core/emscripten/blob/479300efbee029835f5fb60a0b6a9a8968461bd7/tests/test_browser.py#L3703

[kripken]

@kleisauke

I think that shell file is only compatible with MODULARIZE - it creates an instance - and not MODULARIZE_INSTANCE. It works with the _INSTANCE removed.

(Or have I missed the point here? Sorry if so.)

[kleisauke]

I used that shell file to hide this error:

shared:ERROR: Customizing EXPORT_NAME requires that the HTML be customized to use that name

(I've added -s EXPORT_NAME=MyModule for reproducing this error, without that flag it runs correctly)

Here is a slightly improved test case (with a workaround):

  # Test that basic thread creation works in combination with MODULARIZE_INSTANCE=1 and EXPORT_NAME=MyModule
  @requires_threads
  def test_pthread_modularize_export_name(self):
    def test(args, workaround=None):
      print(args)
      print(('without' if workaround is None else 'with') + ' workaround')
      self.btest(path_from_root('tests', 'pthread', 'test_pthread_create.cpp'), expected='0', args=['-s', 'TOTAL_MEMORY=64MB', '-s', 'USE_PTHREADS=1', '-s', 'PTHREAD_POOL_SIZE=8'] + args, post_build=workaround)

    # we need to replace all Module instances with EXPORT_NAME as workaround
    def workaround(*args):
      worker = open('test.worker.js').read()
      worker = worker.replace('Module = {};', 'MyModule = {}')
      worker = worker.replace('Module[\'', 'MyModule[\'')
      worker = worker.replace('global.Module = Module', 'global.MyModule = MyModule')
      worker = worker.replace('out(', 'console.log(')  # out is not defined for some reason? (this also fixes the warnings within test_pthread_create)
      create_test_file('test.worker.js', worker)

    create_test_file('shell.html', '''
        <body>
          {{{ SCRIPT }}}
        </body>
      ''')
    test(['-s', 'MODULARIZE_INSTANCE=1', '-s', 'EXPORT_NAME=MyModule', '--shell-file', 'shell.html'], workaround)
    test(['-s', 'MODULARIZE_INSTANCE=1', '-s', 'EXPORT_NAME=MyModule', '--shell-file', 'shell.html'], None)

[kripken]

Thanks @kleisauke ! Great testcase, fix in #10449.