[esm-integration] Add pthread support (#24555)

Under ESM integration all dependencies must be satisfied at import time,
but pthreads requires that we have supply the memory via postMessage, so
the memory is, by definition, not available at import time.

On order to work around this issue we create a smaller pthread
stub/loader file that delays the import of the main program until the
initial `postMessage` has been received. Once the memory is received we
load main program using a dynamic `import` statement.

Author: sbc100

eslint.config.mjs

@@ -35,6 +35,7 @@ export default [{
     'src/postamble*.js',
     'src/closure-externs/',
     'src/embind/',
+    'src/pthread_esm_startup.mjs',
     'src/emrun_postjs.js',
     'src/wasm_worker.js',
     'src/audio_worklet.js',

package.json

@@ -26,7 +26,7 @@
   },
   "scripts": {
     "lint": "eslint .",
-    "fmt": "prettier --write src/*.mjs tools/*.mjs",
-    "check": "prettier --check src/*.mjs tools/*.mjs"
+    "fmt": "prettier --write src/*.mjs tools/*.mjs --ignore-path src/pthread_esm_startup.mjs",
+    "check": "prettier --check src/*.mjs tools/*.mjs --ignore-path src/pthread_esm_startup.mjs"
   }
 }

site/source/docs/compiling/Modularized-Output.rst

@@ -163,7 +163,7 @@ This setting implicitly enables :ref:`export_es6` and sets :ref:`MODULARIZE` to
 
 Some additional limitations are:
 
-* ``-pthread`` / :ref:`wasm_workers` are not yet supported.
+* :ref:`wasm_workers` is not yet supported.
 
 * :ref:`abort_on_wasm_exceptions` is not supported (requires wrapping wasm
   exports).

src/lib/libpthread.js

@@ -29,6 +29,13 @@ const MAX_PTR = Number((2n ** 64n) - 1n);
 #else
 const MAX_PTR = (2 ** 32) - 1
 #endif
+
+#if WASM_ESM_INTEGRATION
+const pthreadWorkerScript = TARGET_BASENAME + '.pthread.mjs';
+#else
+const pthreadWorkerScript = TARGET_JS_NAME;
+#endif
+
 // Use a macro to avoid duplicating pthread worker options.
 // We cannot use a normal JS variable since the vite bundler requires that worker
 // options be inline.
@@ -295,7 +302,9 @@ var LibraryPThread = {
 
 #if ASSERTIONS
       assert(wasmMemory instanceof WebAssembly.Memory, 'WebAssembly memory should have been loaded by now!');
+#if !WASM_ESM_INTEGRATION
       assert(wasmModule instanceof WebAssembly.Module, 'WebAssembly Module should have been loaded by now!');
+#endif
 #endif
 
       // When running on a pthread, none of the incoming parameters on the module
@@ -333,7 +342,9 @@ var LibraryPThread = {
 #else // WASM2JS
         wasmMemory,
 #endif // WASM2JS
+#if !WASM_ESM_INTEGRATION
         wasmModule,
+#endif
 #if LOAD_SOURCE_MAP
         wasmSourceMap,
 #endif
@@ -391,7 +402,7 @@ var LibraryPThread = {
 #if TRUSTED_TYPES
       // Use Trusted Types compatible wrappers.
       if (typeof trustedTypes != 'undefined' && trustedTypes.createPolicy) {
-        var p = trustedTypes.createPolicy('emscripten#workerPolicy1', { createScriptURL: (ignored) => import.meta.url });
+        var p = trustedTypes.createPolicy('emscripten#workerPolicy1', { createScriptURL: (ignored) => new URL('{{{ pthreadWorkerScript }}}', import.meta.url) });
         worker = new Worker(p.createScriptURL('ignored'), {{{ pthreadWorkerOptions }}});
       } else
 #endif
@@ -409,7 +420,7 @@ var LibraryPThread = {
       // the first case in their bundling step. The latter ends up producing an invalid
       // URL to import from the server (e.g., for webpack the file:// path).
       // See https://github.com/webpack/webpack/issues/12638
-      worker = new Worker(new URL('{{{ TARGET_JS_NAME }}}', import.meta.url), {{{ pthreadWorkerOptions }}});
+      worker = new Worker(new URL('{{{ pthreadWorkerScript }}}', import.meta.url), {{{ pthreadWorkerOptions }}});
 #else // EXPORT_ES6
       var pthreadMainJs = _scriptName;
 #if expectToReceiveOnModule('mainScriptUrlOrBlob')

src/modularize.js

@@ -8,7 +8,7 @@
 // JS program code (INNER_JS_CODE) and wrapping it in a factory function.
 
 #if SOURCE_PHASE_IMPORTS
-import source wasmModule from './{settings.WASM_BINARY_FILE}';
+import source wasmModule from './{{{ WASM_BINARY_FILE }}}';
 #endif
 
 #if ENVIRONMENT_MAY_BE_WEB && !EXPORT_ES6 && !(MINIMAL_RUNTIME && !PTHREADS)

src/postamble.js

@@ -306,6 +306,9 @@ export default async function init(moduleArg = {}) {
   Object.assign(Module, moduleArg);
   processModuleArgs();
 #if WASM_ESM_INTEGRATION
+#if PTHREADS
+  registerTLSInit(__emscripten_tls_init);
+#endif
   updateMemoryViews();
 #if DYNCALLS && '$dynCalls' in addedLibraryItems
 
@@ -318,7 +321,7 @@ export default async function init(moduleArg = {}) {
   run();
 }
 
-#if PTHREADS || WASM_WORKERS
+#if (WASM_WORKERS || PTHREADS) && !WASM_ESM_INTEGRATION
 // When run as a worker thread run `init` immediately.
 if ({{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) await init()
 #endif

src/preamble.js

@@ -990,7 +990,7 @@ function getWasmImports() {
 #endif // WASM_ASYNC_COMPILATION
 #endif // SOURCE_PHASE_IMPORTS
 }
-#endif
+#endif // WASM_ESM_INTEGRATION
 
 #if !WASM_BIGINT
 // Globals used by JS i64 conversions (see makeSetValue)

src/pthread_esm_startup.mjs

@@ -0,0 +1,53 @@
+/**
+ * @license
+ * Copyright 2025 The Emscripten Authors
+ * SPDX-License-Identifier: MIT
+ */
+
+// This file is used as the initial script loaded into pthread workers when
+// running in WASM_ESM_INTEGRATION mode.
+// Tyhe point of this file is to delay the loading of the main program module
+// until the wasm memory has been received via postMessage.
+
+#if RUNTIME_DEBUG
+console.log("Running pthread_esm_startup");
+#endif
+
+#if ENVIRONMENT_MAY_BE_NODE
+// Create as web-worker-like an environment as we can.
+var worker_threads = await import('worker_threads');
+global.Worker = worker_threads.Worker;
+var parentPort = worker_threads['parentPort'];
+parentPort.on('message', (msg) => global.onmessage?.({ data: msg }));
+Object.assign(globalThis, {
+  self: global,
+  postMessage: (msg) => parentPort['postMessage'](msg),
+});
+#endif
+
+self.onmessage = async (msg) => {
+#if RUNTIME_DEBUG
+  console.log('pthread_esm_startup', msg.data.cmd);
+#endif
+  if (msg.data.cmd == 'load') {
+    // Until we initialize the runtime, queue up any further incoming messages
+    // that can arrive while the async import (await import below) is happening.
+    // For examples the `run` message often arrives right away before the import
+    // is complete.
+    let messageQueue = [msg];
+    self.onmessage = (e) => messageQueue.push(e);
+
+    // Now that we have the wasmMemory we can import the main program
+    globalThis.wasmMemory = msg.data.wasmMemory;
+    const prog = await import('./{{{ TARGET_JS_NAME }}}');
+
+    // Now that the import is completed the main program will have installed
+    // its own `onmessage` handler and replaced our handler.
+    // Now we can dispatch any queued messages to this new handler.
+    for (let msg of messageQueue) {
+      await self.onmessage(msg);
+    }
+
+    await prog.default()
+  }
+};

src/runtime_common.js

@@ -29,7 +29,7 @@ var readyPromiseResolve, readyPromiseReject;
 var wasmModuleReceived;
 #endif
 
-#if ENVIRONMENT_MAY_BE_NODE
+#if ENVIRONMENT_MAY_BE_NODE && !WASM_ESM_INTEGRATION
 if (ENVIRONMENT_IS_NODE && {{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) {
   // Create as web-worker-like an environment as we can.
   var parentPort = worker_threads['parentPort'];
@@ -39,7 +39,7 @@ if (ENVIRONMENT_IS_NODE && {{{ ENVIRONMENT_IS_WORKER_THREAD() }}}) {
     postMessage: (msg) => parentPort['postMessage'](msg),
   });
 }
-#endif // ENVIRONMENT_MAY_BE_NODE
+#endif // ENVIRONMENT_MAY_BE_NODE && !WASM_ESM_INTEGRATION
 #endif
 
 #if PTHREADS

src/runtime_init_memory.js

@@ -12,6 +12,14 @@
 // check for full engine support (use string 'subarray' to avoid closure compiler confusion)
 
 function initMemory() {
+#if WASM_ESM_INTEGRATION && PTHREADS
+  if (ENVIRONMENT_IS_PTHREAD) {
+    wasmMemory = globalThis.wasmMemory;
+    assert(wasmMemory);
+    updateMemoryViews();
+  }
+#endif
+
   {{{ runIfWorkerThread('return') }}}
 
 #if expectToReceiveOnModule('wasmMemory')