Enable --proxy-to-worker under node. (#19582)

At least enough to run hello world.  The idea here is not so much that
folks really want use `--proxy-to-worker` under node, but more for
testing, so we can run tests where the module is instantiated on a
worker.

In particular we can write tests for issues such as #19577 that run
under node.

Also, because `proxyClient.js` now goes through the preprocessor we can
use the normal `#include` mechanism to include `IDBStore.js`.

Author: sbc100

emcc.py

@@ -4150,12 +4150,11 @@ def generate_html(target, options, js_target, target_basename,
 
 
 def generate_worker_js(target, js_target, target_basename):
-  # compiler output is embedded as base64
   if settings.SINGLE_FILE:
+    # compiler output is embedded as base64
     proxy_worker_filename = get_subresource_location(js_target)
-
-  # compiler output goes in .worker.js file
   else:
+    # compiler output goes in .worker.js file
     move_file(js_target, shared.replace_suffix(js_target, '.worker.js'))
     worker_target_basename = target_basename + '.worker'
     proxy_worker_filename = (settings.PROXY_TO_WORKER_FILENAME or worker_target_basename) + '.js'
@@ -4166,10 +4165,10 @@ def generate_worker_js(target, js_target, target_basename):
 
 def worker_js_script(proxy_worker_filename):
   web_gl_client_src = read_file(utils.path_from_root('src/webGLClient.js'))
-  idb_store_src = read_file(utils.path_from_root('src/IDBStore.js'))
-  proxy_client_src = read_file(utils.path_from_root('src/proxyClient.js'))
-  proxy_client_src = do_replace(proxy_client_src, '{{{ filename }}}', proxy_worker_filename)
-  proxy_client_src = do_replace(proxy_client_src, '{{{ IDBStore.js }}}', idb_store_src)
+  proxy_client_src = shared.read_and_preprocess(utils.path_from_root('src/proxyClient.js'), expand_macros=True)
+  if not os.path.dirname(proxy_worker_filename):
+    proxy_worker_filename = './' + proxy_worker_filename
+  proxy_client_src = do_replace(proxy_client_src, '<<< filename >>>', proxy_worker_filename)
   return web_gl_client_src + '\n' + proxy_client_src
 
 

src/IDBStore.js

@@ -4,7 +4,7 @@
  * SPDX-License-Identifier: MIT
  */
 
-{
+var IDBStore = {
   indexedDB: function() {
     if (typeof indexedDB != 'undefined') return indexedDB;
     var ret = null;
@@ -102,5 +102,4 @@
       req.onerror = (error) => callback(error);
     });
   },
-}
-
+};

src/library_idbstore.js

@@ -4,15 +4,15 @@
  * SPDX-License-Identifier: MIT
  */
 
+#include "IDBStore.js"
+
 var LibraryIDBStore = {
   // A simple IDB-backed storage mechanism. Suitable for saving and loading large files asynchronously. This does
   // *NOT* use the emscripten filesystem, intentionally, to avoid overhead. It lets you application define whatever
   // filesystem-like layer you want, with the overhead 100% controlled by you. At the extremes, you could either
   // just store large files, with almost no extra code; or you could implement a file b-tree using posix-compliant
   // filesystem on top.
-  $IDBStore:
-#include IDBStore.js
-  ,
+  $IDBStore: IDBStore,
   emscripten_idb_async_load__deps: ['$UTF8ToString', 'malloc', 'free'],
   emscripten_idb_async_load: function(db, id, arg, onload, onerror) {
     IDBStore.getFile(UTF8ToString(db), UTF8ToString(id), function(error, byteArray) {

src/proxyClient.js

@@ -6,26 +6,36 @@
 
 // proxy to/from worker
 
+#if ENVIRONMENT_MAY_BE_NODE
+var ENVIRONMENT_IS_NODE = typeof process == 'object' && typeof process.versions == 'object' && typeof process.versions.node == 'string';
+if (ENVIRONMENT_IS_NODE) {
+  let nodeWorkerThreads;
+  try {
+    nodeWorkerThreads = require('worker_threads');
+  } catch (e) {
+    console.error('The "worker_threads" module is not supported in this node.js build - perhaps a newer version is needed?');
+    throw e;
+  }
+  global.Worker = nodeWorkerThreads.Worker;
+  var Module = Module || {}
+} else
+#endif
 if (typeof Module == 'undefined') {
   console.warn('no Module object defined - cannot proxy canvas rendering and input events, etc.');
   Module = {
     canvas: {
-      addEventListener: function() {},
-      getBoundingClientRect: function() { return { bottom: 0, height: 0, left: 0, right: 0, top: 0, width: 0 } },
+      addEventListener: () => {},
+      getBoundingClientRect: () => { return { bottom: 0, height: 0, left: 0, right: 0, top: 0, width: 0 }; },
     },
   };
 }
 
 if (!Module.hasOwnProperty('print')) {
-  Module['print'] = function(x) {
-    console.log(x);
-  };
+  Module['print'] = (x) => console.log(x);
 }
 
 if (!Module.hasOwnProperty('printErr')) {
-  Module['printErr'] = function(x) {
-    console.error(x);
-  };
+  Module['printErr'] = (x) => console.error(x);
 }
 
 // utils
@@ -34,7 +44,7 @@ function FPSTracker(text) {
   var last = 0;
   var mean = 0;
   var counter = 0;
-  this.tick = function() {
+  this.tick = () => {
     var now = Date.now();
     if (last > 0) {
       var diff = now - last;
@@ -52,7 +62,7 @@ function FPSTracker(text) {
 function GenericTracker(text) {
   var mean = 0;
   var counter = 0;
-  this.tick = function(value) {
+  this.tick = (value) => {
     mean = 0.99*mean + 0.01*value;
     if (counter++ === 60) {
       counter = 0;
@@ -79,16 +89,18 @@ function renderFrame() {
   renderFrameData = null;
 }
 
-window.requestAnimationFrame = window.requestAnimationFrame || window.mozRequestAnimationFrame ||
-                               window.webkitRequestAnimationFrame || window.msRequestAnimationFrame ||
-                               renderFrame;
+if (typeof window != 'undefined') {
+  window.requestAnimationFrame = window.requestAnimationFrame || window.mozRequestAnimationFrame ||
+                                 window.webkitRequestAnimationFrame || window.msRequestAnimationFrame ||
+                                 renderFrame;
+}
 
 /*
 (function() {
   var trueRAF = window.requestAnimationFrame;
   var tracker = new FPSTracker('client');
-  window.requestAnimationFrame = function(func) {
-    trueRAF(function() {
+  window.requestAnimationFrame = (func) => {
+    trueRAF(() => {
       tracker.tick();
       func();
     });
@@ -100,7 +112,7 @@ window.requestAnimationFrame = window.requestAnimationFrame || window.mozRequest
 
 // IDBStore
 
-var IDBStore = {{{ IDBStore.js }}};
+#include "IDBStore.js"
 
 // Frame throttling
 
@@ -114,7 +126,7 @@ var SUPPORT_BASE64_EMBEDDING;
 
 var filename;
 if (!filename) {
-  filename = '{{{ filename }}}';
+  filename = '<<< filename >>>';
 }
 
 var workerURL = filename;
@@ -126,9 +138,14 @@ if (SUPPORT_BASE64_EMBEDDING) {
 }
 var worker = new Worker(workerURL);
 
+#if ENVIRONMENT_MAY_BE_NODE
+if (ENVIRONMENT_IS_NODE) {
+  worker.postMessage({target: 'worker-init'});
+} else {
+#endif
 WebGLClient.prefetch();
 
-setTimeout(function() {
+setTimeout(() => {
   worker.postMessage({
     target: 'worker-init',
     width: Module.canvas.width,
@@ -138,10 +155,13 @@ setTimeout(function() {
     currentScriptUrl: filename,
     preMain: true });
 }, 0); // delay til next frame, to make sure html is ready
+#if ENVIRONMENT_MAY_BE_NODE
+}
+#endif
 
 var workerResponded = false;
 
-worker.onmessage = function worker_onmessage(event) {
+worker.onmessage = (event) => {
   //dump('\nclient got ' + JSON.stringify(event.data).substr(0, 150) + '\n');
   if (!workerResponded) {
     workerResponded = true;
@@ -230,7 +250,7 @@ worker.onmessage = function worker_onmessage(event) {
     case 'IDBStore': {
       switch (data.method) {
         case 'loadBlob': {
-          IDBStore.getFile(data.db, data.id, function(error, blob) {
+          IDBStore.getFile(data.db, data.id, (error, blob) => {
             worker.postMessage({
               target: 'IDBStore',
               method: 'response',
@@ -240,7 +260,7 @@ worker.onmessage = function worker_onmessage(event) {
           break;
         }
         case 'storeBlob': {
-          IDBStore.setFile(data.db, data.id, data.blob, function(error) {
+          IDBStore.setFile(data.db, data.id, data.blob, (error) => {
             worker.postMessage({
               target: 'IDBStore',
               method: 'response',
@@ -282,6 +302,10 @@ function cloneObject(event) {
   return ret;
 };
 
+#if ENVIRONMENT_MAY_BE_NODE
+if (!ENVIRONMENT_IS_NODE) {
+#endif
+
 // Only prevent default on backspace/tab because we don't want unexpected navigation.
 // Do not prevent default on the rest as we need the keypress event.
 function shouldPreventDefault(event) {
@@ -292,26 +316,30 @@ function shouldPreventDefault(event) {
   }
 };
 
-['keydown', 'keyup', 'keypress', 'blur', 'visibilitychange'].forEach(function(event) {
-  document.addEventListener(event, function(event) {
+
+['keydown', 'keyup', 'keypress', 'blur', 'visibilitychange'].forEach((event) => {
+  document.addEventListener(event, (event) => {
     worker.postMessage({ target: 'document', event: cloneObject(event) });
-    
+
     if (shouldPreventDefault(event)) {
       event.preventDefault();
     }
   });
 });
 
-['unload'].forEach(function(event) {
-  window.addEventListener(event, function(event) {
+['unload'].forEach((event) => {
+  window.addEventListener(event, (event) => {
     worker.postMessage({ target: 'window', event: cloneObject(event) });
   });
 });
 
-['mousedown', 'mouseup', 'mousemove', 'DOMMouseScroll', 'mousewheel', 'mouseout'].forEach(function(event) {
-  Module.canvas.addEventListener(event, function(event) {
+['mousedown', 'mouseup', 'mousemove', 'DOMMouseScroll', 'mousewheel', 'mouseout'].forEach((event) => {
+  Module.canvas.addEventListener(event, (event) => {
     worker.postMessage({ target: 'canvas', event: cloneObject(event) });
     event.preventDefault();
   }, true);
 });
 
+#if ENVIRONMENT_MAY_BE_NODE
+}
+#endif

src/proxyWorker.js

@@ -4,27 +4,6 @@
  * SPDX-License-Identifier: MIT
  */
 
-if (typeof console == 'undefined') {
-  // we can't call Module.printErr because that might be circular
-  var console = {
-    log: function(x) {
-      if (typeof dump == 'function') dump(`log: ${x}\n`);
-    },
-    debug: function(x) {
-      if (typeof dump == 'function') dump(`debug: ${x}\n`);
-    },
-    info: function(x) {
-      if (typeof dump == 'function') dump(`info: ${x}\n`);
-    },
-    warn: function(x) {
-      if (typeof dump == 'function') dump(`warn: ${x}\n`);
-    },
-    error: function(x) {
-      if (typeof dump == 'function') dump(`error: ${x}\n`);
-    },
-  };
-}
-
 /*
 function proxify(object, nick) {
   return new Proxy(object, {
@@ -37,6 +16,10 @@ function proxify(object, nick) {
 }
 */
 
+#if ENVIRONMENT_MAY_BE_NODE
+if (!ENVIRONMENT_IS_NODE) {
+#endif
+
 function FPSTracker(text) {
   var last = 0;
   var mean = 0;
@@ -354,13 +337,13 @@ var screen = {
 
 Module.canvas = document.createElement('canvas');
 
-Module.setStatus = function(){};
+Module.setStatus = () => {};
 
-out = function Module_print(x) {
+out = function(x) {
   //dump('OUT: ' + x + '\n');
   postMessage({ target: 'stdout', content: x });
 };
-err = function Module_printErr(x) {
+err = function(x) {
   //dump('ERR: ' + x + '\n');
   postMessage({ target: 'stderr', content: x });
 };
@@ -389,6 +372,10 @@ if (!ENVIRONMENT_IS_PTHREAD) {
 }
 #endif
 
+#if ENVIRONMENT_MAY_BE_NODE
+}
+#endif
+
 // buffer messages until the program starts to run
 
 var messageBuffer = null;
@@ -479,6 +466,9 @@ function onMessageFromMainEmscriptenThread(message) {
       screen.width = Module.canvas.width_ = message.data.width;
       screen.height = Module.canvas.height_ = message.data.height;
       Module.canvas.boundingClientRect = message.data.boundingClientRect;
+#if ENVIRONMENT_MAY_BE_NODE
+      if (ENVIRONMENT_IS_NODE)
+#endif
       document.URL = message.data.URL;
 #if PTHREADS
       currentScriptUrl = message.data.currentScriptUrl;

test/test_browser.py

@@ -1759,14 +1759,12 @@ def test_glgears_pthreads(self, extra_args=[]):  # noqa
 
   @requires_graphics_hardware
   @parameterized({
-    '': ([False],),
+    '': ([],),
     # Enabling FULL_ES3 also enables ES2 automatically
-    'proxy': ([True],)
+    'proxy': (['--proxy-to-worker'],)
   })
-  def test_glgears_long(self, proxy):
-    args = ['-DHAVE_BUILTIN_SINCOS', '-DLONGTEST', '-lGL', '-lglut', '-DANIMATE']
-    if proxy:
-      args += ['--proxy-to-worker']
+  def test_glgears_long(self, args):
+    args += ['-DHAVE_BUILTIN_SINCOS', '-DLONGTEST', '-lGL', '-lglut', '-DANIMATE']
     self.btest('hello_world_gles.c', expected='0', args=args)
 
   @requires_graphics_hardware

test/test_other.py

@@ -13534,3 +13534,6 @@ def test_preload_module(self, args):
   def test_standalone_whole_archive(self):
     self.emcc_args += ['-sSTANDALONE_WASM', '-pthread', '-Wl,--whole-archive', '-lbulkmemory', '-lstandalonewasm', '-Wl,--no-whole-archive']
     self.do_runf(test_file('hello_world.c'))
+
+  def test_proxy_to_worker(self):
+    self.do_runf(test_file('hello_world.c'), emcc_args=['--proxy-to-worker'])