Add another layer of webpack for the worker

Author: martinRenou

package.json

@@ -35,8 +35,10 @@
     "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/python_data.data lib",
     "build:wasm": "jlpm run build:dockerimage && jlpm run build:emscripten",
     "build:wasm_no_cache": "jlpm run build:dockerimage_no_cache && jlpm run build:emscripten",
-    "build": "jlpm run build:lib && jlpm run copy-files && jlpm run build:labextension:dev",
-    "build:prod": "jlpm run clean && jlpm run build:wasm && jlpm run build:lib && jlpm run copy-files && jlpm run build:labextension",
+    "build:worker": "webpack --config worker.webpack.config.js --mode=development",
+    "build:worker:prod": "webpack --config worker.webpack.config.js --mode=production",
+    "build": "jlpm run build:worker && jlpm run build:lib && jlpm run copy-files && jlpm run build:labextension:dev",
+    "build:prod": "jlpm run clean && jlpm run build:worker:prod && jlpm run build:wasm && jlpm run build:lib && jlpm run copy-files && jlpm run build:labextension",
     "build:labextension": "jupyter labextension build .",
     "build:labextension:dev": "jupyter labextension build --development True .",
     "build:lib": "tsc",
@@ -72,6 +74,8 @@
     "npm-run-all": "^4.1.5",
     "prettier": "^2.1.1",
     "rimraf": "^3.0.2",
+    "source-map-loader": "^4.0.0",
+    "ts-loader": "^9.3.0",
     "typescript": "~4.2.3"
   },
   "sideEffects": [

src/worker.ts

@@ -46,6 +46,8 @@ class XeusPythonKernel {
   mount(driveName: string, mountpoint: string, baseUrl: string): void {
     const { FS, PATH, ERRNO_CODES } = globalThis.Module;
 
+    console.log('mounting drivefs', driveName, mountpoint, baseUrl);
+
     this._drive = new DriveFS({
       FS,
       PATH,
@@ -54,19 +56,28 @@ class XeusPythonKernel {
       driveName,
       mountpoint
     });
+    console.log('mkdir mountpoint');
 
     FS.mkdir(mountpoint);
     FS.mount(this._drive, {}, mountpoint);
+    console.log('chdir mountpoint');
     FS.chdir(mountpoint);
+    console.log('done chdir mountpoint');
   }
 
   cd(path: string) {
+    if (!path) {
+      return;
+    }
+
     const { FS } = globalThis.Module;
 
+    console.log('chdir path', path);
     FS.chdir(path);
+    console.log('done chdir path');
   }
 
-  async processMessage(msg: any): Promise<void> {
+  async processMessage(event: any): Promise<void> {
     await this._ready;
 
     if (
@@ -79,24 +90,34 @@ class XeusPythonKernel {
       globalThis.toplevel_promise = null;
     }
 
-    const msg_type = msg.header.msg_type;
+    console.log('received this in the worker', event);
+
+    const msg_type = event.msg.header.msg_type;
 
     if (msg_type === 'input_reply') {
-      resolveInputReply(msg);
+      resolveInputReply(event.msg);
     } else {
-      this._raw_xserver.notify_listener(msg);
+      this._raw_xserver.notify_listener(event.msg);
     }
   }
 
   private async initialize(resolve: () => void) {
     importScripts('./xpython_wasm.js');
 
+    console.log('init xeus kernel');
+
     globalThis.Module = await createXeusModule({});
 
+    console.log('done init xeus kernel');
+
     importScripts('./python_data.js');
 
+    console.log('loaded python data');
+
     await this.waitRunDependency();
 
+    console.log('waited run deps');
+
     this._raw_xkernel = new globalThis.Module.xkernel();
     this._raw_xserver = this._raw_xkernel.get_server();
 

src/xeus_server_kernel.ts

@@ -91,6 +91,10 @@ export class XeusServerKernel implements IKernel {
    * @param msg The worker message to process.
    */
   private _processWorkerMessage(msg: any): void {
+    if (!msg.header) {
+      return;
+    }
+
     msg.header.session = this._parentHeader?.session ?? '';
     msg.session = this._parentHeader?.session ?? '';
     this._sendMessage(msg);

worker.webpack.config.js

@@ -0,0 +1,40 @@
+const path = require('path');
+const rules = [
+  {
+    test: /\.ts$/,
+    loader: 'ts-loader',
+    options: {
+      configFile: path.resolve('./tsconfig.json')
+    }
+  },
+  {
+      test: /\.js$/,
+      exclude: /node_modules/,
+      loader: 'source-map-loader'
+  }
+];
+
+const resolve = {
+  fallback: {
+    fs: false,
+    child_process: false,
+    crypto: false
+  },
+  extensions: [".ts", ".js"],
+};
+
+module.exports = [
+  {
+    entry: './src/worker.ts',
+    output: {
+      filename: 'worker.js',
+      path: path.resolve(__dirname, 'lib'),
+      libraryTarget: 'amd'
+    },
+    module: {
+      rules
+    },
+    devtool: 'source-map',
+    resolve,
+  }
+];

yarn.lock

@@ -791,7 +791,7 @@ JSONStream@1.3.5:
     jsonparse "^1.2.0"
     through ">=2.2.7 <3"
 
-abab@^2.0.0:
+abab@^2.0.0, abab@^2.0.6:
   version "2.0.6"
   resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291"
   integrity sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA==
@@ -1694,7 +1694,7 @@ end-of-stream@^1.1.0:
   dependencies:
     once "^1.4.0"
 
-enhanced-resolve@^5.9.3:
+enhanced-resolve@^5.0.0, enhanced-resolve@^5.9.3:
   version "5.9.3"
   resolved "https://registry.yarnpkg.com/enhanced-resolve/-/enhanced-resolve-5.9.3.tgz#44a342c012cbc473254af5cc6ae20ebd0aae5d88"
   integrity sha512-Bq9VSor+kjvW3f9/MiiR4eE3XYgOl7/rS8lnSxbRbF3kS0B2r+Y9w5krBWxZgDxASVZbdYrn5wT4j/Wb0J9qow==
@@ -2642,6 +2642,13 @@ iconv-lite@0.4.24, iconv-lite@^0.4.24:
   dependencies:
     safer-buffer ">= 2.1.2 < 3"
 
+iconv-lite@^0.6.3:
+  version "0.6.3"
+  resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501"
+  integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==
+  dependencies:
+    safer-buffer ">= 2.1.2 < 3.0.0"
+
 icss-utils@^5.0.0, icss-utils@^5.1.0:
   version "5.1.0"
   resolved "https://registry.yarnpkg.com/icss-utils/-/icss-utils-5.1.0.tgz#c6be6858abd013d768e98366ae47e25d5887b1ae"
@@ -3383,7 +3390,7 @@ methods@~1.1.2:
   resolved "https://registry.yarnpkg.com/methods/-/methods-1.1.2.tgz#5529a4d67654134edcc5266656835b0f851afcee"
   integrity sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4=
 
-micromatch@^4.0.4:
+micromatch@^4.0.0, micromatch@^4.0.4:
   version "4.0.5"
   resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.5.tgz#bc8999a7cbbf77cdc89f132f6e467051b49090c6"
   integrity sha512-DMy+ERcEW2q8Z2Po+WNXuw3c5YaUSFjAO5GsJqfEl7UjvtIuFKO6ZrKvcItdy98dwFI2N1tg3zNIdKaQT+aNdA==
@@ -4400,7 +4407,7 @@ safe-buffer@5.2.1, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.2:
   resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.2.1.tgz#1eaf9fa9bdb1fdd4ec75f58f9cdb4e6b7827eec6"
   integrity sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==
 
-"safer-buffer@>= 2.1.2 < 3", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
+"safer-buffer@>= 2.1.2 < 3", "safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0:
   version "2.1.2"
   resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
   integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==
@@ -4435,7 +4442,7 @@ schema-utils@^3.0.0, schema-utils@^3.1.0, schema-utils@^3.1.1:
   resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7"
   integrity sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==
 
-semver@7.3.7, semver@^7.2.1, semver@^7.3.2, semver@^7.3.5:
+semver@7.3.7, semver@^7.2.1, semver@^7.3.2, semver@^7.3.4, semver@^7.3.5:
   version "7.3.7"
   resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.7.tgz#12c5b649afdbf9049707796e22a4028814ce523f"
   integrity sha512-QlYTucUYOews+WeEujDoEGziz4K6c47V/Bd+LjSSYcA94p+DmINdf7ncaUinThfvZyu13lN9OY1XDxt8C0Tw0g==
@@ -4623,6 +4630,15 @@ source-map-js@^1.0.2:
   resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"
   integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==
 
+source-map-loader@^4.0.0:
+  version "4.0.0"
+  resolved "https://registry.yarnpkg.com/source-map-loader/-/source-map-loader-4.0.0.tgz#bdc6b118bc6c87ee4d8d851f2d4efcc5abdb2ef5"
+  integrity sha512-i3KVgM3+QPAHNbGavK+VBq03YoJl24m9JWNbLgsjTj8aJzXG9M61bantBTNBt7CNwY2FYf+RJRYJ3pzalKjIrw==
+  dependencies:
+    abab "^2.0.6"
+    iconv-lite "^0.6.3"
+    source-map-js "^1.0.2"
+
 source-map-support@~0.5.20:
   version "0.5.21"
   resolved "https://registry.yarnpkg.com/source-map-support/-/source-map-support-0.5.21.tgz#04fe7c7f9e1ed2d662233c28cb2b35b9f63f6e4f"
@@ -4998,6 +5014,16 @@ tr46@~0.0.3:
   resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
   integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
 
+ts-loader@^9.3.0:
+  version "9.3.0"
+  resolved "https://registry.yarnpkg.com/ts-loader/-/ts-loader-9.3.0.tgz#980f4dbfb60e517179e15e10ed98e454b132159f"
+  integrity sha512-2kLLAdAD+FCKijvGKi9sS0OzoqxLCF3CxHpok7rVgCZ5UldRzH0TkbwG9XECKjBzHsAewntC5oDaI/FwKzEUog==
+  dependencies:
+    chalk "^4.1.0"
+    enhanced-resolve "^5.0.0"
+    micromatch "^4.0.0"
+    semver "^7.3.4"
+
 tslib@^1.8.1, tslib@^1.9.0:
   version "1.14.1"
   resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00"