New emscripten + Fix CI (#108)

* changes for new emscripten

* cleanup

* cleanup

* cleanup

* cleanup

* Put back what has been removed

* Put back filter

* Update empack in build env

* CI split build

* Empack 2.0.6

* Bump empack

* Bump empack

* Use mamba

* Stupid conda

* Syntax error

* Missing eslint ignore entries

* More updates

* Verbose build

* Try

* Empack 2.0.8

* Install empack from conda again

---------

Co-authored-by: DerThorsten <derthorstenbeier@gmail.com>

Author: martinRenou

.eslintignore

@@ -3,3 +3,5 @@ dist
 coverage
 **/*.d.ts
 tests
+src/worker.ts
+src/web_worker_kernel.ts

.github/workflows/build.yml

@@ -16,53 +16,25 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v2
-      - name: Install node
-        uses: actions/setup-node@v1
-        with:
-          node-version: '14.x'
-      - name: Install Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: '3.8'
-          architecture: 'x64'
 
-      - name: Setup pip cache
-        uses: actions/cache@v2
-        with:
-          path: ~/.cache/pip
-          key: pip-3.8-${{ hashFiles('package.json') }}
-          restore-keys: |
-            pip-3.8-
-            pip-
-
-      - name: Get yarn cache directory path
-        id: yarn-cache-dir-path
-        run: echo "::set-output name=dir::$(yarn cache dir)"
-      - name: Setup yarn cache
-        uses: actions/cache@v2
-        id: yarn-cache # use this to check for `cache-hit` (`steps.yarn-cache.outputs.cache-hit != 'true'`)
+      - name: Install Conda environment with Micromamba
+        uses: mamba-org/provision-with-micromamba@main
         with:
-          path: ${{ steps.yarn-cache-dir-path.outputs.dir }}
-          key: yarn-${{ hashFiles('**/yarn.lock') }}
-          restore-keys: |
-            yarn-
-
-      - name: Install dependencies
-        run: python -m pip install -U jupyterlab~=3.1 check-manifest typer empack
+          environment-name: xeus-python-kernel
+          environment-file: environment.yml
 
-      - name: Build the extension
+      - name: Dev install
         run: |
           set -eux
           jlpm
           jlpm run build
           jlpm run eslint:check
-          python -m pip install .
+          python -m pip install -v .
 
           jupyter labextension list 2>&1 | grep -ie "@jupyterlite/xeus-python-kernel.*OK"
 
-          # TODO: add JupyterLite browser check
-          # python -m jupyterlab.browser_check
-
+      - name: Build the extension
+        run: |
           pip install build
           python -m build --sdist
           cp dist/*.tar.gz jupyterlite-xeus-python.tar.gz
@@ -120,7 +92,9 @@ jobs:
           activate-environment: xeus-python-kernel
           environment-file: environment.yml
           python-version: '3.10'
-          mamba-version: '*'
+
+      - name: Install mamba
+        run: conda install -c conda-forge mamba
 
       - name: Make sure the Mamba Python API is available
         run: |
@@ -152,7 +126,9 @@ jobs:
           activate-environment: xeus-python-kernel
           environment-file: environment.yml
           python-version: '3.10'
-          mamba-version: '*'
+
+      - name: Install mamba
+        run: conda install -c conda-forge mamba
 
       - name: Install
         run: pip install jupyterlite-xeus-python.tar.gz

.gitignore

@@ -114,7 +114,8 @@ _output
 *.doit.db
 
 python_data.js
-python_data.data
+*.data
+src/*.js
 xpython_wasm.js
 xpython_wasm.wasm
 xpython_wasm.hash

docs/build-environment.yml

@@ -11,7 +11,7 @@ dependencies:
   - pydata-sphinx-theme
   - yarn
   - jupyterlab
-  - empack>=2.0.3
+  - empack >=2.0.8,<3
   - pip:
     - jupyterlite==0.1.0b17
     - jupyterlite-sphinx

environment.yml

@@ -4,11 +4,12 @@ channels:
 dependencies:
   - python
   - pip
+  - jupyterlab
   - traitlets
   - requests
   - typer
   - pytest
   - emsdk >=3.1.11
-  - empack >=2.0.2
+  - empack >=2.0.8,<3
   - pip:
-      - jupyterlite
+    - jupyterlite

jupyterlite_xeus_python/build.py

@@ -9,7 +9,7 @@
 
 import yaml
 
-from empack.file_packager import pack_environment
+from empack.file_packager import split_pack_environment
 from empack.file_patterns import PkgFileFilter, pkg_file_filter_from_yaml
 
 import typer
@@ -206,10 +206,12 @@ def build_and_pack_emscripten_env(
                 )
 
         # Pack the environment
-        pack_environment(
+        split_pack_environment(
             env_prefix=prefix_path,
-            outname=Path(output_path) / "python_data",
+            outname="python_data",
+            pack_outdir=output_path,
             export_name="globalThis.Module",
+            with_export_default_statement=False,
             **pack_kwargs,
         )
 
@@ -230,9 +232,14 @@ def build_and_pack_emscripten_env(
 
             worker = worker.replace("XEUS_KERNEL_FILE", "'xpython_wasm.js'")
             worker = worker.replace("LANGUAGE_DATA_FILE", "'python_data.js'")
-
+            worker = worker.replace("importScripts(DATA_FILE);", """
+                importScripts(DATA_FILE);
+                await globalThis.Module.importPackages();
+                await globalThis.Module.init();
+            """ )
             with open(Path(output_path) / "worker.ts", "w") as fobj:
                 fobj.write(worker)
+
     except Exception as e:
         raise e
     finally:

jupyterlite_xeus_python/env_build_addon.py

@@ -93,10 +93,27 @@ def post_build(self, manager):
         # We should really find a nicer way.
         # (make jupyterlite-xeus-python extension somewhat configurable?)
         dest = self.output_extensions / "@jupyterlite" / "xeus-python-kernel" / "static"
+        
+        # copy *.data/*.js for all side packages
+        for item in Path(self.cwd.name) .iterdir():
+            if item.suffix == ".data":
+
+                file = item.name 
+                yield dict(
+                    name=f"xeus:copy:{file}",
+                    actions=[(self.copy_one, [item, dest / file])],
+                )
+
+                js_item  = Path(self.cwd.name) / (str(item.stem) + '.js')
+                js_file = js_item.name 
+                yield dict(
+                    name=f"xeus:copy:{js_file}",
+                    actions=[(self.copy_one, [js_item, dest / js_file])],
+                )
+
 
         for file in [
             "python_data.js",
-            "python_data.data",
             "xpython_wasm.js",
             "xpython_wasm.wasm",
         ]:

package.json

@@ -26,7 +26,7 @@
     "url": "https://github.com/jupyterlite/xeus-python-kernel.git"
   },
   "scripts": {
-    "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/python_data.data lib",
+    "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/*.data src/*.js lib",
     "build:worker": "webpack --config worker.webpack.config.js --mode=development",
     "build": "jlpm run build:xeus-python && jlpm run build:lib && jlpm run build:worker && jlpm run copy-files && jlpm run build:labextension:dev",
     "build:xeus-python": "python jupyterlite_xeus_python/build.py --output-path src --build-worker",

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["jupyter_packaging~=0.10,<2", "jupyterlab~=3.1", "empack"]
+requires = ["jupyter_packaging~=0.10,<2", "jupyterlab~=3.1", "empack>=2.0.8,<3"]
 build-backend = "jupyter_packaging.build_api"
 
 [tool.jupyter-packaging.options]
@@ -14,4 +14,4 @@ build_cmd = "build:prod"
 npm = ["jlpm"]
 
 [tool.check-manifest]
-ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/python_data.data", "src/python_data.js", "src/worker.ts", "src/web_worker_kernel.ts", "*.sh"]
+ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/*.data", "src/*.js", "src/worker.ts", "src/web_worker_kernel.ts", "*.sh"]

setup.py

@@ -20,7 +20,6 @@
 ensured_targets = [
     str(lab_path / "package.json"),
     str(lab_path / "static" / "python_data.js"),
-    str(lab_path / "static" / "python_data.data"),
     str(lab_path / "static" / "xpython_wasm.js"),
     str(lab_path / "static" / "xpython_wasm.wasm"),
     str(lab_path / "static" / "style.js"),
@@ -62,7 +61,7 @@
         "traitlets",
         "jupyterlite",
         "requests",
-        "empack>=2.0.2,<3",
+        "empack>=2.0.8,<3",
         "typer",
     ],
     zip_safe=False,

tests/test_xeus_python_env.py

@@ -28,10 +28,8 @@ def test_python_env():
 
     # Check empack output
     assert os.path.isfile(Path(addon.cwd.name) / "python_data.js")
-    assert os.path.isfile(Path(addon.cwd.name) / "python_data.data")
 
     os.remove(Path(addon.cwd.name) / "python_data.js")
-    os.remove(Path(addon.cwd.name) / "python_data.data")
 
 
 def test_python_env_from_file_1():
@@ -53,7 +51,5 @@ def test_python_env_from_file_1():
 
     # Check empack output
     assert os.path.isfile(Path(addon.cwd.name) / "python_data.js")
-    assert os.path.isfile(Path(addon.cwd.name) / "python_data.data")
 
     os.remove(Path(addon.cwd.name) / "python_data.js")
-    os.remove(Path(addon.cwd.name) / "python_data.data")

webpack.config.js

@@ -13,13 +13,17 @@ module.exports = {
           to: '.'
         },
         {
-          from: 'src/python_data.data',
+          from: 'src/python_data.js',
           to: '.'
         },
         {
-          from: 'src/python_data.js',
-          to: '.'
-        }
+          from: 'src/*.data',
+          to: './[name].data'
+        },
+        {
+          from: 'src/*.js',
+          to: './[name].js'
+        },
       ]
     })
   ]