use empack 3.0 (#134)

* use empack 3.0

* use empack 3.0 everywhere

* revert yarn lock

* revert yarn lock

* revert yarn lock

* revert yarn lock

* no change

* yarn

* no yarn lock

* recreated yarn with 3.x

* yarn 1.x

* Use yarn 3 + missing NPM dependencies

* Fix nodejs version in check-release

* Fix packages versions + proper yarn 3 config

* Fix jupyterlab version in the build env

* Fix jupyterlab builder version

* Fix lab version

* use tar.gz instead of .data

* fix tests

* fix setup.py

* try dev

* ...

* Fix Lumino version

* Fix test workflow

* import proper emapck

* proper syntax

* only tar.gz

* added missing ,

---------

Co-authored-by: martinRenou <martin.renou@gmail.com>

Author: DerThorsten

.github/workflows/build.yml

@@ -30,7 +30,6 @@ jobs:
           jlpm run build
           jlpm run eslint:check
           python -m pip install -v .
-
           jupyter labextension list 2>&1 | grep -ie "@jupyterlite/xeus-python-kernel.*OK"
 
       - name: Build the extension
@@ -68,7 +67,7 @@ jobs:
           sudo rm -rf $(which node)
           sudo rm -rf $(which node)
           pip install jupyterlite-xeus-python.tar.gz
-          pip install jupyterlab
+          pip install "jupyterlab==3"
           jupyter labextension list 2>&1 | grep -ie "@jupyterlite/xeus-python-kernel.*OK"
 
           # TODO: add JupyterLite browser check

.github/workflows/check-release.yml

@@ -24,7 +24,7 @@ jobs:
       - name: Install node
         uses: actions/setup-node@v2
         with:
-          node-version: '14.x'
+          node-version: '16.x'
       - name: Get pip cache dir
         id: pip-cache
         run: |

.gitignore

@@ -4,7 +4,8 @@ node_modules/
 *.egg-info/
 .ipynb_checkpoints
 *.tsbuildinfo
-
+*.tar.gz
+empack_env_meta.json
 # Created by https://www.gitignore.io/api/python
 # Edit at https://www.gitignore.io/?templates=python
 

.yarnrc.yml

@@ -0,0 +1 @@
+nodeLinker: node-modules

docs/build-environment.yml

@@ -14,7 +14,7 @@ dependencies:
   - yarn
   - jupyterlab >=3.5.3,<3.6
   - jupyterlite-core >=0.1.0,<0.2.0
-  - empack >=2.0.9,<3
+  - empack >=3,<4
   - pip:
     - jupyterlite-sphinx
     - ..

environment.yml

@@ -4,12 +4,12 @@ channels:
 dependencies:
   - python
   - pip
-  - jupyterlab
+  - jupyterlab >=3.6,<4
   - traitlets
   - requests
   - typer
   - pytest
-  - emsdk >=3.1.11
-  - empack >=2.0.9,<3
+  - empack >=3,<4
+  - yarn =3
   - pip:
     - jupyterlite

jupyterlite_xeus_python/build.py

@@ -9,7 +9,7 @@
 
 import yaml
 
-from empack.file_packager import split_pack_environment
+from empack.pack import pack_env, DEFAULT_CONFIG_PATH
 from empack.file_patterns import PkgFileFilter, pkg_file_filter_from_yaml
 
 import typer
@@ -27,7 +27,7 @@
 
 PYTHON_VERSION = "3.10"
 
-XEUS_PYTHON_VERSION = "0.15.7"
+XEUS_PYTHON_VERSION = "0.15.9"
 
 CHANNELS = [
     "https://repo.mamba.pm/emscripten-forge",
@@ -199,21 +199,22 @@ def build_and_pack_emscripten_env(
             empack_config_is_url = urlparse(empack_config).scheme in ("http", "https")
             if empack_config_is_url:
                 empack_config_content = requests.get(empack_config).content
-                pack_kwargs["pkg_file_filter"] = PkgFileFilter.parse_obj(
+                pack_kwargs["file_filters"] = PkgFileFilter.parse_obj(
                     yaml.safe_load(empack_config_content)
                 )
             else:
-                pack_kwargs["pkg_file_filter"] = pkg_file_filter_from_yaml(
+                pack_kwargs["file_filters"] = pkg_file_filter_from_yaml(
                     empack_config
                 )
+        else:
+            pack_kwargs["file_filters"] = pkg_file_filter_from_yaml(DEFAULT_CONFIG_PATH)
 
         # Pack the environment
-        split_pack_environment(
+        pack_env(
             env_prefix=prefix_path,
-            outname="python_data",
-            pack_outdir=output_path,
-            export_name="globalThis.Module",
-            with_export_default_statement=False,
+            relocate_prefix="/",
+            outdir=output_path,
+            use_cache=False,
             **pack_kwargs,
         )
 
@@ -235,10 +236,12 @@ def build_and_pack_emscripten_env(
             worker = worker.replace("XEUS_KERNEL_FILE", "'xpython_wasm.js'")
             worker = worker.replace("LANGUAGE_DATA_FILE", "'python_data.js'")
             worker = worker.replace("importScripts(DATA_FILE);", """
-                importScripts(DATA_FILE);
-                await globalThis.Module.importPackages();
-                await globalThis.Module.init();
-            """ )
+                await globalThis.Module.bootstrap_from_empack_packed_environment(
+                `./empack_env_meta.json`, /* packages_json_url */
+                ".",               /* package_tarballs_root_url */
+                false              /* verbose */
+            );
+            """)
             with open(Path(output_path) / "worker.ts", "w") as fobj:
                 fobj.write(worker)
 

jupyterlite_xeus_python/env_build_addon.py

@@ -12,7 +12,7 @@
 
 from traitlets import List, Unicode
 
-from empack.file_packager import pack_environment
+from empack.pack import pack_env, DEFAULT_CONFIG_PATH
 from empack.file_patterns import PkgFileFilter, pkg_file_filter_from_yaml
 
 from jupyterlite_core.constants import (
@@ -94,26 +94,20 @@ def post_build(self, manager):
         # (make jupyterlite-xeus-python extension somewhat configurable?)
         dest = self.output_extensions / "@jupyterlite" / "xeus-python-kernel" / "static"
 
-        # copy *.data/*.js for all side packages
+        # copy *.tar.gz for all side packages
         for item in Path(self.cwd.name) .iterdir():
-            if item.suffix == ".data":
+            if item.suffix == ".gz":
 
                 file = item.name
                 yield dict(
                     name=f"xeus:copy:{file}",
                     actions=[(self.copy_one, [item, dest / file])],
                 )
 
-                js_item  = Path(self.cwd.name) / (str(item.stem) + '.js')
-                js_file = js_item.name
-                yield dict(
-                    name=f"xeus:copy:{js_file}",
-                    actions=[(self.copy_one, [js_item, dest / js_file])],
-                )
 
 
         for file in [
-            "python_data.js",
+            "empack_env_meta.json",
             "xpython_wasm.js",
             "xpython_wasm.wasm",
         ]:

package.json

@@ -26,7 +26,7 @@
     "url": "https://github.com/jupyterlite/xeus-python-kernel.git"
   },
   "scripts": {
-    "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/*.data src/*.js lib",
+    "copy-files": "copyfiles -u 1 src/xpython_wasm.wasm src/xpython_wasm.js src/python_data.js src/*.tar.gz src/*.js src/*.json lib",
     "build:worker": "webpack --config worker.webpack.config.js --mode=development",
     "build": "jlpm run build:xeus-python && jlpm run build:lib && jlpm run build:worker && jlpm run copy-files && jlpm run build:labextension:dev",
     "build:xeus-python": "python jupyterlite_xeus_python/build.py --output-path src --build-worker",
@@ -51,12 +51,17 @@
     "watch:labextension": "jupyter labextension watch ."
   },
   "dependencies": {
+    "@jupyterlab/coreutils": "^5",
+    "@jupyterlab/services": "^6",
     "@jupyterlite/contents": "^0.1.0",
+    "@jupyterlite/kernel": "^0.1.0",
     "@jupyterlite/server": "^0.1.0",
+    "@lumino/coreutils": "^1",
+    "@lumino/signaling": "^1",
     "comlink": "^4.3.1"
   },
   "devDependencies": {
-    "@jupyterlab/builder": "^3.4.1",
+    "@jupyterlab/builder": "^3.6.4",
     "@typescript-eslint/eslint-plugin": "^4.8.1",
     "@typescript-eslint/parser": "^4.8.1",
     "copy-webpack-plugin": "^9.0.1",
@@ -68,9 +73,12 @@
     "npm-run-all": "^4.1.5",
     "prettier": "^2.1.1",
     "rimraf": "^3.0.2",
+    "shx": "^0.3.0",
     "source-map-loader": "^4.0.0",
     "typescript": "~4.2.3",
-    "shx": "^0.3.0"
+    "webpack": "^5.87.0",
+    "webpack-cli": "^5.1.4",
+    "webpack-dev-server": "^4.15.1"
   },
   "sideEffects": [
     "style/*.css",

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["jupyter_packaging~=0.10,<2", "jupyterlab~=3.1", "empack>=2.0.9,<3"]
+requires = ["jupyter_packaging~=0.10,<2", "jupyterlab~=3.1", "empack>=3,<4"]
 build-backend = "jupyter_packaging.build_api"
 
 [tool.jupyter-packaging.options]
@@ -14,14 +14,14 @@ build_cmd = "build:prod"
 npm = ["jlpm"]
 
 [tool.check-manifest]
-ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/*.data", "src/*.js", "src/worker.ts", "src/web_worker_kernel.ts", "*.sh"]
+ignore = ["share/jupyter/labextensions/@jupyterlite/xeus-python-kernel/**", "yarn.lock", ".*", "package-lock.json", "src/xpython_wasm.js", "src/xpython_wasm.wasm", "src/*.tar.gz", "src/*.js", "src/worker.ts", "src/web_worker_kernel.ts", "*.sh"]
 
 [tool.jupyter-releaser.options]
 skip = ["check-links"]
 
 [tool.jupyter-releaser.hooks]
 before-build-npm = [
-    "python -m pip install jupyterlab~=3.1 empack~=2.0 jupyter_packaging~=0.10",
+    "python -m pip install jupyterlab~=3.1 empack~=3.0 jupyter_packaging~=0.10",
     "jlpm",
     "jlpm build:prod",
 ]

setup.py

@@ -19,7 +19,7 @@
 # Representative files that should exist after a successful build
 ensured_targets = [
     str(lab_path / "package.json"),
-    str(lab_path / "static" / "python_data.js"),
+    str(lab_path / "static" / "empack_env_meta.json"),
     str(lab_path / "static" / "xpython_wasm.js"),
     str(lab_path / "static" / "xpython_wasm.wasm"),
     str(lab_path / "static" / "style.js"),
@@ -61,7 +61,7 @@
         "traitlets",
         "jupyterlite-core>=0.1.0",
         "requests",
-        "empack>=2.0.9,<3",
+        "empack>=3,<4",
         "typer",
     ],
     zip_safe=False,

tests/test_xeus_python_env.py

@@ -27,9 +27,9 @@ def test_python_env():
     assert os.path.isfile("/tmp/xeus-python-kernel/envs/xeus-python-kernel/bin/xpython_wasm.wasm")
 
     # Check empack output
-    assert os.path.isfile(Path(addon.cwd.name) / "python_data.js")
+    assert os.path.isfile(Path(addon.cwd.name) / "empack_env_meta.json")
 
-    os.remove(Path(addon.cwd.name) / "python_data.js")
+    os.remove(Path(addon.cwd.name) / "empack_env_meta.json")
 
 
 def test_python_env_from_file_1():
@@ -50,6 +50,6 @@ def test_python_env_from_file_1():
     assert os.path.isfile("/tmp/xeus-python-kernel/envs/xeus-python-kernel-1/bin/xpython_wasm.wasm")
 
     # Check empack output
-    assert os.path.isfile(Path(addon.cwd.name) / "python_data.js")
+    assert os.path.isfile(Path(addon.cwd.name) / "empack_env_meta.json")
 
-    os.remove(Path(addon.cwd.name) / "python_data.js")
+    os.remove(Path(addon.cwd.name) / "empack_env_meta.json")

webpack.config.js

@@ -13,17 +13,17 @@ module.exports = {
           to: '.'
         },
         {
-          from: 'src/python_data.js',
-          to: '.'
-        },
-        {
-          from: 'src/*.data',
-          to: './[name].data'
+          from: 'src/*.gz',
+          to: './[name].gz'
         },
         {
           from: 'src/*.js',
           to: './[name].js'
         },
+        {
+          from: 'src/empack_env_meta.json',
+          to: '.'
+        }
       ]
     })
   ]