Publish a dual CJS/ESM package with platform-specific loaders (#167)

Proper async loading of the WASM module, with custom loaders for different environments.

Author: sandhose

.eslintrc.js renamed to .eslintrc.cjs

@@ -1,6 +1,17 @@
 # UNRELEASED
 
 -   Update matrix-rusk-sdk to `e99939db857ca`.
+-   The published package is now a proper dual CommonJS/ESM package.
+-   The WebAssembly module is now loaded using `fetch` on Web platforms, reducing
+    the bundle size significantly, as well as the time it takes to compile it.
+    ([#167](https://github.com/matrix-org/matrix-rust-sdk-crypto-wasm/pull/167))
+
+**BREAKING CHANGES**
+
+-   The WebAssembly module is no longer synchronously loaded on Web platforms
+    when used. This means that the `initAsync` function **must** be called before any
+    other functions are used. The behaviour is unchanged and still available on
+    Node.js.
 
 # matrix-sdk-crypto-wasm v11.0.0
 

.prettierrc.js renamed to .prettierrc.cjs

@@ -27,26 +27,27 @@ Encryption](https://en.wikipedia.org/wiki/End-to-end_encryption)) for
 
 2. Import the library into your project and initialise it.
 
-    It is recommended that you use a dynamic import, particularly in a Web
-    environment, because the WASM artifiact is large:
+    On Web platforms, the library must be initialised by calling `initAsync`
+    before it can be used, else it will throw an error. This is also recommended
+    on other platforms, as it allows the WebAssembly module to be loaded
+    asynchronously.
 
     ```javascript
+    import { initAsync, Tracing, LoggerLevel, OlmMachine, UserId, DeviceId } from "@matrix-org/matrix-sdk-crypto-wasm";
+
     async function loadCrypto(userId, deviceId) {
-        const matrixSdkCrypto = await import("@matrix-org/matrix-sdk-crypto-wasm");
-        await matrixSdkCrypto.initAsync();
+        // Do this before any other calls to the library
+        await initAsync();
 
         // Optional: enable tracing in the rust-sdk
-        new matrixSdkCrypto.Tracing(matrixSdkCrypto.LoggerLevel.Trace).turnOn();
+        new Tracing(LoggerLevel.Trace).turnOn();
 
         // Create a new OlmMachine
         //
         // The following will use an in-memory store. It is recommended to use
         // indexedDB where that is available.
         // See https://matrix-org.github.io/matrix-rust-sdk-crypto-wasm/classes/OlmMachine.html#initialize
-        const olmMachine = await matrixSdkCrypto.OlmMachine.initialize(
-            new matrixSdkCrypto.UserId(userId),
-            new matrixSdkCrypto.DeviceId(deviceId),
-        );
+        const olmMachine = await OlmMachine.initialize(new UserId(userId), new DeviceId(deviceId));
 
         return olmMachine;
     }

CHANGELOG.md

@@ -0,0 +1,24 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+export * from "./pkg/matrix_sdk_crypto_wasm.d";
+
+/**
+ * Load the WebAssembly module in the background, if it has not already been loaded.
+ *
+ * Returns a promise which will resolve once the other methods are ready.
+ *
+ * @returns {Promise<void>}
+ */
+export function initAsync(): Promise<void>;

README.md

@@ -0,0 +1,93 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// @ts-check
+
+/**
+ * This is the entrypoint on non-node CommonJS environments.
+ * `asyncLoad` will load the WASM module using a `fetch` call.
+ */
+
+const bindings = require("./pkg/matrix_sdk_crypto_wasm_bg.cjs");
+
+const moduleUrl = require.resolve("./pkg/matrix_sdk_crypto_wasm_bg.wasm");
+
+// We want to throw an error if the user tries to use the bindings before
+// calling `initAsync`.
+bindings.__wbg_set_wasm(
+    new Proxy(
+        {},
+        {
+            get() {
+                throw new Error(
+                    "@matrix-org/matrix-sdk-crypto-wasm was used before it was initialized. Call `initAsync` first.",
+                );
+            },
+        },
+    ),
+);
+
+/**
+ * Stores a promise of the `loadModule` call
+ * @type {Promise<void> | null}
+ */
+let modPromise = null;
+
+/**
+ * Loads the WASM module asynchronously
+ *
+ * @returns {Promise<void>}
+ */
+async function loadModule() {
+    let mod;
+    if (typeof WebAssembly.compileStreaming === "function") {
+        mod = await WebAssembly.compileStreaming(fetch(moduleUrl));
+    } else {
+        // Fallback to fetch and compile
+        const response = await fetch(moduleUrl);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch wasm module: ${moduleUrl}`);
+        }
+        const bytes = await response.arrayBuffer();
+        mod = await WebAssembly.compile(bytes);
+    }
+
+    /** @type {{exports: typeof import("./pkg/matrix_sdk_crypto_wasm_bg.wasm.d")}} */
+    // @ts-expect-error: Typescript doesn't know what the instance exports exactly
+    const instance = new WebAssembly.Instance(mod, {
+        // @ts-expect-error: The bindings don't exactly match the 'ExportValue' type
+        "./matrix_sdk_crypto_wasm_bg.js": bindings,
+    });
+
+    bindings.__wbg_set_wasm(instance.exports);
+    instance.exports.__wbindgen_start();
+}
+
+/**
+ * Load the WebAssembly module in the background, if it has not already been loaded.
+ *
+ * Returns a promise which will resolve once the other methods are ready.
+ *
+ * @returns {Promise<void>}
+ */
+async function initAsync() {
+    if (!modPromise) modPromise = loadModule();
+    await modPromise;
+}
+
+module.exports = {
+    // Re-export everything from the generated javascript wrappers
+    ...bindings,
+    initAsync,
+};

index.d.ts

@@ -0,0 +1,90 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// @ts-check
+
+/**
+ * This is the entrypoint on non-node ESM environments (such as Element Web).
+ * `asyncLoad` will load the WASM module using a `fetch` call.
+ */
+
+import * as bindings from "./pkg/matrix_sdk_crypto_wasm_bg.js";
+
+const moduleUrl = new URL("./pkg/matrix_sdk_crypto_wasm_bg.wasm", import.meta.url);
+
+// We want to throw an error if the user tries to use the bindings before
+// calling `initAsync`.
+bindings.__wbg_set_wasm(
+    new Proxy(
+        {},
+        {
+            get() {
+                throw new Error(
+                    "@matrix-org/matrix-sdk-crypto-wasm was used before it was initialized. Call `initAsync` first.",
+                );
+            },
+        },
+    ),
+);
+
+/**
+ * Stores a promise of the `loadModule` call
+ * @type {Promise<void> | null}
+ */
+let modPromise = null;
+
+/**
+ * Loads the WASM module asynchronously
+ *
+ * @returns {Promise<void>}
+ */
+async function loadModule() {
+    let mod;
+    if (typeof WebAssembly.compileStreaming === "function") {
+        mod = await WebAssembly.compileStreaming(fetch(moduleUrl));
+    } else {
+        // Fallback to fetch and compile
+        const response = await fetch(moduleUrl);
+        if (!response.ok) {
+            throw new Error(`Failed to fetch wasm module: ${moduleUrl}`);
+        }
+        const bytes = await response.arrayBuffer();
+        mod = await WebAssembly.compile(bytes);
+    }
+
+    /** @type {{exports: typeof import("./pkg/matrix_sdk_crypto_wasm_bg.wasm.d")}} */
+    // @ts-expect-error: Typescript doesn't know what the instance exports exactly
+    const instance = new WebAssembly.Instance(mod, {
+        // @ts-expect-error: The bindings don't exactly match the 'ExportValue' type
+        "./matrix_sdk_crypto_wasm_bg.js": bindings,
+    });
+
+    bindings.__wbg_set_wasm(instance.exports);
+    instance.exports.__wbindgen_start();
+}
+
+/**
+ * Load the WebAssembly module in the background, if it has not already been loaded.
+ *
+ * Returns a promise which will resolve once the other methods are ready.
+ *
+ * @returns {Promise<void>}
+ */
+export async function initAsync() {
+    if (!modPromise) modPromise = loadModule();
+    await modPromise;
+}
+
+// Re-export everything from the generated javascript wrappers
+export * from "./pkg/matrix_sdk_crypto_wasm_bg.js";

index.js

@@ -0,0 +1,118 @@
+// Copyright 2024 The Matrix.org Foundation C.I.C.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// @ts-check
+
+/**
+ * This is the entrypoint on node-compatible CommonJS environments.
+ * `asyncLoad` will use `fs.readFile` to load the WASM module.
+ */
+
+const { readFileSync } = require("node:fs");
+const { readFile } = require("node:fs/promises");
+const path = require("node:path");
+const bindings = require("./pkg/matrix_sdk_crypto_wasm_bg.cjs");
+
+const filename = path.join(__dirname, "pkg/matrix_sdk_crypto_wasm_bg.wasm");
+
+// In node environments, we want to automatically load the WASM module
+// synchronously if the consumer did not call `initAsync`. To do so, we install
+// a `Proxy` that will intercept calls to the WASM module.
+bindings.__wbg_set_wasm(
+    new Proxy(
+        {},
+        {
+            get(_target, prop) {
+                const mod = loadModuleSync();
+                return initInstance(mod)[prop];
+            },
+        },
+    ),
+);
+
+/**
+ * Stores a promise which resolves to the WebAssembly module
+ * @type {Promise<WebAssembly.Module> | null}
+ */
+let modPromise = null;
+
+/**
+ * Tracks whether the module has been instantiated or not
+ * @type {boolean}
+ */
+let initialised = false;
+
+/**
+ * Loads the WASM module synchronously
+ *
+ * It will throw if there is an attempt to laod the module asynchronously running
+ *
+ * @returns {WebAssembly.Module}
+ */
+function loadModuleSync() {
+    if (modPromise) throw new Error("The WASM module is being loadded asynchronously but hasn't finished");
+    const bytes = readFileSync(filename);
+    return new WebAssembly.Module(bytes);
+}
+
+/**
+ * Loads the WASM module asynchronously
+ *
+ * @returns {Promise<WebAssembly.Module>}
+ */
+async function loadModule() {
+    const bytes = await readFile(filename);
+    return await WebAssembly.compile(bytes);
+}
+
+/**
+ * Initializes the WASM module and returns the exports from the WASM module.
+ *
+ * @param {WebAssembly.Module} mod
+ * @returns {typeof import("./pkg/matrix_sdk_crypto_wasm_bg.wasm.d")}
+ */
+function initInstance(mod) {
+    if (initialised) throw new Error("initInstance called twice");
+
+    /** @type {{exports: typeof import("./pkg/matrix_sdk_crypto_wasm_bg.wasm.d")}} */
+    // @ts-expect-error: Typescript doesn't know what the instance exports exactly
+    const instance = new WebAssembly.Instance(mod, {
+        // @ts-expect-error: The bindings don't exactly match the 'ExportValue' type
+        "./matrix_sdk_crypto_wasm_bg.js": bindings,
+    });
+
+    bindings.__wbg_set_wasm(instance.exports);
+    instance.exports.__wbindgen_start();
+    initialised = true;
+    return instance.exports;
+}
+
+/**
+ * Load the WebAssembly module in the background, if it has not already been loaded.
+ *
+ * Returns a promise which will resolve once the other methods are ready.
+ *
+ * @returns {Promise<void>}
+ */
+async function initAsync() {
+    if (initialised) return;
+    if (!modPromise) modPromise = loadModule().then(initInstance);
+    await modPromise;
+}
+
+module.exports = {
+    // Re-export everything from the generated javascript wrappers
+    ...bindings,
+    initAsync,
+};