Skip to content

JS Packaging and bundling with ESBuild #8792

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 17 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build_test_deploy.yml
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ jobs:
env:
WK_VERSION: ${{ github.run_id }}

- name: Build webknossos (webpack)
- name: Build webknossos (esbuild)
run: yarn build

- name: Build webknossos (sbt)
Expand Down
9 changes: 1 addition & 8 deletions app/views/main.scala.html
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,6 @@
<meta name="robot" content="noindex" />
}
<link rel="shortcut icon" type="image/png" href="/assets/images/favicon.png" />
<link
rel="stylesheet"
type="text/css"
media="screen"
href="/assets/bundle/vendors~main.css?nocache=@(webknossos.BuildInfo.commitHash)"
/>
<link
rel="stylesheet"
type="text/css"
Expand All @@ -44,8 +38,7 @@
data-airbrake-project-key="@(conf.Airbrake.projectKey)"
data-airbrake-environment-name="@(conf.Airbrake.environment)"
></script>
<script src="/assets/bundle/vendors~main.js?nocache=@(webknossos.BuildInfo.commitHash)"></script>
<script src="/assets/bundle/main.js?nocache=@(webknossos.BuildInfo.commitHash)"></script>
<script src="/assets/bundle/main.js?nocache=@(webknossos.BuildInfo.commitHash)" type="module"></script>
<script type="text/javascript" src="https://app.olvy.co/script.js" defer="defer"></script>
</head>
<body>
Expand Down
163 changes: 163 additions & 0 deletions esbuild_config.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,163 @@
const esbuild = require("esbuild");
const path = require("node:path");
const fs = require("node:fs");
const os = require("node:os");

const srcPath = path.resolve(__dirname, "frontend/javascripts/");
const outputPath = path.resolve(__dirname, "public/bundle/");
const protoPath = path.join(__dirname, "webknossos-datastore/proto");

// Community plugins
const browserslistToEsbuild = require("browserslist-to-esbuild");
const { lessLoader } = require("esbuild-plugin-less");
const copyPlugin = require("esbuild-plugin-copy").default;
const polyfillNode = require("esbuild-plugin-polyfill-node").polyfillNode;
const esbuildPluginWorker = require("@chialab/esbuild-plugin-worker").default;


// Custom Plugins for Webknossos
const { createWorkerPlugin } = require("./tools/esbuild/workerPlugin.js");
const { createProtoPlugin } = require("./tools/esbuild/protoPlugin.js");

const target = browserslistToEsbuild([
"last 3 Chrome versions",
"last 3 Firefox versions",
"last 2 Edge versions",
"last 1 Safari versions",
"last 1 iOS versions",
]);

async function build(env = {}) {
const isProduction = env.production || process.env.NODE_ENV === "production";
const isWatch = env.watch;

// Determine output directory for bundles.
// In watch mode, it's a temp dir. In production, it's the public bundle dir.
const buildOutDir = isWatch
? fs.mkdtempSync(path.join(os.tmpdir(), "esbuild-dev"))
: outputPath;

// Base plugins
const plugins = [
polyfillNode(),
createProtoPlugin(protoPath),
lessLoader({
javascriptEnabled: true,
}),
copyPlugin({
patterns: [
{
from: "node_modules/@zip.js/zip.js/dist/z-worker.js",
to: path.join(buildOutDir, "z-worker.js"),
},
],
}),
createWorkerPlugin(buildOutDir, srcPath, target, polyfillNode, lessLoader, __dirname, env.logLevel), // Resolves import Worker from myFunc.worker;
esbuildPluginWorker() // Resolves new Worker(myWorker.js)
];


const buildOptions = {
entryPoints: {
main: path.resolve(srcPath, "main.tsx"),
// generateMeshBVHWorker: require.resolve("three-mesh-bvh/src/workers/generateMeshBVH.worker.js"),

},
bundle: true,
outdir: buildOutDir,
format: "esm",
target: target,
platform: "browser",
splitting: true,
chunkNames: "[name].[hash]",
assetNames: "[name].[hash]",
sourcemap: isProduction ? "external" : "inline",
minify: isProduction,
define: {
"process.env.NODE_ENV": JSON.stringify(isProduction ? "production" : "development"),
"process.env.BABEL_ENV": JSON.stringify(process.env.BABEL_ENV || "development"),
"process.browser": "true",
"global": "window"
},
loader: {
".woff": "file",
".woff2": "file",
".ttf": "file",
".eot": "file",
".svg": "file",
".png": "file",
".jpg": "file",
".wasm": "file",
},
resolveExtensions: [".ts", ".tsx", ".js", ".json", ".proto", ".wasm"],
alias: {
react: path.resolve(__dirname, "node_modules/react"),
three: path.resolve(__dirname, "node_modules/three/src/Three.js"),
url: require.resolve("url/"),
},
plugins: plugins,
external: ["/assets/images/*", "fs", "path", "util", "module"],
publicPath: "/assets/bundle/",
metafile: !isWatch, // Don"t generate metafile for dev server
logOverride: {
"direct-eval": "silent",
},
};

if (env.watch) {
// Development server mode
const ctx = await esbuild.context(buildOptions);

const { host, port } = await ctx.serve({
servedir: buildOutDir,
port: env.PORT || 9002,
onRequest: (args) => {
if (env.logLevel === "verbose") {
console.log(`[${args.method}] ${args.path} - status ${args.status}`);
}
},
});

console.log(`Development server running at http://${host}:${port}`);
console.log(`Serving files from temporary directory: ${buildOutDir}`);

await ctx.watch();

process.on("SIGINT", async () => {
await ctx.dispose();
process.exit(0);
});
} else {
// Production build
const result = await esbuild.build(buildOptions);

if (result.metafile) {
await fs.promises.writeFile(
path.join(buildOutDir, "metafile.json"),
JSON.stringify(result.metafile, null, 2)
);
}

console.log("Build completed successfully!");
}
}

module.exports = { build };

// If called directly
if (require.main === module) {
const args = process.argv.slice(2);
const env = {
logLevel: "info", // Default log level
};

args.forEach(arg => {
if (arg === "--production") env.production = true;
if (arg === "--watch") env.watch = true;
if (arg.startsWith("--port=")) env.PORT = Number.parseInt(arg.split("=")[1]);
if (arg === "--verbose") env.logLevel = "verbose";
if (arg === "--silent") env.logLevel = "silent";
});

build(env).catch(console.error);
}
1 change: 0 additions & 1 deletion frontend/javascripts/libs/DRACOLoader.ts
Original file line number Diff line number Diff line change
Expand Up @@ -237,7 +237,6 @@ class DRACOLoader extends Loader {
_getWorker(taskID, taskCost) {
return this._initDecoder().then(() => {
if (this.workerPool.length < this.workerLimit) {
// See https://webpack.js.org/guides/web-workers/
const worker = new Worker(new URL("./DRACOWorker.worker.js", import.meta.url));

worker._callbacks = {};
Expand Down
6 changes: 0 additions & 6 deletions frontend/javascripts/test/_ava_polyfill_provider.ts

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import deepFreezeLib from "deep-freeze";
import _ from "lodash";

// Do not use the deep-freeze library in production
// process.env.NODE_ENV is being substituted by webpack
// process.env.NODE_ENV is being substituted by esbuild
let deepFreeze = deepFreezeLib;
if (process.env.NODE_ENV === "production") deepFreeze = _.identity;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import {
} from "@ant-design/icons";
import { Tree as AntdTree, Dropdown, type GetRef, Space, Tooltip, type TreeProps } from "antd";
import type { EventDataNode } from "antd/es/tree";
import { useLifecycle } from "beautiful-react-hooks";
import useLifecycle from "beautiful-react-hooks/useLifecycle";
import { InputKeyboard } from "libs/input";
import { useEffectOnlyOnce } from "libs/react_hooks";
import { useWkSelector } from "libs/react_hooks";
Expand Down
19 changes: 13 additions & 6 deletions frontend/javascripts/viewer/workers/comlink_wrapper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,14 @@ import {
throwTransferHandlerWithResponseSupport,
} from "viewer/workers/headers_transfer_handler";

// Worker modules export bare functions and typically instantiated with new Worker("./path/to/my/worker.js");

// JS bundlers (like esbuild) try to match these calls and generate new entry points/bundles for the worker code.
// However, our esbuild pipeline is a bit different:
// 1) We import our worker code with regular import statements, e.g. import worker from './my.worker';
// 2) We consolidated all worker/Comlink stuff into this wrapper, calling new Worker(workerFunction). Since workerFunction is a variable it is usually not identified by esbuild as it matches not pattern.
// Similar to the webpack worker-loader, we have a custom esbuild plugin to load worker codes. See tools/esbuild/workerPlugin.js.

function importComlink() {
const isNodeContext = typeof process !== "undefined" && process.title !== "browser";

Expand Down Expand Up @@ -36,8 +44,7 @@ const { wrap, transferHandlers, _expose, _transfer } = importComlink();
transferHandlers.set("requestOptions", requestOptionsTransferHandler);
// Overwrite the default throw handler with ours that supports responses.
transferHandlers.set("throw", throwTransferHandlerWithResponseSupport);
// Worker modules export bare functions, but webpack turns these into Worker classes which need to be
// instantiated first.

// To ensure that code always executes the necessary instantiation, we cheat a bit with the typing in the following code.
// In reality, `expose` receives a function and returns it again. However, we tell flow that it wraps the function, so that
// unwrapping becomes necessary.
Expand All @@ -47,18 +54,18 @@ type UseCreateWorkerToUseMe<T> = {
readonly _wrapped: T;
};
export function createWorker<T extends (...args: any) => any>(
WorkerClass: UseCreateWorkerToUseMe<T>,
workerFunction: UseCreateWorkerToUseMe<T>,
): (...params: Parameters<T>) => Promise<ReturnType<T>> {
if (wrap == null) {
// In a node context (e.g., when executing tests), we don't create web workers which is why
// we can simply return the input function here.
// @ts-ignore
return WorkerClass;
// @ts-expect-error
return workerFunction;
}

return wrap(
// @ts-ignore
new WorkerClass(),
new Worker(workerFunction),
);
}
export function expose<T>(fn: T): UseCreateWorkerToUseMe<T> {
Expand Down
2 changes: 1 addition & 1 deletion frontend/javascripts/viewer/workers/readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,6 @@ See `compress.worker.js` for an example.
- Accessing global state (e.g., the Store) is not directly possible from web workers, since they have their own execution context. Pass necessary information into web workers via parameters.
- By default, parameters and return values are either structurally cloned or transferred (if they support it) to/from the web worker. Copying is potentially performance-intensive and also won't propagate any mutations across the main-thread/webworker border. If objects are transferable (e.g., for ArrayBuffers, but not TypedArrays), they are moved to the new context, which means that they cannot be accessed in the old thread, anymore. In both cases, care has to be taken. In general, web workers should only be responsible for a very small (but cpu intensive) task with a bare minimum of dependencies.
- Not all objects can be passed between main thread and web workers (e.g., Header objects). For these cases, you have to implement and register a specific transfer handler for the object type. See `headers_transfer_handler.js` as an example.
- Web worker files can import NPM modules and also modules from within this code base, but beware that the execution context between the main thread and web workers is strictly isolated. Webpack will create a separate JS file for each web worker into which all imported code is compiled.
- Web worker files can import NPM modules and also modules from within this code base, but beware that the execution context between the main thread and web workers is strictly isolated. esbuild will create a separate JS file for each web worker into which all imported code is compiled.

Learn more about the Comlink module we use [here](https://github.com/GoogleChromeLabs/comlink).
29 changes: 11 additions & 18 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
},
"devDependencies": {
"@biomejs/biome": "^1.9.4",
"@chialab/esbuild-plugin-worker": "^0.18.1",
"@redux-saga/testing-utils": "^1.1.5",
"@shaderfrog/glsl-parser": "^0.3.0",
"@types/color-hash": "^1.0.2",
Expand All @@ -35,20 +36,19 @@
"@vitest/coverage-v8": "3.1.1",
"abort-controller": "^3.0.0",
"browserslist-to-esbuild": "^1.2.0",
"copy-webpack-plugin": "^12.0.2",
"coveralls": "^3.0.2",
"css-loader": "^6.5.1",
"dependency-cruiser": "^16.10.0",
"documentation": "^14.0.2",
"dpdm": "^3.14.0",
"esbuild": "^0.25",
"esbuild": "^0.25.8",
"esbuild-plugin-copy": "^2.1.1",
"esbuild-plugin-less": "^1.3.24",
"esbuild-plugin-polyfill-node": "^0.3.0",
"espree": "^3.5.4",
"husky": "^9.1.5",
"jsdoc": "^3.5.5",
"jsdom": "^26.1.0",
"json-loader": "^0.5.7",
"less": "^4.0.0",
"less-loader": "^10.2.0",
"lz4-wasm-nodejs": "^0.9.2",
"merge-img": "^2.1.2",
"pg": "^7.4.1",
Expand All @@ -59,25 +59,21 @@
"redux-mock-store": "^1.2.2",
"shelljs": "^0.8.5",
"tmp": "0.0.33",
"ts-loader": "^9.4.1",
"typescript": "^5.8.0",
"typescript-coverage-report": "^0.8.0",
"vite-tsconfig-paths": "^5.1.4",
"vitest": "^3.1.1",
"webpack": "^5.97.1",
"webpack-cli": "^5.1.4",
"webpack-dev-server": "^5.2.0"
"vitest": "^3.1.1"
},
"scripts": {
"start": "node tools/proxy/proxy.js",
"build": "node --max-old-space-size=4096 node_modules/.bin/webpack --env production",
"build": "node esbuild_config.js --production",
"@comment build-backend": "Only check for errors in the backend code like done by the CI. This command is not needed to run WEBKNOSSOS",
"build-backend": "yarn build-wk-backend && yarn build-wk-datastore && yarn build-wk-tracingstore && rm webknossos-tracingstore/conf/messages webknossos-datastore/conf/messages",
"build-wk-backend": "sbt -no-colors -DfailOnWarning compile stage",
"build-wk-datastore": "sbt -no-colors -DfailOnWarning \"project webknossosDatastore\" copyMessages compile stage",
"build-wk-tracingstore": "sbt -no-colors -DfailOnWarning \"project webknossosTracingstore\" copyMessages compile stage",
"build-dev": "node_modules/.bin/webpack",
"build-watch": "node_modules/.bin/webpack -w",
"build-dev": "node esbuild_config.js",
"build-watch": "node esbuild_config.js --watch",
"listening": "lsof -i:5005,7155,9000,9001,9002",
"kill-listeners": "kill -9 $(lsof -t -i:5005,7155,9000,9001,9002)",
"rm-fossil-lock": "rm fossildb/data/LOCK",
Expand Down Expand Up @@ -135,7 +131,7 @@
"antd": "5.22",
"ball-morphology": "^0.1.0",
"base64-js": "^1.2.1",
"beautiful-react-hooks": "^3.11.1",
"beautiful-react-hooks": "^3.12",
"chalk": "^5.0.1",
"classnames": "^2.2.5",
"color-hash": "^2.0.1",
Expand All @@ -147,7 +143,6 @@
"deep-freeze": "0.0.1",
"dice-coefficient": "^2.1.0",
"distance-transform": "^1.0.2",
"esbuild-loader": "^4.1.0",
"file-saver": "^2.0.1",
"flexlayout-react": "0.7.15",
"hammerjs": "^2.0.8",
Expand All @@ -160,7 +155,6 @@
"lz-string": "^1.4.4",
"lz4-wasm": "^0.9.2",
"memoize-one": "^6.0.0",
"mini-css-extract-plugin": "^2.5.2",
"minisearch": "^5.0.0",
"mjs": "^1.0.0",
"ml-matrix": "^6.10.4",
Expand Down Expand Up @@ -196,8 +190,7 @@
"tween.js": "^16.3.1",
"typed-redux-saga": "^1.4.0",
"url": "^0.11.0",
"url-join": "^4.0.0",
"worker-loader": "^3.0.8"
"url-join": "^4.0.0"
},
"packageManager": "yarn@4.9.2"
}
Loading