Skip to content

[Bazel] Dynamically generate the Emscripten cache #1402

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 1 commit into from
Closed

[Bazel] Dynamically generate the Emscripten cache #1402

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
91 changes: 84 additions & 7 deletions bazel/emscripten_deps.bzl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ def _parse_version(v):

BUILD_FILE_CONTENT_TEMPLATE = """
package(default_visibility = ['//visibility:public'])
load("@bazel_skylib//rules:write_file.bzl", "write_file")

filegroup(
name = "all",
Expand All @@ -26,6 +27,7 @@ filegroup(
filegroup(
name = "emcc_common",
srcs = [
"emscripten/embuilder.py",
"emscripten/emcc.py",
"emscripten/emscripten-version.txt",
"emscripten/cache/sysroot_install.stamp",
Expand Down Expand Up @@ -68,6 +70,7 @@ filegroup(
"bin/wasm-split{bin_extension}",
"bin/wasm2js{bin_extension}",
":emcc_common",
":embuilder",
] + glob(
include = [
"emscripten/cache/sysroot/lib/**",
Expand All @@ -94,11 +97,65 @@ filegroup(
],
),
)

write_file(
name = "embuilder_config",
out = "emscripten_config",
)

genrule(
name = "embuilder",
tools = [
":emscripten/embuilder.py",
":compiler_files",
":ar_files"
],
srcs = [":embuilder_config"],
cmd = \"\"\"
export EM_BINARYEN_ROOT=$$(realpath $$(dirname $$(dirname $(location :emscripten/embuilder.py))))
export EM_LLVM_ROOT=$$EM_BINARYEN_ROOT/bin
export EM_EMSCRIPTEN_ROOT=$$EM_BINARYEN_ROOT/emscripten
export EM_CACHE=$(RULEDIR)/emscripten/cache
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These values normally part of the config file. See https://github.com/emscripten-core/emsdk/blob/main/bazel/emscripten_toolchain/emscripten_config

I think the only one you want to override here is EM_CACHE. The others should all come from that config file.

Copy link
Contributor Author

@allsey87 allsey87 Jun 13, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The problem there (I think/I will test this) is that emscripten_config is part of the @emsdk// while this code is inside @emscripten_bin_XXX// so this might create a circular dependency.

I think the only one you want to override here is EM_CACHE. The others should all come from that config file.

Also the values of EMSCRIPTEN_ROOT, BINARYEN_ROOT, LLVM_ROOT in emscripten_config are based on environment variables which are set in env.sh/env.bat, which in turn, depends on environment variables set in toolchain.bzl.

$(location :emscripten/embuilder.py) \
--em-config $(location :embuilder_config) \
{embuilder_build_args}
\"\"\",
outs = {embuilder_build_outs}
)
"""

def emscripten_deps(emscripten_version = "latest"):
version = emscripten_version
def emscripten_deps(
emscripten_version = "latest",
embuilder_args = ["--pic"],
embuilder_libs = ["crtbegin", "libprintf_long_double-debug"]
):
embuilder_mode = []
if "--lto=thin" in embuilder_args:
embuilder_mode.append("thinlto")
elif "--lto" in embuilder_args:
embuilder_mode.append("lto")
if "--pic" in embuilder_args:
embuilder_mode.append("pic")
if "--wasm64" in embuilder_args:
embuilder_output_path = "wasm64-emscripten/"
else:
embuilder_output_path = "wasm32-emscripten/"
if embuilder_mode:
embuilder_output_path += "{}/".format("-".join(embuilder_mode))

# build up the command line for embuilder
embuilder_build_args = " ".join(embuilder_args + ["build"] + embuilder_libs)

# TODO how to map libs to output names? Some end with .o and others with .a
embuilder_build_outs = [
"emscripten/cache/sysroot/lib/{}crtbegin.o".format(embuilder_output_path),
"emscripten/cache/sysroot/lib/{}libprintf_long_double-debug.a".format(embuilder_output_path),
]

embuilder_build_outs = "[" + ", ".join(["\"{}\"".format(out) for out in embuilder_build_outs]) + "]"
print(embuilder_build_outs)

version = emscripten_version
if version == "latest":
version = reversed(sorted(EMSCRIPTEN_TAGS.keys(), key = _parse_version))[0]

Expand Down Expand Up @@ -127,7 +184,11 @@ def emscripten_deps(emscripten_version = "latest"):
strip_prefix = "install",
url = emscripten_url.format("linux", revision.hash, "", "tar.xz"),
sha256 = revision.sha_linux,
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ""),
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(
embuilder_build_args = embuilder_build_args,
embuilder_build_outs = embuilder_build_outs,
bin_extension = ""
),
type = "tar.xz",
)

Expand All @@ -138,7 +199,11 @@ def emscripten_deps(emscripten_version = "latest"):
url = emscripten_url.format("linux", revision.hash, "-arm64", "tar.xz"),
# Not all versions have a linux/arm64 release: https://github.com/emscripten-core/emsdk/issues/547
sha256 = getattr(revision, "sha_linux_arm64", None),
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ""),
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(
embuilder_build_args = embuilder_build_args,
embuilder_build_outs = embuilder_build_outs,
bin_extension = ""
),
type = "tar.xz",
)

Expand All @@ -148,7 +213,11 @@ def emscripten_deps(emscripten_version = "latest"):
strip_prefix = "install",
url = emscripten_url.format("mac", revision.hash, "", "tar.xz"),
sha256 = revision.sha_mac,
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ""),
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(
embuilder_build_args = embuilder_build_args,
embuilder_build_outs = embuilder_build_outs,
bin_extension = ""
),
type = "tar.xz",
)

Expand All @@ -158,7 +227,11 @@ def emscripten_deps(emscripten_version = "latest"):
strip_prefix = "install",
url = emscripten_url.format("mac", revision.hash, "-arm64", "tar.xz"),
sha256 = revision.sha_mac_arm64,
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ""),
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(
embuilder_build_args = embuilder_build_args,
embuilder_build_outs = embuilder_build_outs,
bin_extension = ""
),
type = "tar.xz",
)

Expand All @@ -168,7 +241,11 @@ def emscripten_deps(emscripten_version = "latest"):
strip_prefix = "install",
url = emscripten_url.format("win", revision.hash, "", "zip"),
sha256 = revision.sha_win,
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(bin_extension = ".exe"),
build_file_content = BUILD_FILE_CONTENT_TEMPLATE.format(
embuilder_build_args = embuilder_build_args,
embuilder_build_outs = embuilder_build_outs,
bin_extension = ".exe"
),
type = "zip",
)

Expand Down