From 1f062724e04a728313524cf77251e95a27580769 Mon Sep 17 00:00:00 2001 From: Brentley Jones Date: Mon, 24 Mar 2025 12:54:06 -0500 Subject: [PATCH] feat: add ability to cache json responses in-repo Besides a performance improvement, it allows cross-platform builds to work (e.g. Linux host building Apple code). Signed-off-by: Brentley Jones --- swiftpkg/bzlmod/swift_deps.bzl | 43 ++++++++++++++++++-- swiftpkg/internal/local_swift_package.bzl | 15 ++++++- swiftpkg/internal/pkg_ctxs.bzl | 2 + swiftpkg/internal/pkginfos.bzl | 20 ++++++++- swiftpkg/internal/registry_swift_package.bzl | 8 ++++ swiftpkg/internal/repository_utils.bzl | 26 +++++++++++- swiftpkg/internal/swift_package.bzl | 14 ++++++- 7 files changed, 119 insertions(+), 9 deletions(-) diff --git a/swiftpkg/bzlmod/swift_deps.bzl b/swiftpkg/bzlmod/swift_deps.bzl index b7913e104..d4a0ce22e 100644 --- a/swiftpkg/bzlmod/swift_deps.bzl +++ b/swiftpkg/bzlmod/swift_deps.bzl @@ -1,6 +1,7 @@ """Implementation for `swift_deps` bzlmod extension.""" load("@bazel_skylib//lib:dicts.bzl", "dicts") +load("@bazel_skylib//lib:paths.bzl", "paths") load("//swiftpkg/internal:bazel_repo_names.bzl", "bazel_repo_names") load("//swiftpkg/internal:local_swift_package.bzl", "local_swift_package") load("//swiftpkg/internal:pkginfos.bzl", "pkginfos") @@ -59,11 +60,21 @@ def _declare_pkgs_from_package(module_ctx, from_package, config_pkgs, config_swi # Get the package info. pkg_swift = module_ctx.path(from_package.swift) debug_path = module_ctx.path(".") + workspace_root = str(pkg_swift.dirname) + + root_cached_json_directory = None + if from_package.cached_json_directory: + root_cached_json_directory = paths.join( + workspace_root, + from_package.cached_json_directory, + ) + pkg_info = pkginfos.get( module_ctx, - directory = str(pkg_swift.dirname), + directory = workspace_root, env = env, debug_path = str(debug_path), + cached_json_directory = root_cached_json_directory, resolved_pkg_map = resolved_pkg_map, collect_src_info = False, registries_directory = registries_directory, @@ -157,10 +168,18 @@ the Swift package to make it available.\ processing = to_process to_process = [] for dep in processing: + dep_cached_json_directory = None + + if from_package.cached_json_directory: + dep_cached_json_directory = paths.join( + dep.file_system.path, + from_package.cached_json_directory, + ) dep_pkg_info = pkginfos.get( module_ctx, directory = dep.file_system.path, debug_path = None, + cached_json_directory = dep_cached_json_directory, resolved_pkg_map = None, collect_src_info = False, ) @@ -185,7 +204,13 @@ the Swift package to make it available.\ config_pkg = config_pkgs.get( bazel_repo_names.from_identity(dep.identity), ) - _declare_pkg_from_dependency(dep, config_pkg, from_package, config_swift_package) + _declare_pkg_from_dependency( + dep, + config_pkg, + from_package, + config_swift_package, + from_package.cached_json_directory, + ) # Add all transitive dependencies to direct_dep_repo_names if `publicly_expose_all_targets` flag is set. for dep in all_deps_by_id.values(): @@ -199,7 +224,15 @@ the Swift package to make it available.\ return direct_dep_repo_names -def _declare_pkg_from_dependency(dep, config_pkg, from_package, config_swift_package): +def _declare_pkg_from_dependency( + dep, + config_pkg, + from_package, + config_swift_package, + cached_json_directory): + if cached_json_directory: + cached_json_directory = paths.join(cached_json_directory, dep.name) + name = bazel_repo_names.from_identity(dep.identity) if dep.source_control: init_submodules = None @@ -236,7 +269,7 @@ def _declare_pkg_from_dependency(dep, config_pkg, from_package, config_swift_pac dependencies_index = None, env = from_package.env, env_inherit = from_package.env_inherit, - init_submodules = init_submodules, + cached_json_directory = cached_json_directory, init_submodules = init_submodules, recursive_init_submodules = recursive_init_submodules, patch_args = patch_args, patch_cmds = patch_cmds, @@ -256,6 +289,7 @@ def _declare_pkg_from_dependency(dep, config_pkg, from_package, config_swift_pac env_inherit = from_package.env_inherit, path = dep.file_system.path, dependencies_index = None, + cached_json_directory = cached_json_directory, ) elif dep.registry: @@ -350,6 +384,7 @@ bazel run @swift_package//:resolve ``` """, ), + "cached_json_directory": attr.string(), "env": attr.string_dict( doc = """\ Environment variables that will be passed to the execution environments for \ diff --git a/swiftpkg/internal/local_swift_package.bzl b/swiftpkg/internal/local_swift_package.bzl index 79ee56581..f697a625f 100644 --- a/swiftpkg/internal/local_swift_package.bzl +++ b/swiftpkg/internal/local_swift_package.bzl @@ -62,7 +62,19 @@ def _local_swift_package_impl(repository_ctx): repo_rules.write_workspace_file(repository_ctx, repo_dir) # Generate the build file - pkg_ctx = pkg_ctxs.read(repository_ctx, repo_dir, env) + cached_json_directory = None + if repository_ctx.attr.cached_json_directory: + cached_json_directory = paths.join( + str(repository_ctx.workspace_root), + repository_ctx.attr.cached_json_directory, + ) + + pkg_ctx = pkg_ctxs.read( + repository_ctx, + repo_dir, + env, + cached_json_directory, + ) repo_rules.gen_build_files(repository_ctx, pkg_ctx) return update_attrs(repository_ctx.attr, _ALL_ATTRS.keys(), {}) @@ -78,6 +90,7 @@ _ALL_ATTRS = dicts.add( repo_rules.env_attrs, repo_rules.swift_attrs, _PATH_ATTRS, + {"cached_json_directory": attr.string()}, ) local_swift_package = repository_rule( diff --git a/swiftpkg/internal/pkg_ctxs.bzl b/swiftpkg/internal/pkg_ctxs.bzl index 806d34a9a..399030e84 100644 --- a/swiftpkg/internal/pkg_ctxs.bzl +++ b/swiftpkg/internal/pkg_ctxs.bzl @@ -7,6 +7,7 @@ def _read( repository_ctx, repo_dir, env, + cached_json_directory, resolved_pkg_map = None, registries_directory = None, replace_scm_with_registry = False): @@ -14,6 +15,7 @@ def _read( repository_ctx = repository_ctx, directory = repo_dir, env = env, + cached_json_directory = cached_json_directory, resolved_pkg_map = resolved_pkg_map, registries_directory = registries_directory, replace_scm_with_registry = replace_scm_with_registry, diff --git a/swiftpkg/internal/pkginfos.bzl b/swiftpkg/internal/pkginfos.bzl index 9d33a666c..596c61b16 100644 --- a/swiftpkg/internal/pkginfos.bzl +++ b/swiftpkg/internal/pkginfos.bzl @@ -28,6 +28,7 @@ def _get_dump_manifest( env = {}, working_directory = "", debug_path = None, + cache_path = None, registries_directory = None, replace_scm_with_registry = False): """Returns a dict representing the package dump for an SPM package. @@ -52,6 +53,10 @@ def _get_dump_manifest( debug_path = str(repository_ctx.path(".")) debug_json_path = paths.join(debug_path, "dump.json") + cached_json_path = None + if cache_path: + cached_json_path = paths.join(cache_path, "dump.json") + args = ["swift", "package"] if registries_directory: @@ -66,6 +71,7 @@ def _get_dump_manifest( env = env, working_directory = working_directory, debug_json_path = debug_json_path, + cached_json_path = cached_json_path, ) def _get_desc_manifest( @@ -73,6 +79,7 @@ def _get_desc_manifest( env = {}, working_directory = "", debug_path = None, + cache_path = None, registries_directory = None, replace_scm_with_registry = False): """Returns a dict representing the package description for an SPM package. @@ -97,6 +104,10 @@ def _get_desc_manifest( debug_path = str(repository_ctx.path(".")) debug_json_path = paths.join(debug_path, "desc.json") + cached_json_path = None + if cache_path: + cached_json_path = paths.join(cache_path, "desc.json") + args = ["swift", "package"] if registries_directory: @@ -112,6 +123,7 @@ def _get_desc_manifest( env = env, working_directory = working_directory, debug_json_path = debug_json_path, + cached_json_path = cached_json_path, ) def _get( @@ -119,6 +131,7 @@ def _get( directory, env = {}, debug_path = None, + cached_json_directory = None, resolved_pkg_map = None, collect_src_info = True, registries_directory = None, @@ -150,11 +163,13 @@ def _get( if not paths.is_absolute(debug_path): # For backwards compatibility, resolve relative to the working directory. debug_path = paths.join(directory, debug_path) + dump_manifest = _get_dump_manifest( repository_ctx, env = env, working_directory = directory, debug_path = debug_path, + cache_path = cached_json_directory, registries_directory = registries_directory, replace_scm_with_registry = replace_scm_with_registry, ) @@ -163,6 +178,7 @@ def _get( env = env, working_directory = directory, debug_path = debug_path, + cache_path = cached_json_directory, registries_directory = registries_directory, replace_scm_with_registry = replace_scm_with_registry, ) @@ -1648,7 +1664,9 @@ def _new_resource_rule_process(localization = None): def _new_resource_from_desc_map(desc_map, pkg_path): path = desc_map["path"] - if paths.is_absolute(path): + if path.startswith("./"): + path = path[2:] + elif paths.is_absolute(path): path = paths.relativize(path, pkg_path) return _new_resource( path = path, diff --git a/swiftpkg/internal/registry_swift_package.bzl b/swiftpkg/internal/registry_swift_package.bzl index 7948edf1e..73e4bd919 100644 --- a/swiftpkg/internal/registry_swift_package.bzl +++ b/swiftpkg/internal/registry_swift_package.bzl @@ -235,10 +235,18 @@ def _registry_swift_package_impl(repository_ctx): else: resolved_pkg_map = dict() + cached_json_directory = None + if repository_ctx.attr.cached_json_directory: + cached_json_directory = paths.join( + str(repository_ctx.workspace_root), + repository_ctx.attr.cached_json_directory, + ) + pkg_ctx = pkg_ctxs.read( repository_ctx, directory, env, + cached_json_directory, resolved_pkg_map = resolved_pkg_map, registries_directory = registries_directory, replace_scm_with_registry = attr.replace_scm_with_registry, diff --git a/swiftpkg/internal/repository_utils.bzl b/swiftpkg/internal/repository_utils.bzl index 1dad0607a..97c230187 100644 --- a/swiftpkg/internal/repository_utils.bzl +++ b/swiftpkg/internal/repository_utils.bzl @@ -78,17 +78,39 @@ def _parsed_json_from_spm_command( arguments, env = {}, working_directory = "", - debug_json_path = None): + debug_json_path = None, + cached_json_path = None): + if cached_json_path: + cached_json_path_path = repository_ctx.path(cached_json_path) + if cached_json_path_path.exists: + return json.decode( + repository_ctx.read(cached_json_path_path, watch = "yes"), + ) + json_str = repository_utils.exec_spm_command( repository_ctx, arguments, env = env, working_directory = working_directory, - ) + ).replace(working_directory, ".") + if debug_json_path: if not paths.is_absolute(debug_json_path): debug_json_path = paths.join(working_directory, debug_json_path) repository_ctx.file(debug_json_path, content = json_str, executable = False) + if cached_json_path: + if debug_json_path: + tmp_path = debug_json_path + else: + tmp_path = paths.join( + str(repository_ctx.path(".")), + paths.basename(cached_json_path), + ) + repository_ctx.file(tmp_path, content = json_str, executable = False) + repository_ctx.execute(["mkdir", "-p", paths.dirname(cached_json_path)]) + repository_ctx.execute(["cp", tmp_path, cached_json_path]) + repository_ctx.watch(cached_json_path) + return json.decode(json_str) def _package_name(repository_ctx): diff --git a/swiftpkg/internal/swift_package.bzl b/swiftpkg/internal/swift_package.bzl index 2cae8d596..4196b3507 100644 --- a/swiftpkg/internal/swift_package.bzl +++ b/swiftpkg/internal/swift_package.bzl @@ -1,6 +1,7 @@ """Implementation for `swift_package`.""" load("@bazel_skylib//lib:dicts.bzl", "dicts") +load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_tools//tools/build_defs/repo:git_worker.bzl", "git_repo") load( "@bazel_tools//tools/build_defs/repo:utils.bzl", @@ -68,10 +69,18 @@ def _swift_package_impl(repository_ctx): registries_directory = None # Generate the build file + cached_json_directory = None + if repository_ctx.attr.cached_json_directory: + cached_json_directory = paths.join( + str(repository_ctx.workspace_root), + repository_ctx.attr.cached_json_directory, + ) + pkg_ctx = pkg_ctxs.read( repository_ctx, directory, env, + cached_json_directory, registries_directory = registries_directory, replace_scm_with_registry = replace_scm_with_registry, ) @@ -212,7 +221,10 @@ _ALL_ATTRS = dicts.add( _GIT_ATTRS, repo_rules.env_attrs, repo_rules.swift_attrs, - {"version": attr.string(doc = "The resolved version of the package.")}, + { + "cached_json_directory": attr.string(), + "version": attr.string(doc = "The resolved version of the package."), + }, ) swift_package = repository_rule(