Fix BBB to run on latest master (#97)

Author: staticfloat

.github/workflows/ci.yml

@@ -37,10 +37,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - name: Setup Julia
-        run: |
-          curl -LsS "https://github.com/JuliaPackaging/BinaryBuilderBase.jl/releases/download/tmp/julia-377aa809eb-linux64.tar.gz" | tar -xz
-          echo "${PWD}/julia-377aa809eb/bin" >> ${GITHUB_PATH}
+      - uses: julia-actions/setup-julia@v1
+        with:
+          version: "1.6-nightly"
+          arch: x64
       - uses: julia-actions/julia-buildpkg@latest
       - name: System info
         run: julia --project=. --color=yes -e "using BinaryBuilderBase; BinaryBuilderBase.versioninfo()"
@@ -56,7 +56,7 @@ jobs:
       - uses: actions/checkout@v2
       - uses: julia-actions/setup-julia@latest
         with:
-          version: nightly
+          version: 1.6-nightly
       - name: Cache artifacts
         uses: actions/cache@v2
         env:

Artifacts.toml

@@ -2476,6 +2476,28 @@ os = "linux"
     sha256 = "8b685aa858e864049c3d95b59a3763b2e6236d7af9f49f357287da4d4802aa47"
     url = "https://github.com/JuliaPackaging/Yggdrasil/releases/download/Rootfs-v2021.1.12/Rootfs.v2021.1.12.x86_64-linux-musl.unpacked.tar.gz"
 
+[["Rootfs.v2021.1.22.x86_64-linux-musl.squashfs"]]
+arch = "x86_64"
+git-tree-sha1 = "c00439a5aac2cd79e14acf22c6fd329297be0559"
+lazy = true
+libc = "musl"
+os = "linux"
+
+    [["Rootfs.v2021.1.22.x86_64-linux-musl.squashfs".download]]
+    sha256 = "0ac9b959afd58282751e0ca04b13cf8fc090e18f43dee03d3b27a964318f5136"
+    url = "https://github.com/JuliaPackaging/Yggdrasil/releases/download/Rootfs-v2021.1.22/Rootfs.v2021.1.22.x86_64-linux-musl.squashfs.tar.gz"
+
+[["Rootfs.v2021.1.22.x86_64-linux-musl.unpacked"]]
+arch = "x86_64"
+git-tree-sha1 = "e61855d1fe79df053c1d5ebfe0c7e75e8b899b23"
+lazy = true
+libc = "musl"
+os = "linux"
+
+    [["Rootfs.v2021.1.22.x86_64-linux-musl.unpacked".download]]
+    sha256 = "aac80e9ba2f6551fe5eee54c23de2a5bb0bc3d3173a591df0a1c67bab5b9fd55"
+    url = "https://github.com/JuliaPackaging/Yggdrasil/releases/download/Rootfs-v2021.1.22/Rootfs.v2021.1.22.x86_64-linux-musl.unpacked.tar.gz"
+
 [["RustBase.v1.43.0.x86_64-linux-musl.squashfs"]]
 arch = "x86_64"
 git-tree-sha1 = "bcd3deb4047a5fdae6abf9808da90e7de1be3924"

Manifest.toml

@@ -43,13 +43,17 @@ deps = ["LibCURL_jll", "MozillaCACerts_jll"]
 uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
 
 [[LibCURL_jll]]
-deps = ["Libdl"]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
 uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
 
 [[LibGit2]]
 deps = ["Base64", "NetworkOptions", "Printf", "SHA"]
 uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
 
+[[LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+
 [[Libdl]]
 uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
 
@@ -60,6 +64,10 @@ uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
 deps = ["Base64"]
 uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
 
+[[MbedTLS_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+
 [[Mmap]]
 uuid = "a63ad114-7e13-5084-954f-fe012c677804"
 
@@ -99,6 +107,12 @@ uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 [[SHA]]
 uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
 
+[[Scratch]]
+deps = ["Dates"]
+git-tree-sha1 = "ad4b278adb62d185bbcb6864dc24959ab0627bf6"
+uuid = "6c6a2e73-6563-6170-7368-637461726353"
+version = "1.0.3"
+
 [[Serialization]]
 uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
 
@@ -136,16 +150,16 @@ uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
 
 [[Zlib_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "32085929ad61a5ed99abea288ec7242be392bb8b"
+deps = ["Libdl"]
 uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
-version = "1.2.12+0"
+
+[[nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
 
 [[p7zip_jll]]
-deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"]
-git-tree-sha1 = "ee65cfa19bea645698a0224bfa216f2b1c8b559f"
+deps = ["Artifacts", "Libdl"]
 uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
-version = "16.2.0+3"
 
 [[pigz_jll]]
 deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"]

Project.toml

@@ -15,6 +15,7 @@ OutputCollectors = "6c11c7d4-943b-4e2b-80de-f2cfc2930a8c"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 SHA = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+Scratch = "6c6a2e73-6563-6170-7368-637461726353"
 SimpleBufferStream = "777ac1f9-54b0-4bf8-805c-2214025038e7"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Tar = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"

src/BinaryBuilderBase.jl

@@ -3,7 +3,7 @@ module BinaryBuilderBase
 using Pkg, Pkg.Artifacts, Random, Libdl, InteractiveUtils
 using Base.BinaryPlatforms
 using Downloads
-using JSON, OutputCollectors
+using JSON, OutputCollectors, Scratch
 
 # Re-export useful stuff from Base.BinaryPlatforms:
 export HostPlatform, platform_dlext, valid_dl_path, arch, libc,

src/Prefix.jl

@@ -453,22 +453,23 @@ function setup_dependencies(prefix::Prefix, dependencies::Vector{PkgSpec}, platf
             Pkg.Types.PackageSpec(name=p.name, uuid=u, tree_hash=p.tree_hash) for (u, p) in ctx.env.manifest if endswith(p.name, "_jll")
         ]
 
-        # Some JLLs are also standard libraries that may be present in the
-        # manifest because pulled by other stdlibs.  Filter them out if they're
-        # present in the manifest but aren't direct dependencies or dependencies
-        # of other JLLS.
+        # Some JLLs are also standard libraries that may be present in the manifest because
+        # they were pulled by other stdlibs (e.g. through dependence on `Pkg`), not beacuse
+        # they were actually required for this package. Filter them out if they're present
+        # in the manifest but aren't direct dependencies or dependencies of other JLLS.
         manifest = TOML.parsefile(ctx.env.manifest_file)
         dependencies_names = Set(getfield.(dependencies, :name))
-        jlls = collect_jlls(manifest, dependencies_names)
-        for jll in filter(s -> endswith(s, "_jll"), collect(values(Pkg.Types.stdlibs())))
-            if jll ∉ jlls
-                filter!(p -> p.name != jll, installed_jlls)
-            end
-        end
+        jll_direct_deps = collect_jlls(manifest, dependencies_names)
+        filter!(jll -> jll.name ∈ jll_direct_deps, installed_jlls)
 
         # Load their Artifacts.toml files
         for dep in installed_jlls
-            dep_path = Pkg.Operations.source_path(ctx, dep)
+            local dep_path
+            if dep.tree_hash === nothing
+                dep_path = Pkg.Types.stdlib_path(dep.name)
+            else
+                dep_path = Pkg.Operations.find_installed(dep.name, dep.uuid, dep.tree_hash)
+            end
             name = getname(dep)
 
             # Skip dependencies that didn't get installed?
@@ -480,26 +481,25 @@ function setup_dependencies(prefix::Prefix, dependencies::Vector{PkgSpec}, platf
             # Load the Artifacts.toml file
             artifacts_toml = joinpath(dep_path, "Artifacts.toml")
             if !isfile(artifacts_toml)
-                @warn("Dependency $(name) does not have an Artifacts.toml at $(artifacts_toml)!")
-                continue
-            end
-
-            # Get the path to the main artifact
-            artifact_hash = Pkg.Artifacts.artifact_hash(name[1:end-4], artifacts_toml; platform=platform)
-            if artifact_hash === nothing
-                @warn("Dependency $(name) does not have a mapping for artifact $(name[1:end-4]) for platform $(platform)")
-                continue
+                # Try `StdlibArtifacts.toml` instead
+                artifacts_toml = joinpath(dep_path, "StdlibArtifacts.toml")
+                if !isfile(artifacts_toml)
+                    @warn("Dependency $(name) does not have an (Stdlib)Artifacts.toml in $(dep_path)!")
+                    continue
+                end
             end
 
             # If the artifact is available for the given platform, make sure it
-            # is also installed.  It may not be the case for lazy artifacts
+            # is also installed.  It may not be the case for lazy artifacts or stdlibs.
             meta = artifact_meta(name[1:end-4], artifacts_toml; platform=platform)
-            if meta !== nothing
-                ensure_artifact_installed(name[1:end-4], meta, artifacts_toml; platform=platform)
+            if meta === nothing
+                @warn("Dependency $(name) does not have a mapping for artifact $(name[1:end-4]) for platform $(triplet(platform))")
+                continue
             end
+            ensure_artifact_installed(name[1:end-4], meta, artifacts_toml; platform=platform)
 
             # Copy the artifact from the global installation location into this build-specific artifacts collection
-            src_path = Pkg.Artifacts.artifact_path(artifact_hash)
+            src_path = Pkg.Artifacts.artifact_path(Base.SHA1(meta["git-tree-sha1"]))
             dest_path = joinpath(prefix, triplet(platform), "artifacts", basename(src_path))
             cp(src_path, dest_path)
 

src/Rootfs.jl

@@ -121,33 +121,38 @@ function all_compiler_shards()
     return ALL_SHARDS[]
 end
 
+function shard_source_artifact_hash(cs::CompilerShard)
+    artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml")
+    name = artifact_name(cs)
+    hash = artifact_hash(
+        name,
+        artifacts_toml;
+        platform=cs.host,
+    )
+    if hash === nothing
+        error("Compiler shard $(name) for platform $(triplet(something(cs.target, cs.host))) not found in $(artifacts_toml)")
+    end
+    return hash
+end
+
 """
     shard_path(cs::CompilerShard)
 
 Return the path to this shard on-disk; for unpacked shards, this is a directory.
 For squashfs shards, this is a file.  This will not cause a shard to be downloaded.
 """
-function shard_path(cs::CompilerShard)
-    if cs.shard_type == :squashfs
-        mutable_artifacts_toml = joinpath(dirname(@__DIR__), "MutableArtifacts.toml")
-        artifacts_dict = artifact_meta(
-            artifact_name(cs),
-            artifacts_toml;
-            platform=something(cs.target, cs.host),
-        )
-    end
+function shard_path(cs::CompilerShard; uid = getuid())
+    hash = shard_source_artifact_hash(cs)
 
-    artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml")
-    artifacts_dict = artifact_meta(
-        artifact_name(cs),
-        artifacts_toml;
-        platform=something(cs.target, cs.host),
-    )
-    if artifacts_dict == nothing
-        error("CompilerShard $(artifact_name(cs)) not registered in Artifacts.toml!")
+    # .squashfs files get modified per-UID
+    if cs.archive_type == :squashfs
+        # Store the UID-altered .squashfs in a scratchspace specific to this artifact,
+        # and keyed by the name/UUID.  This allows old scratchspaces to be cleaned up
+        # as artifacts are retired, and also for multiple users to share a scratchspace.
+        return joinpath(@get_scratch!(bytes2hex(hash.bytes)), "$(artifact_name(cs))_$(getuid()).squashfs")
     end
 
-    return artifact_path(artifacts_dict["git-tree-sha1"])
+    return artifact_path(hash)
 end
 
 """
@@ -481,7 +486,7 @@ consists of four shards, but that may not always be the case.
 """
 function choose_shards(p::AbstractPlatform;
             compilers::Vector{Symbol} = [:c],
-            rootfs_build::VersionNumber=v"2021.1.12",
+            rootfs_build::VersionNumber=v"2021.1.22",
             ps_build::VersionNumber=v"2021.01.28",
             GCC_builds::Vector{GCCBuild}=available_gcc_builds,
             LLVM_builds::Vector{LLVMBuild}=available_llvm_builds,
@@ -931,33 +936,3 @@ function unmount_shards(ur::Runner; verbose::Bool = false)
     catch
     end
 end
-
-
-
-"""
-    create_and_bind_mutable_artifact!(f::Function, art_name::String)
-
-Create (and bind) an artifact to `MutableArtifacts.toml` in one fell swoop.
-Used in things like .squashfs UID remapping and BB wizard state serialization.
-"""
-function create_and_bind_mutable_artifact!(f::Function, art_name::String)
-    mutable_artifacts_toml = joinpath(dirname(@__DIR__), "MutableArtifacts.toml")
-    art_hash = create_artifact(f)
-    bind_artifact!(mutable_artifacts_toml, art_name, art_hash; force=true)
-end
-
-"""
-    get_mutable_artifact_path(art_name::String)
-
-Convenience wrapper to get an artifact bound within `MutableArtifacts.toml`.
-Returns `nothing` if artifact not bound yet.
-"""
-function get_mutable_artifact_path(art_name::String)
-    mutable_artifacts_toml = joinpath(dirname(@__DIR__), "MutableArtifacts.toml")
-    hash = artifact_hash(art_name, mutable_artifacts_toml)
-    if hash === nothing
-        return nothing
-    end
-
-    return artifact_path(hash)
-end