Add HostBuildDependency (#43)

* Add HostBuildDependency

Main changes:

* we now have two "prefixes", one for the target and one for the host.  To
  implement this,
  * `setup_workspace` takes in input the platforms of the target and the host,
    and it creates the symlink
    `/workspace/destdir` -> `/workspace/TARGET_TRIPLET/destdir`
  * the user will call `setup_dependencies` twice, once for the host and once
    for the target
* a new helper function `destdir(prefix, platform)` gives consistently the path
  to the destdir for the given platform within the given prefix
* added relevant environment variables `host_*dir` in the runners, and add
  `host_bindir` to the `PATH`
* added `is_host_dependency`, `is_target_dependency`, `is_build_dependency`,
  `is_runtime_dependency` functions which are _very_ useful to filter
  dependencies in different situations: as we add more and more of these
  `AbstractDependency` types, there are multiple types matching each trait and
  it's hard to consistently list all of them in all places

* Apply suggestions from code review

Co-authored-by: Elliot Saba <staticfloat@gmail.com>

Co-authored-by: Elliot Saba <staticfloat@gmail.com>

Author: giordano

src/Dependencies.jl

@@ -1,6 +1,8 @@
 using UUIDs
 
-export Dependency, BuildDependency
+export Dependency, BuildDependency, HostBuildDependency,
+    is_host_dependency, is_target_dependency, is_build_dependency, is_runtime_dependency
+
 
 # Pkg.PackageSpec return different types in different Julia versions so...
 const PkgSpec = typeof(Pkg.PackageSpec(name="dummy"))
@@ -15,9 +17,40 @@ Concrete subtypes of `AbstractDependency` are
   and to load the generated JLL package.
 * [`BuildDependency`](@ref): a JLL package that is necessary only to build the
   package.  This will not be a dependency of the generated JLL package.
+* [`HostBuildDependency`](@ref): similar to `BuildDependency`, but it will
+  install the artifact for the host platform, instead of that for the target
+  platform.
 """
 abstract type AbstractDependency end
 
+"""
+    is_host_dependency(dep::AbstractDependency) -> Bool
+
+Return whether `dep` is a dependency of the host platform or not.
+"""
+is_host_dependency
+
+"""
+    is_target_dependency(dep::AbstractDependency) -> Bool
+
+Return whether `dep` is a dependency of the target platform or not.
+"""
+is_target_dependency(dep::AbstractDependency) = !is_host_dependency(dep)
+
+"""
+    is_build_dependency(dep::AbstractDependency) -> Bool
+
+Return whether `dep` is a build-time dependency or not.
+"""
+is_build_dependency
+
+"""
+    is_runtime_dependency(dep::AbstractDependency) -> Bool
+
+Return whether `dep` is a runtime dependency or not.
+"""
+is_runtime_dependency
+
 """
     Dependency(dep::Union{PackageSpec,String})
 
@@ -31,6 +64,9 @@ struct Dependency <: AbstractDependency
     Dependency(pkg::PkgSpec, build_version = nothing) = new(pkg, build_version)
 end
 Dependency(dep::AbstractString, build_version = nothing) = Dependency(PackageSpec(; name = dep), build_version)
+is_host_dependency(::Dependency) = false
+is_build_dependency(::Dependency) = true
+is_runtime_dependency(::Dependency) = true
 
 """
     BuildDependency(dep::Union{PackageSpec,String})
@@ -43,6 +79,27 @@ struct BuildDependency <: AbstractDependency
     pkg::PkgSpec
 end
 BuildDependency(dep::AbstractString) = BuildDependency(PackageSpec(; name = dep))
+is_host_dependency(::BuildDependency) = false
+is_build_dependency(::BuildDependency) = true
+is_runtime_dependency(::BuildDependency) = false
+
+"""
+    HostBuildDependency(dep::Union{PackageSpec,String})
+
+Define a binary dependency that is necessary only to build the package.
+Different from the [`BuildDependency`](@ref), the artifact for the host
+platform will be installed, instead of that for the target platform.
+
+The argument can be either a string with the name of the JLL package or a
+`Pkg.PackageSpec`.
+"""
+struct HostBuildDependency <: AbstractDependency
+    pkg::PkgSpec
+end
+HostBuildDependency(dep::AbstractString) = HostBuildDependency(PackageSpec(; name = dep))
+is_host_dependency(::HostBuildDependency) = true
+is_build_dependency(::HostBuildDependency) = true
+is_runtime_dependency(::HostBuildDependency) = false
 
 getpkg(d::AbstractDependency) = d.pkg
 function getpkg(d::Dependency)
@@ -124,7 +181,7 @@ __version(v::Pkg.Types.VersionSpec) = v.ranges[1].lower
 version(d::AbstractDependency) = __version(getpkg(d).version)
 version(d::Dependency) = __version(d.pkg.version)
 
-for (type, type_descr) in ((Dependency, "dependency"), (BuildDependency, "builddependency"))
+for (type, type_descr) in ((Dependency, "dependency"), (BuildDependency, "builddependency"), (HostBuildDependency, "hostdependency"))
     JSON.lower(d::type) = Dict("type" => type_descr,
                                "name" => d.pkg.name,
                                "uuid" => string_or_nothing(d.pkg.uuid),
@@ -137,16 +194,18 @@ end
 # dictionaries.  This function converts the dictionary back to the appropriate
 # AbstractDependency.
 function dependencify(d::Dict)
-    if d["type"] == "dependency"
+    if d["type"] in ("dependency", "builddependency", "hostdependency")
         uuid = isnothing(d["uuid"]) ? d["uuid"] : UUID(d["uuid"])
         version = VersionNumber(d["version-major"], d["version-minor"], d["version-patch"])
         version = version == v"0" ? nothing : version
-        return Dependency(PackageSpec(; name = d["name"], uuid = uuid, version = version))
-    elseif d["type"] == "builddependency"
-        uuid = isnothing(d["uuid"]) ? d["uuid"] : UUID(d["uuid"])
-        version = VersionNumber(d["version-major"], d["version-minor"], d["version-patch"])
-        version = version == v"0" ? nothing : version
-        return BuildDependency(PackageSpec(; name = d["name"], uuid = uuid, version = version))
+        T = if d["type"] == "dependency"
+            Dependency
+        elseif d["type"] == "builddependency"
+            BuildDependency
+        elseif d["type"] == "hostdependency"
+            HostBuildDependency
+        end
+        return T(PackageSpec(; name = d["name"], uuid = uuid, version = version))
     else
         error("Cannot convert to dependency")
     end

src/Prefix.jl

@@ -321,6 +321,9 @@ function setup(source::PatchSource, targetdir, verbose)
     open(f->write(f, source.patch), joinpath(patches_dir, source.name), "w")
 end
 
+destdir(prefix, platform::AbstractPlatform) =
+    joinpath(prefix, triplet(platform), "destdir")
+
 """
     setup_workspace(build_path::String, sources::Vector{SetupSource};
                     verbose::Bool = false)
@@ -332,7 +335,9 @@ the environment variables that will be defined within the sandbox environment.
 This method returns the `Prefix` to install things into, and the runner
 that can be used to launch commands within this workspace.
 """
-function setup_workspace(build_path::AbstractString, sources::Vector;
+function setup_workspace(build_path::AbstractString, sources::Vector,
+                         target_platform::AbstractPlatform,
+                         host_platform::AbstractPlatform=default_host_platform;
                          verbose::Bool = false)
     # Use a random nonce to make detection of paths in embedded binary easier
     nonce = randstring()
@@ -341,10 +346,14 @@ function setup_workspace(build_path::AbstractString, sources::Vector;
 
     # We now set up two directories, one as a source dir, one as a dest dir
     srcdir = joinpath(workspace, "srcdir")
-    destdir = joinpath(workspace, "destdir")
+    target_destdir = destdir(workspace, target_platform)
+    host_destdir = destdir(workspace, host_platform)
     metadir = joinpath(workspace, "metadir")
-    wrapperdir = joinpath(workspace, "compiler_wrappers")
-    mkdir.((srcdir, destdir, metadir))
+    mkpath.((srcdir, target_destdir, host_destdir, metadir))
+    # Create the symlink /workspace/destdir -> /workspace/TARGET_TRIPLET/destdir
+    # Necessary for compatibility with recipes that hardcode `/workspace/destdir` in them,
+    # as well as `.pc` files that contain absolute paths to `/workspace/destdir/...`
+    symlink("$(triplet(target_platform))/destdir", joinpath(workspace, "destdir"))
 
     # Setup all sources
     for source in sources
@@ -428,8 +437,8 @@ function setup_dependencies(prefix::Prefix, dependencies::Vector{PkgSpec}, platf
     # We're going to create a project and install all dependent packages within
     # it, then create symlinks from those installed products to our build prefix
 
-    mkpath(joinpath(prefix, "artifacts"))
-    deps_project = joinpath(prefix, ".project")
+    mkpath(joinpath(prefix, triplet(platform), "artifacts"))
+    deps_project = joinpath(prefix, triplet(platform), ".project")
     Pkg.activate(deps_project) do
         # Update registry first, in case the jll packages we're looking for have just been registered/updated
         ctx = Pkg.Types.Context(;julia_version = julia_version)
@@ -491,7 +500,7 @@ function setup_dependencies(prefix::Prefix, dependencies::Vector{PkgSpec}, platf
 
             # Copy the artifact from the global installation location into this build-specific artifacts collection
             src_path = Pkg.Artifacts.artifact_path(artifact_hash)
-            dest_path = joinpath(prefix, "artifacts", basename(src_path))
+            dest_path = joinpath(prefix, triplet(platform), "artifacts", basename(src_path))
             cp(src_path, dest_path)
 
             # Keep track of our dep paths for later symlinking
@@ -501,7 +510,7 @@ function setup_dependencies(prefix::Prefix, dependencies::Vector{PkgSpec}, platf
 
     # Symlink all the deps into the prefix
     for art_path in artifact_paths
-        symlink_tree(art_path, joinpath(prefix, "destdir"))
+        symlink_tree(art_path, destdir(prefix, platform))
     end
 
     # Return the artifact_paths so that we can clean them up later
@@ -510,7 +519,13 @@ end
 
 function cleanup_dependencies(prefix::Prefix, artifact_paths)
     for art_path in artifact_paths
-        unsymlink_tree(art_path, joinpath(prefix, "destdir"))
+        # Unsymlink all destdirs within the prefix
+        for dir in readdir(prefix.path; join=true)
+            destdir = joinpath(dir, "destdir")
+            if isdir(destdir)
+                unsymlink_tree(art_path, destdir)
+            end
+        end
     end
 end
 

src/Runner.jl

@@ -1,9 +1,16 @@
 import Base: strip
 abstract type Runner; end
 
+export default_host_platform
+
 # Host platform _must_ match the C++ string ABI of the binaries we get from the
 # repositories.  Note: when preferred_gcc_version=v"4" we can't really build for
 # that C++ string ABI :-(
+"""
+    default_host_platform
+
+The default host platform in the build environment.
+"""
 const default_host_platform = Platform("x86_64", "linux"; libc="musl", cxxstring_abi="cxx11")
 
 function nbits(p::AbstractPlatform)
@@ -741,11 +748,17 @@ function platform_envs(platform::AbstractPlatform, src_name::AbstractString;
 
     # Prefix, libdir, etc...
     prefix = "/workspace/destdir"
+    host_prefix = "/workspace/$(triplet(host_platform))/destdir"
     if Sys.iswindows(platform)
         libdir = "$(prefix)/bin"
     else
         libdir = "$(prefix)/lib"
     end
+    if Sys.iswindows(host_platform)
+        host_libdir = "$(host_prefix)/bin"
+    else
+        host_libdir = "$(host_prefix)/lib"
+    end
 
     if get(stdout, :color, false)
         PS1 = string(
@@ -795,6 +808,12 @@ function platform_envs(platform::AbstractPlatform, src_name::AbstractString;
         "libdir" => libdir,
         "includedir" => "$(prefix)/include",
 
+        # Set variables for the host prefix
+        "host_prefix" => host_prefix,
+        "host_bindir" => "$(host_prefix)/bin",
+        "host_libdir" => host_libdir,
+        "host_includedir" => "$(host_prefix)/include",
+
         # Fancyness!
         "USER" => get(ENV, "USER", "julia"),
         # Docker filters out `PS1` so we route around it
@@ -862,8 +881,10 @@ function platform_envs(platform::AbstractPlatform, src_name::AbstractString;
             "/opt/$(host_target)/bin",
             # Default alpine PATH
             mapping["PATH"],
+            # Host tools, installed as `HostBuildDependency`
+            mapping["host_bindir"],
             # Finally, dependency tools
-            "$(prefix)/bin",
+            mapping["bindir"],
         ), ":"),
 
         "LD_LIBRARY_PATH" => join((