Skip to content

Commit d7affb9

Browse files
giordanogiordano
authored
Upgrade Rust to v1.57.0 and JIT-generate Cargo config file (#184)
* [Runners] JIT-generate Cargo config file This will replace the file we currently hard-code in the Rust compiler shard. * [Rust] Update to v1.56.1 * Add RustToolchain v1.56.1 * Bad feeling number 2 * [BuildToolchains] Hack for aarch64-linux-musl no more needed * [Artifacts.toml] Remove RustToolchain and RustBase v1.43 * [Runners] Generate Go and Rust env vars only if necessary * [Rootfs] Rename `armv6l` -> `armv7l` only for the GCCBootstrap shard * Upgrade Rust to v1.57.0 * [Runners] Also create Cargo config file only if not bootstrapping * [Rootfs] With Rust on x86-64 Windows require GCC 5
1 parent 2b9abd2 commit d7affb9

File tree

8 files changed

+342
-187
lines changed

8 files changed

+342
-187
lines changed

‎Artifacts.toml

Lines changed: 206 additions & 140 deletions
Large diffs are not rendered by default.

‎Project.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name = "BinaryBuilderBase"
22
uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
33
authors = ["Elliot Saba <staticfloat@gmail.com>"]
4-
version = "1.0.5"
4+
version = "1.1.0"
55

66
[deps]
77
CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"

‎src/BuildToolchains.jl

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -214,8 +214,8 @@ function toolchain_file(bt::Meson, p::AbstractPlatform, envs::Dict{String,String
214214
"""
215215
end
216216

217-
function generate_toolchain_files!(platform::AbstractPlatform, envs::Dict{String,String};
218-
toolchains_path::AbstractString,
217+
function generate_toolchain_files!(platform::AbstractPlatform, envs::Dict{String,String},
218+
toolchains_path::AbstractString;
219219
host_platform::AbstractPlatform = default_host_platform,
220220
)
221221

@@ -253,3 +253,22 @@ function generate_toolchain_files!(platform::AbstractPlatform, envs::Dict{String
253253
end
254254
end
255255
end
256+
257+
function cargo_config_file!(dir::AbstractString)
258+
# Generate "${CARGO_HOME}/config.toml" file for Cargo where we give it the
259+
# linkers for all our targets
260+
open(joinpath(dir, "config.toml"), "w") do io
261+
write(io, """
262+
# Configuration file for `cargo`
263+
""")
264+
for platform in supported_platforms(; experimental=true)
265+
# Use `aatriplet` for the linker to match how the wrappers are
266+
# written in
267+
# https://github.com/JuliaPackaging/BinaryBuilderBase.jl/blob/30d056ef68f81dca9cb91ededcce6b68c6466b37/src/Runner.jl#L599.
268+
write(io, """
269+
[target.$(map_rust_target(platform))]
270+
linker = "$(aatriplet(platform))-gcc"
271+
""")
272+
end
273+
end
274+
end

‎src/DockerRunner.jl

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -84,18 +84,30 @@ function DockerRunner(workspace_root::String;
8484
# encrypted directory, as that can trigger kernel bugs
8585
check_encryption(workspace_root; verbose=verbose)
8686

87+
# Extract compilers argument
88+
compilers = collect(extract_kwargs(kwargs, (:compilers,)))
89+
8790
# Construct environment variables we'll use from here on out
88-
platform = get_concrete_platform(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:compilers))...)
89-
envs = merge(platform_envs(platform, src_name; verbose=verbose), extra_env)
91+
platform = get_concrete_platform(platform; compilers..., extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version))...)
92+
envs = merge(platform_envs(platform, src_name; verbose, compilers...), extra_env)
9093

9194
# JIT out some compiler wrappers, add it to our mounts
92-
generate_compiler_wrappers!(platform; bin_path=compiler_wrapper_path, extract_kwargs(kwargs, (:compilers,:allow_unsafe_flags,:lock_microarchitecture))...)
95+
generate_compiler_wrappers!(platform; bin_path=compiler_wrapper_path, compilers..., extract_kwargs(kwargs, (:allow_unsafe_flags,:lock_microarchitecture))...)
9396
push!(workspaces, compiler_wrapper_path => "/opt/bin")
9497

9598
if isempty(bootstrap_list)
9699
# Generate CMake and Meson files, only if we are not bootstrapping
97-
generate_toolchain_files!(platform, envs; toolchains_path=toolchains_path)
100+
generate_toolchain_files!(platform, envs, toolchains_path)
98101
push!(workspaces, toolchains_path => "/opt/toolchains")
102+
103+
# Generate directory where to write Cargo config files
104+
if isone(length(collect(compilers))) && :rust in collect(compilers)[1].second
105+
cargo_dir = mktempdir()
106+
cargo_config_file!(cargo_dir)
107+
# Add to the list of mappings a subdirectory of ${CARGO_HOME}, whose content
108+
# will be put in ${CARGO_HOME}.
109+
push!(workspaces, cargo_dir => envs["CARGO_HOME"] * "/" * randstring())
110+
end
99111
end
100112

101113
# the workspace_root is always a workspace, and we always mount it first
@@ -111,7 +123,7 @@ function DockerRunner(workspace_root::String;
111123

112124
if isnothing(shards)
113125
# Choose the shards we're going to mount
114-
shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
126+
shards = choose_shards(platform; compilers..., extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list))...)
115127
end
116128

117129
# Import docker image

‎src/Rootfs.jl

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,10 @@ function artifact_name(cs::CompilerShard)
6767
if cs.target != nothing
6868
target_str = "-$(triplet(cs.target))"
6969

70-
# armv6l uses the same shards as armv7l, so we just rename here.
71-
target_str = replace(target_str, "-armv6l-linux" => "-armv7l-linux")
70+
if cs.name == "GCCBootstrap"
71+
# armv6l uses the same GCC shards as armv7l, so we just rename here.
72+
target_str = replace(target_str, "-armv6l-linux" => "-armv7l-linux")
73+
end
7274
end
7375
ext = Dict(:squashfs => "squashfs", :unpacked => "unpacked")[cs.archive_type]
7476
return "$(cs.name)$(target_str).v$(cs.version).$(triplet(cs.host)).$(ext)"
@@ -395,14 +397,17 @@ const available_llvm_builds = [
395397
]
396398

397399
"""
398-
gcc_version(p::AbstractPlatform, , GCC_builds::Vector{GCCBuild};
400+
gcc_version(p::AbstractPlatform, GCC_builds::Vector{GCCBuild},
401+
compilers::Vector{Symbol}=[:c];
399402
llvm_version::Union{Nothing,VersionNumber}=nothing)
400403
401404
Returns the closest matching GCC version number for the given particular
402405
platform, from the given set of options. The compiler ABI and the
403406
microarchitecture of the platform will be taken into account. If no match is
404-
found, returns an empty list. If the keyword argument `llvm_version` is passed,
405-
it is used to filter the version of GCC for FreeBSD platforms.
407+
found, returns an empty list. `compilers` is the list of compilers used in the
408+
build, to choose the right version of GCC to couple with them if necessary. If
409+
the keyword argument `llvm_version` is passed, it is used to filter the version
410+
of GCC for FreeBSD platforms.
406411
407412
This method assumes that the compiler ABI of the platform represents a platform
408413
that binaries will be run on, and thus versions are always rounded down; e.g. if
@@ -411,7 +416,8 @@ the only GCC versions available to be picked from are `4.8.5` and `5.2.0`, it
411416
will return `4.8.5`, as binaries compiled with that version will run on this
412417
platform, whereas binaries compiled with `5.2.0` may not.
413418
"""
414-
function gcc_version(p::AbstractPlatform, GCC_builds::Vector{GCCBuild};
419+
function gcc_version(p::AbstractPlatform,GCC_builds::Vector{GCCBuild},
420+
compilers::Vector{Symbol}=[:c];
415421
llvm_version::Union{Nothing,VersionNumber}=nothing)
416422
# First, filter by libgfortran version.
417423
if libgfortran_version(p) !== nothing
@@ -445,6 +451,12 @@ function gcc_version(p::AbstractPlatform, GCC_builds::Vector{GCCBuild};
445451
GCC_builds = filter(b -> getversion(b) ≥ v"6", GCC_builds)
446452
end
447453

454+
# Rust on Windows requires binutils 2.25 (it invokes `ld` with `--high-entropy-va`),
455+
# which we bundle with GCC 5.
456+
if :rust in compilers && Sys.iswindows(p)
457+
GCC_builds = filter(b -> getversion(b) ≥ v"5", GCC_builds)
458+
end
459+
448460
# Filter the possible GCC versions depending on the microarchitecture
449461
if march(p) in ("avx", "avx2", "neonvfpv4")
450462
# "sandybridge", "haswell", "cortex-a53" introduced in GCC v4.9.0:
@@ -475,6 +487,7 @@ function llvm_version(p::AbstractPlatform, LLVM_builds::Vector{LLVMBuild})
475487
end
476488

477489
function select_compiler_versions(p::AbstractPlatform,
490+
compilers::Vector{Symbol},
478491
GCC_builds::Vector{GCCBuild} = available_gcc_builds,
479492
LLVM_builds::Vector{LLVMBuild} = available_llvm_builds,
480493
preferred_gcc_version::VersionNumber = getversion(GCC_builds[1]),
@@ -488,7 +501,7 @@ function select_compiler_versions(p::AbstractPlatform,
488501
end
489502
llvmv = select_closest_version(preferred_llvm_version, filtered_llvm_builds)
490503

491-
filtered_gcc_builds = gcc_version(p, GCC_builds; llvm_version=llvmv)
504+
filtered_gcc_builds = gcc_version(p, GCC_builds, compilers; llvm_version=llvmv)
492505
if isempty(filtered_gcc_builds)
493506
error("Impossible compiler constraints $(p) upon $(GCC_builds)!")
494507
end
@@ -512,7 +525,7 @@ function choose_shards(p::AbstractPlatform;
512525
ps_build::VersionNumber=v"2021.08.10",
513526
GCC_builds::Vector{GCCBuild}=available_gcc_builds,
514527
LLVM_builds::Vector{LLVMBuild}=available_llvm_builds,
515-
Rust_build::VersionNumber=v"1.43.0",
528+
Rust_build::VersionNumber=v"1.57.0",
516529
Go_build::VersionNumber=v"1.16.3",
517530
archive_type::Symbol = (use_squashfs ? :squashfs : :unpacked),
518531
bootstrap_list::Vector{Symbol} = bootstrap_list,
@@ -567,6 +580,7 @@ function choose_shards(p::AbstractPlatform;
567580
if isempty(bootstrap_list)
568581
# Select GCC and LLVM versions given the compiler ABI and target requirements given in `p`
569582
GCC_build, LLVM_build = select_compiler_versions(p,
583+
compilers,
570584
this_platform_GCC_builds,
571585
LLVM_builds,
572586
preferred_gcc_version,

‎src/Runner.jl

Lines changed: 52 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -372,7 +372,7 @@ function generate_compiler_wrappers!(platform::AbstractPlatform; bin_path::Abstr
372372
function macos_gcc_flags!(p::AbstractPlatform, flags::Vector{String} = String[])
373373
# On macOS, if we're on an old GCC, the default -syslibroot that gets
374374
# passed to the linker isn't calculated correctly, so we have to manually set it.
375-
gcc_version, llvm_version = select_compiler_versions(p)
375+
gcc_version, llvm_version = select_compiler_versions(p, compilers)
376376
if gcc_version.major in (4, 5)
377377
push!(flags, "-Wl,-syslibroot,/opt/$(aatriplet(p))/$(aatriplet(p))/sys-root")
378378
end
@@ -423,7 +423,7 @@ function generate_compiler_wrappers!(platform::AbstractPlatform; bin_path::Abstr
423423
function gcc_link_flags!(p::AbstractPlatform, flags::Vector{String} = String[])
424424
# Yes, it does seem that the inclusion of `/lib64` on `powerpc64le` was fixed
425425
# in GCC 6, broken again in GCC 7, and then fixed again for GCC 8 and 9
426-
gcc_version, llvm_version = select_compiler_versions(p)
426+
gcc_version, llvm_version = select_compiler_versions(p, compilers)
427427
if arch(p) == "powerpc64le" && Sys.islinux(p) && gcc_version.major in (4, 5, 7)
428428
append!(flags, String[
429429
"-L/opt/$(aatriplet(p))/$(aatriplet(p))/sys-root/lib64",
@@ -817,8 +817,10 @@ function generate_compiler_wrappers!(platform::AbstractPlatform; bin_path::Abstr
817817
end
818818
end
819819

820-
# Translation mappers for our target names to cargo-compatible ones
821-
map_rust_arch(p::AbstractPlatform) = replace(arch(p), "armv7l" => "armv7")
820+
# Translation mappers for our target names to cargo-compatible ones. See
821+
# https://doc.rust-lang.org/rustc/platform-support.html
822+
map_rust_arch(p::AbstractPlatform) =
823+
replace(replace(arch(p), "armv7l" => "armv7"), "armv6l" => "arm")
822824
function map_rust_target(p::AbstractPlatform)
823825
if Sys.isapple(p)
824826
return "$(map_rust_arch(p))-apple-darwin"
@@ -834,17 +836,32 @@ function map_rust_target(p::AbstractPlatform)
834836
end
835837

836838
"""
837-
platform_envs(platform::AbstractPlatform)
838-
839-
Given a `platform`, generate a `Dict` mapping representing all the environment
840-
variables to be set within the build environment to force compiles toward the
841-
defined target architecture. Examples of things set are `PATH`, `CC`,
842-
`RANLIB`, as well as nonstandard things like `target`.
839+
platform_envs(platform::AbstractPlatform, src_name::AbstractString;
840+
host_platform = default_host_platform,
841+
bootstrap::Bool=!isempty(bootstrap_list),
842+
compilers::Vector{Symbol}=[:c],
843+
verbose::Bool = false,
844+
)
845+
846+
Given a `platform` and a `src_name`, generate a `Dict` mapping representing all
847+
the environment variables to be set within the build environment to force
848+
compiles toward the defined target architecture. Examples of things set are
849+
`PATH`, `CC`, `RANLIB`, as well as nonstandard things like `target`.
850+
851+
Accepted keyword arguments are:
852+
853+
* `host_platform`: the platform of the host system,
854+
* `bootstraop`: if `true`, only basic environment variables will be generated,
855+
* `compilers`: list of compilers, some environment variables will be generated
856+
only if the relevant compilers are used (e.g., for Go and Rust)
857+
* `verbose`: holds the value of the `V` and `VERBOSE` environment variables.
843858
"""
844859
function platform_envs(platform::AbstractPlatform, src_name::AbstractString;
845860
host_platform = default_host_platform,
846861
bootstrap::Bool=!isempty(bootstrap_list),
847-
verbose::Bool = false)
862+
compilers::Vector{Symbol}=[:c],
863+
verbose::Bool = false,
864+
)
848865
global use_ccache
849866

850867
# Convert platform to a triplet, but strip out the ABI parts
@@ -949,6 +966,30 @@ function platform_envs(platform::AbstractPlatform, src_name::AbstractString;
949966
end
950967
end
951968

969+
970+
# Go stuff
971+
if :go in compilers
972+
merge!(mapping, Dict(
973+
"GO" => "go",
974+
"GOCACHE" => "/workspace/.gocache",
975+
"GOPATH" => "/workspace/.gopath",
976+
"GOARM" => GOARM(platform),
977+
))
978+
end
979+
980+
# Rust stuff
981+
if :rust in compilers
982+
merge!(mapping, Dict(
983+
"RUSTC" => "rustc",
984+
"CARGO" => "cargo",
985+
"CARGO_BUILD_TARGET" => map_rust_target(platform),
986+
"CARGO_HOME" => "/opt/$(host_target)",
987+
"RUSTUP_HOME" => "/opt/$(host_target)",
988+
# TODO: we'll need a way to parameterize this toolchain number
989+
"RUSTUP_TOOLCHAIN" => "1.57.0-$(map_rust_target(host_platform))",
990+
))
991+
end
992+
952993
merge!(mapping, Dict(
953994
"PATH" => join((
954995
# First things first, our compiler wrappers trump all
@@ -972,21 +1013,6 @@ function platform_envs(platform::AbstractPlatform, src_name::AbstractString;
9721013
"CC" => "cc",
9731014
"CXX" => "c++",
9741015
"FC" => "gfortran",
975-
"GO" => "go",
976-
"RUSTC" => "rustc",
977-
"CARGO" => "cargo",
978-
979-
# Go stuff
980-
"GOCACHE" => "/workspace/.gocache",
981-
"GOPATH" => "/workspace/.gopath",
982-
"GOARM" => GOARM(platform),
983-
984-
# Rust stuff
985-
"CARGO_BUILD_TARGET" => map_rust_target(platform),
986-
"CARGO_HOME" => "/opt/$(host_target)",
987-
"RUSTUP_HOME" => "/opt/$(host_target)",
988-
# TODO: we'll need a way to parameterize this toolchain number
989-
"RUSTUP_TOOLCHAIN" => "1.43.0-$(map_rust_target(host_platform))",
9901016

9911017
# We conditionally add on some compiler flags; we'll cull empty ones at the end
9921018
"USE_CCACHE" => "$(use_ccache)",

‎src/UserNSRunner.jl

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,18 +39,30 @@ function UserNSRunner(workspace_root::String;
3939
# encrypted directory, as that triggers kernel bugs
4040
check_encryption(workspace_root; verbose=verbose)
4141

42+
# Extract compilers argument
43+
compilers = extract_kwargs(kwargs, (:compilers,))
44+
4245
# Construct environment variables we'll use from here on out
43-
platform = get_concrete_platform(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:compilers))...)
44-
envs = merge(platform_envs(platform, src_name; verbose=verbose), extra_env)
46+
platform = get_concrete_platform(platform; compilers..., extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version))...)
47+
envs = merge(platform_envs(platform, src_name; verbose, compilers...), extra_env)
4548

4649
# JIT out some compiler wrappers, add it to our mounts
47-
generate_compiler_wrappers!(platform; bin_path=compiler_wrapper_path, extract_kwargs(kwargs, (:compilers,:allow_unsafe_flags,:lock_microarchitecture))...)
50+
generate_compiler_wrappers!(platform; bin_path=compiler_wrapper_path, compilers..., extract_kwargs(kwargs, (:allow_unsafe_flags,:lock_microarchitecture))...)
4851
push!(workspaces, compiler_wrapper_path => "/opt/bin")
4952

5053
if isempty(bootstrap_list)
5154
# Generate CMake and Meson files, only if we are not bootstrapping
52-
generate_toolchain_files!(platform, envs; toolchains_path=toolchains_path)
55+
generate_toolchain_files!(platform, envs, toolchains_path)
5356
push!(workspaces, toolchains_path => "/opt/toolchains")
57+
58+
# Generate directory where to write Cargo config files
59+
if isone(length(collect(compilers))) && :rust in collect(compilers)[1].second
60+
cargo_dir = mktempdir()
61+
cargo_config_file!(cargo_dir)
62+
# Add to the list of mappings a subdirectory of ${CARGO_HOME}, whose content
63+
# will be put in ${CARGO_HOME}.
64+
push!(mappings, cargo_dir => envs["CARGO_HOME"] * "/" * randstring())
65+
end
5466
end
5567

5668
# the workspace_root is always a workspace, and we always mount it first
@@ -81,7 +93,7 @@ function UserNSRunner(workspace_root::String;
8193

8294
if isnothing(shards)
8395
# Choose the shards we're going to mount
84-
shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
96+
shards = choose_shards(platform; compilers..., extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list))...)
8597
end
8698

8799
# Construct sandbox command to look at the location it'll be mounted under

‎test/rootfs.jl

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,12 @@ end
149149

150150
p = Platform("armv7l", "linux"; march="neonvfpv4")
151151
@test gcc_version(p, available_gcc_builds) == [v"5.2.0", v"6.1.0", v"7.1.0", v"8.1.0", v"9.1.0", v"10.2.0", v"11.1.0", v"11.0.0-iains"]
152+
153+
# When Rust is used on x86_64 Windows, we have to use at least binutils
154+
# 2.25, which we bundle with at least GCC 5.
155+
p = Platform("x86_64", "windows"; libgfortran_version=v"3")
156+
@test gcc_version(p, available_gcc_builds, [:c, :go]) == [v"4.8.5", v"5.2.0", v"6.1.0"]
157+
@test gcc_version(p, available_gcc_builds, [:c, :rust]) == [v"5.2.0", v"6.1.0"]
152158
end
153159

154160
@testset "Compiler wrappers" begin

0 commit comments

Comments
 (0)