Upgrade Rust to v1.57.0 and JIT-generate Cargo config file (#184)

* [Runners] JIT-generate Cargo config file This will replace the file we currently hard-code in the Rust compiler shard. * [Rust] Update to v1.56.1 * Add RustToolchain v1.56.1 * Bad feeling number 2 * [BuildToolchains] Hack for aarch64-linux-musl no more needed * [Artifacts.toml] Remove RustToolchain and RustBase v1.43 * [Runners] Generate Go and Rust env vars only if necessary * [Rootfs] Rename `armv6l` -> `armv7l` only for the GCCBootstrap shard * Upgrade Rust to v1.57.0 * [Runners] Also create Cargo config file only if not bootstrapping * [Rootfs] With Rust on x86-64 Windows require GCC 5
JuliaPackaging · Dec 3, 2021 · d7affb9 · d7affb9 · giordano · Dec 3, 2021
1 parent 2b9abd2
commit d7affb9
Show file tree

Hide file tree

Showing 8 changed files with 342 additions and 187 deletions.
diff --git a/Artifacts.toml b/Artifacts.toml
diff --git a/Project.toml b/Project.toml
@@ -1,7 +1,7 @@
 name = "BinaryBuilderBase"
 uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e"
 authors = ["Elliot Saba <staticfloat@gmail.com>"]
-version = "1.0.5"
+version = "1.1.0"
 
 [deps]
 CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193"

diff --git a/src/BuildToolchains.jl b/src/BuildToolchains.jl
@@ -214,8 +214,8 @@ function toolchain_file(bt::Meson, p::AbstractPlatform, envs::Dict{String,String
     """
 end
 
-function generate_toolchain_files!(platform::AbstractPlatform, envs::Dict{String,String};
-                                   toolchains_path::AbstractString,
+function generate_toolchain_files!(platform::AbstractPlatform, envs::Dict{String,String},
+                                   toolchains_path::AbstractString;
                                    host_platform::AbstractPlatform = default_host_platform,
                                    )
 
@@ -253,3 +253,22 @@ function generate_toolchain_files!(platform::AbstractPlatform, envs::Dict{String
         end
     end
 end
+
+function cargo_config_file!(dir::AbstractString)
+    # Generate "${CARGO_HOME}/config.toml" file for Cargo where we give it the
+    # linkers for all our targets
+    open(joinpath(dir, "config.toml"), "w") do io
+        write(io, """
+        # Configuration file for `cargo`
+        """)
+        for platform in supported_platforms(; experimental=true)
+            # Use `aatriplet` for the linker to match how the wrappers are
+            # written in
+            # https://github.com/JuliaPackaging/BinaryBuilderBase.jl/blob/30d056ef68f81dca9cb91ededcce6b68c6466b37/src/Runner.jl#L599.
+            write(io, """
+            [target.$(map_rust_target(platform))]
+            linker = "$(aatriplet(platform))-gcc"
+            """)
+        end
+    end
+end
diff --git a/src/DockerRunner.jl b/src/DockerRunner.jl
@@ -84,18 +84,30 @@ function DockerRunner(workspace_root::String;
     # encrypted directory, as that can trigger kernel bugs
     check_encryption(workspace_root; verbose=verbose)
 
+    # Extract compilers argument
+    compilers = collect(extract_kwargs(kwargs, (:compilers,)))
+
     # Construct environment variables we'll use from here on out
-    platform = get_concrete_platform(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:compilers))...)
-    envs = merge(platform_envs(platform, src_name; verbose=verbose), extra_env)
+    platform = get_concrete_platform(platform; compilers..., extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version))...)
+    envs = merge(platform_envs(platform, src_name; verbose, compilers...), extra_env)
 
     # JIT out some compiler wrappers, add it to our mounts
-    generate_compiler_wrappers!(platform; bin_path=compiler_wrapper_path, extract_kwargs(kwargs, (:compilers,:allow_unsafe_flags,:lock_microarchitecture))...)
+    generate_compiler_wrappers!(platform; bin_path=compiler_wrapper_path, compilers..., extract_kwargs(kwargs, (:allow_unsafe_flags,:lock_microarchitecture))...)
     push!(workspaces, compiler_wrapper_path => "/opt/bin")
 
     if isempty(bootstrap_list)
         # Generate CMake and Meson files, only if we are not bootstrapping
-        generate_toolchain_files!(platform, envs; toolchains_path=toolchains_path)
+        generate_toolchain_files!(platform, envs, toolchains_path)
         push!(workspaces, toolchains_path => "/opt/toolchains")
+
+        # Generate directory where to write Cargo config files
+        if isone(length(collect(compilers))) && :rust in collect(compilers)[1].second
+            cargo_dir = mktempdir()
+            cargo_config_file!(cargo_dir)
+            # Add to the list of mappings a subdirectory of ${CARGO_HOME}, whose content
+            # will be put in ${CARGO_HOME}.
+            push!(workspaces, cargo_dir => envs["CARGO_HOME"] * "/" * randstring())
+        end
     end
 
     # the workspace_root is always a workspace, and we always mount it first
@@ -111,7 +123,7 @@ function DockerRunner(workspace_root::String;
 
     if isnothing(shards)
         # Choose the shards we're going to mount
-        shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
+        shards = choose_shards(platform; compilers..., extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list))...)
     end
 
     # Import docker image

diff --git a/src/Rootfs.jl b/src/Rootfs.jl
@@ -67,8 +67,10 @@ function artifact_name(cs::CompilerShard)
     if cs.target != nothing
         target_str = "-$(triplet(cs.target))"
 
-        # armv6l uses the same shards as armv7l, so we just rename here.
-        target_str = replace(target_str, "-armv6l-linux" => "-armv7l-linux")
+        if cs.name == "GCCBootstrap"
+            # armv6l uses the same GCC shards as armv7l, so we just rename here.
+            target_str = replace(target_str, "-armv6l-linux" => "-armv7l-linux")
+        end
     end
     ext = Dict(:squashfs => "squashfs", :unpacked => "unpacked")[cs.archive_type]
     return "$(cs.name)$(target_str).v$(cs.version).$(triplet(cs.host)).$(ext)"
@@ -395,14 +397,17 @@ const available_llvm_builds = [
 ]
 
 """
-    gcc_version(p::AbstractPlatform, , GCC_builds::Vector{GCCBuild};
+    gcc_version(p::AbstractPlatform, GCC_builds::Vector{GCCBuild},
+                compilers::Vector{Symbol}=[:c];
                 llvm_version::Union{Nothing,VersionNumber}=nothing)
 
 Returns the closest matching GCC version number for the given particular
 platform, from the given set of options.  The compiler ABI and the
 microarchitecture of the platform will be taken into account.  If no match is
-found, returns an empty list.  If the keyword argument `llvm_version` is passed,
-it is used to filter the version of GCC for FreeBSD platforms.
+found, returns an empty list.  `compilers` is the list of compilers used in the
+build, to choose the right version of GCC to couple with them if necessary.  If
+the keyword argument `llvm_version` is passed, it is used to filter the version
+of GCC for FreeBSD platforms.
 
 This method assumes that the compiler ABI of the platform represents a platform
 that binaries will be run on, and thus versions are always rounded down; e.g. if
@@ -411,7 +416,8 @@ the only GCC versions available to be picked from are `4.8.5` and `5.2.0`, it
 will return `4.8.5`, as binaries compiled with that version will run on this
 platform, whereas binaries compiled with `5.2.0` may not.
 """
-function gcc_version(p::AbstractPlatform, GCC_builds::Vector{GCCBuild};
+function gcc_version(p::AbstractPlatform,GCC_builds::Vector{GCCBuild},
+                     compilers::Vector{Symbol}=[:c];
                      llvm_version::Union{Nothing,VersionNumber}=nothing)
     # First, filter by libgfortran version.
     if libgfortran_version(p) !== nothing
@@ -445,6 +451,12 @@ function gcc_version(p::AbstractPlatform, GCC_builds::Vector{GCCBuild};
         GCC_builds = filter(b -> getversion(b) ≥ v"6", GCC_builds)
     end
 
+    # Rust on Windows requires binutils 2.25 (it invokes `ld` with `--high-entropy-va`),
+    # which we bundle with GCC 5.
+    if :rust in compilers && Sys.iswindows(p)
+        GCC_builds = filter(b -> getversion(b) ≥ v"5", GCC_builds)
+    end
+
     # Filter the possible GCC versions depending on the microarchitecture
     if march(p) in ("avx", "avx2", "neonvfpv4")
         # "sandybridge", "haswell", "cortex-a53" introduced in GCC v4.9.0:
@@ -475,6 +487,7 @@ function llvm_version(p::AbstractPlatform, LLVM_builds::Vector{LLVMBuild})
 end
 
 function select_compiler_versions(p::AbstractPlatform,
+            compilers::Vector{Symbol},
             GCC_builds::Vector{GCCBuild} = available_gcc_builds,
             LLVM_builds::Vector{LLVMBuild} = available_llvm_builds,
             preferred_gcc_version::VersionNumber = getversion(GCC_builds[1]),
@@ -488,7 +501,7 @@ function select_compiler_versions(p::AbstractPlatform,
     end
     llvmv = select_closest_version(preferred_llvm_version, filtered_llvm_builds)
 
-    filtered_gcc_builds = gcc_version(p, GCC_builds; llvm_version=llvmv)
+    filtered_gcc_builds = gcc_version(p, GCC_builds, compilers; llvm_version=llvmv)
     if isempty(filtered_gcc_builds)
         error("Impossible compiler constraints $(p) upon $(GCC_builds)!")
     end
@@ -512,7 +525,7 @@ function choose_shards(p::AbstractPlatform;
             ps_build::VersionNumber=v"2021.08.10",
             GCC_builds::Vector{GCCBuild}=available_gcc_builds,
             LLVM_builds::Vector{LLVMBuild}=available_llvm_builds,
-            Rust_build::VersionNumber=v"1.43.0",
+            Rust_build::VersionNumber=v"1.57.0",
             Go_build::VersionNumber=v"1.16.3",
             archive_type::Symbol = (use_squashfs ? :squashfs : :unpacked),
             bootstrap_list::Vector{Symbol} = bootstrap_list,
@@ -567,6 +580,7 @@ function choose_shards(p::AbstractPlatform;
     if isempty(bootstrap_list)
         # Select GCC and LLVM versions given the compiler ABI and target requirements given in `p`
         GCC_build, LLVM_build = select_compiler_versions(p,
+            compilers,
             this_platform_GCC_builds,
             LLVM_builds,
             preferred_gcc_version,

diff --git a/src/Runner.jl b/src/Runner.jl
@@ -372,7 +372,7 @@ function generate_compiler_wrappers!(platform::AbstractPlatform; bin_path::Abstr
     function macos_gcc_flags!(p::AbstractPlatform, flags::Vector{String} = String[])
         # On macOS, if we're on an old GCC, the default -syslibroot that gets
         # passed to the linker isn't calculated correctly, so we have to manually set it.
-        gcc_version, llvm_version = select_compiler_versions(p)
+        gcc_version, llvm_version = select_compiler_versions(p, compilers)
         if gcc_version.major in (4, 5)
             push!(flags, "-Wl,-syslibroot,/opt/$(aatriplet(p))/$(aatriplet(p))/sys-root")
         end
@@ -423,7 +423,7 @@ function generate_compiler_wrappers!(platform::AbstractPlatform; bin_path::Abstr
     function gcc_link_flags!(p::AbstractPlatform, flags::Vector{String} = String[])
         # Yes, it does seem that the inclusion of `/lib64` on `powerpc64le` was fixed
         # in GCC 6, broken again in GCC 7, and then fixed again for GCC 8 and 9
-        gcc_version, llvm_version = select_compiler_versions(p)
+        gcc_version, llvm_version = select_compiler_versions(p, compilers)
         if arch(p) == "powerpc64le" && Sys.islinux(p) && gcc_version.major in (4, 5, 7)
             append!(flags, String[
                 "-L/opt/$(aatriplet(p))/$(aatriplet(p))/sys-root/lib64",
@@ -817,8 +817,10 @@ function generate_compiler_wrappers!(platform::AbstractPlatform; bin_path::Abstr
     end
 end
 
-# Translation mappers for our target names to cargo-compatible ones
-map_rust_arch(p::AbstractPlatform) = replace(arch(p), "armv7l" => "armv7")
+# Translation mappers for our target names to cargo-compatible ones.  See
+# https://doc.rust-lang.org/rustc/platform-support.html
+map_rust_arch(p::AbstractPlatform) =
+    replace(replace(arch(p), "armv7l" => "armv7"), "armv6l" => "arm")
 function map_rust_target(p::AbstractPlatform)
     if Sys.isapple(p)
         return "$(map_rust_arch(p))-apple-darwin"
@@ -834,17 +836,32 @@ function map_rust_target(p::AbstractPlatform)
 end
 
 """
-    platform_envs(platform::AbstractPlatform)
-
-Given a `platform`, generate a `Dict` mapping representing all the environment
-variables to be set within the build environment to force compiles toward the
-defined target architecture.  Examples of things set are `PATH`, `CC`,
-`RANLIB`, as well as nonstandard things like `target`.
+    platform_envs(platform::AbstractPlatform, src_name::AbstractString;
+                  host_platform = default_host_platform,
+                  bootstrap::Bool=!isempty(bootstrap_list),
+                  compilers::Vector{Symbol}=[:c],
+                  verbose::Bool = false,
+                  )
+
+Given a `platform` and a `src_name`, generate a `Dict` mapping representing all
+the environment variables to be set within the build environment to force
+compiles toward the defined target architecture.  Examples of things set are
+`PATH`, `CC`, `RANLIB`, as well as nonstandard things like `target`.
+
+Accepted keyword arguments are:
+
+* `host_platform`: the platform of the host system,
+* `bootstraop`: if `true`, only basic environment variables will be generated,
+* `compilers`: list of compilers, some environment variables will be generated
+  only if the relevant compilers are used (e.g., for Go and Rust)
+* `verbose`: holds the value of the `V` and `VERBOSE` environment variables.
 """
 function platform_envs(platform::AbstractPlatform, src_name::AbstractString;
                        host_platform = default_host_platform,
                        bootstrap::Bool=!isempty(bootstrap_list),
-                       verbose::Bool = false)
+                       compilers::Vector{Symbol}=[:c],
+                       verbose::Bool = false,
+                       )
     global use_ccache
 
     # Convert platform to a triplet, but strip out the ABI parts
@@ -949,6 +966,30 @@ function platform_envs(platform::AbstractPlatform, src_name::AbstractString;
         end
     end
 
+
+    # Go stuff
+    if :go in compilers
+        merge!(mapping, Dict(
+            "GO" => "go",
+            "GOCACHE" => "/workspace/.gocache",
+            "GOPATH" => "/workspace/.gopath",
+            "GOARM" => GOARM(platform),
+        ))
+    end
+
+    # Rust stuff
+    if :rust in compilers
+        merge!(mapping, Dict(
+            "RUSTC" => "rustc",
+            "CARGO" => "cargo",
+            "CARGO_BUILD_TARGET" => map_rust_target(platform),
+            "CARGO_HOME" => "/opt/$(host_target)",
+            "RUSTUP_HOME" => "/opt/$(host_target)",
+            # TODO: we'll need a way to parameterize this toolchain number
+            "RUSTUP_TOOLCHAIN" => "1.57.0-$(map_rust_target(host_platform))",
+        ))
+    end
+
     merge!(mapping, Dict(
         "PATH" => join((
             # First things first, our compiler wrappers trump all
@@ -972,21 +1013,6 @@ function platform_envs(platform::AbstractPlatform, src_name::AbstractString;
         "CC" => "cc",
         "CXX" => "c++",
         "FC" => "gfortran",
-        "GO" => "go",
-        "RUSTC" => "rustc",
-        "CARGO" => "cargo",
-
-        # Go stuff
-        "GOCACHE" => "/workspace/.gocache",
-        "GOPATH" => "/workspace/.gopath",
-        "GOARM" => GOARM(platform),
-
-        # Rust stuff
-        "CARGO_BUILD_TARGET" => map_rust_target(platform),
-        "CARGO_HOME" => "/opt/$(host_target)",
-        "RUSTUP_HOME" => "/opt/$(host_target)",
-        # TODO: we'll need a way to parameterize this toolchain number
-        "RUSTUP_TOOLCHAIN" => "1.43.0-$(map_rust_target(host_platform))",
 
         # We conditionally add on some compiler flags; we'll cull empty ones at the end
         "USE_CCACHE" => "$(use_ccache)",

diff --git a/src/UserNSRunner.jl b/src/UserNSRunner.jl
@@ -39,18 +39,30 @@ function UserNSRunner(workspace_root::String;
     # encrypted directory, as that triggers kernel bugs
     check_encryption(workspace_root; verbose=verbose)
 
+    # Extract compilers argument
+    compilers = extract_kwargs(kwargs, (:compilers,))
+
     # Construct environment variables we'll use from here on out
-    platform = get_concrete_platform(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:compilers))...)
-    envs = merge(platform_envs(platform, src_name; verbose=verbose), extra_env)
+    platform = get_concrete_platform(platform; compilers..., extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version))...)
+    envs = merge(platform_envs(platform, src_name; verbose, compilers...), extra_env)
 
     # JIT out some compiler wrappers, add it to our mounts
-    generate_compiler_wrappers!(platform; bin_path=compiler_wrapper_path, extract_kwargs(kwargs, (:compilers,:allow_unsafe_flags,:lock_microarchitecture))...)
+    generate_compiler_wrappers!(platform; bin_path=compiler_wrapper_path, compilers..., extract_kwargs(kwargs, (:allow_unsafe_flags,:lock_microarchitecture))...)
     push!(workspaces, compiler_wrapper_path => "/opt/bin")
 
     if isempty(bootstrap_list)
         # Generate CMake and Meson files, only if we are not bootstrapping
-        generate_toolchain_files!(platform, envs; toolchains_path=toolchains_path)
+        generate_toolchain_files!(platform, envs, toolchains_path)
         push!(workspaces, toolchains_path => "/opt/toolchains")
+
+        # Generate directory where to write Cargo config files
+        if isone(length(collect(compilers))) && :rust in collect(compilers)[1].second
+            cargo_dir = mktempdir()
+            cargo_config_file!(cargo_dir)
+            # Add to the list of mappings a subdirectory of ${CARGO_HOME}, whose content
+            # will be put in ${CARGO_HOME}.
+            push!(mappings, cargo_dir => envs["CARGO_HOME"] * "/" * randstring())
+        end
     end
 
     # the workspace_root is always a workspace, and we always mount it first
@@ -81,7 +93,7 @@ function UserNSRunner(workspace_root::String;
 
     if isnothing(shards)
         # Choose the shards we're going to mount
-        shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list,:compilers))...)
+        shards = choose_shards(platform; compilers..., extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:bootstrap_list))...)
     end
 
     # Construct sandbox command to look at the location it'll be mounted under

diff --git a/test/rootfs.jl b/test/rootfs.jl
@@ -149,6 +149,12 @@ end
 
         p = Platform("armv7l", "linux"; march="neonvfpv4")
         @test gcc_version(p, available_gcc_builds) == [v"5.2.0", v"6.1.0", v"7.1.0", v"8.1.0", v"9.1.0", v"10.2.0", v"11.1.0", v"11.0.0-iains"]
+
+        # When Rust is used on x86_64 Windows, we have to use at least binutils
+        # 2.25, which we bundle with at least GCC 5.
+        p = Platform("x86_64", "windows"; libgfortran_version=v"3")
+        @test gcc_version(p, available_gcc_builds, [:c, :go]) == [v"4.8.5", v"5.2.0", v"6.1.0"]
+        @test gcc_version(p, available_gcc_builds, [:c, :rust]) == [v"5.2.0", v"6.1.0"]
     end
 
     @testset "Compiler wrappers" begin