From 41cbc8341aaa839d0606f9d1d250ad6036f2f482 Mon Sep 17 00:00:00 2001 From: Ian McInerney Date: Fri, 3 Jan 2025 16:20:53 +0000 Subject: [PATCH 1/7] Include compiler information (if any) in the meta JSON output --- src/AutoBuild.jl | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/AutoBuild.jl b/src/AutoBuild.jl index 0b5cd2f5..c31bf30a 100644 --- a/src/AutoBuild.jl +++ b/src/AutoBuild.jl @@ -345,10 +345,11 @@ function build_tarballs(ARGS, src_name, src_version, sources, script, dependencies, ) extra_kwargs = extract_kwargs(kwargs, (:lazy_artifacts, :init_block, :augment_platform_block)) + compiler_kwargs = extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:preferred_rust_version,:preferred_go_version,:compilers,:clang_use_lld)) if meta_json_stream !== nothing # If they've asked for the JSON metadata, by all means, give it to them! - dict = get_meta_json(args...; extra_kwargs..., julia_compat=julia_compat) + dict = get_meta_json(args...; extra_kwargs..., compiler_kwargs..., julia_compat=julia_compat) println(meta_json_stream, JSON.json(dict)) if meta_json_stream !== stdout @@ -665,6 +666,7 @@ function get_meta_json( init_block::String = "", augment_platform_block::String = "", lazy_artifacts::Bool=!isempty(augment_platform_block) && minimum_compat(julia_compat) < v"1.7", + kwargs... ) dict = Dict( @@ -683,6 +685,12 @@ function get_meta_json( if platforms != [AnyPlatform()] dict["platforms"] = triplet.(platforms) end + + # Add any other optional things we want to include + for (k, v) in kwargs + dict["$k"] = v + end + return dict end From 38633d59443f91d6db857f89c9ee82709df9406d Mon Sep 17 00:00:00 2001 From: Ian McInerney Date: Fri, 3 Jan 2025 19:01:00 +0000 Subject: [PATCH 2/7] Implement per-platform compiler version selection --- src/AutoBuild.jl | 56 ++++++++++++++++++++++++++++++++++++++++++++--- test/building.jl | 50 ++++++++++++++++++++++++++++++++++++++++++ test/compilers.jl | 35 +++++++++++++++++++++++++++++ test/runtests.jl | 1 + 4 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 test/compilers.jl diff --git a/src/AutoBuild.jl b/src/AutoBuild.jl index c31bf30a..07cc69f5 100644 --- a/src/AutoBuild.jl +++ b/src/AutoBuild.jl @@ -1,4 +1,4 @@ -export build_tarballs, autobuild, print_artifacts_toml, build, get_meta_json +export build_tarballs, autobuild, print_artifacts_toml, build, get_meta_json, preferred_platform_compiler, set_preferred_compiler_version! import GitHub: gh_get_json, DEFAULT_API import SHA: sha256, sha1 using TOML, Dates, UUIDs @@ -720,6 +720,49 @@ function compose_debug_prompt(workspace) return debug_shell_prompt end +""" + preferred_platform_compiler(platforms::Vector{<:Platform}, version::Union{Nothing,VersionNumber}=nothing) + +Initializes a dictionary with an entry for every platform given +that will store platform-specific compiler versions (if any). +""" +function preferred_platform_compiler(platforms::Vector{<:Platform}, version::Union{Nothing,VersionNumber}=nothing) + compiler_versions = Dict{Platform, Union{Nothing,VersionNumber}}(p => version for p in platforms) + return compiler_versions +end + +""" + set_preferred_compiler_version!(compiler_versions::Dict{Platform, Union{Nothing,VersionNumber}}, + version::VersionNumber, + selector::Union{Vector{<:Platform},Function}) + +Set the preferred compiler version for the platforms matching `selector` to `version`. +""" +function set_preferred_compiler_version!(compiler_versions::Dict{Platform, Union{Nothing,VersionNumber}}, + version::VersionNumber, + selector::Union{Vector{<:Platform},Function}) + matching_platform(selector::Function, platform) = selector(platform) + matching_platform(selector::Vector{<:Platform}, platform) = platform in selector + + for (platform, compiler) in compiler_versions + if matching_platform(selector, platform) + compiler_versions[platform] = version + end + end + + return compiler_versions +end + +function get_platform_compiler_version!(compiler_args, key, platform, version) + if version isa VersionNumber + compiler_args[key] = version + elseif version isa Dict + if !isnothing(version[platform]) + compiler_args[key] = version[platform] + end + end +end + """ autobuild(dir::AbstractString, src_name::AbstractString, src_version::VersionNumber, sources::Vector, @@ -823,11 +866,17 @@ function autobuild(dir::AbstractString, timer = BuildTimer() timer.begin_setup = time() + compiler_args = Dict() + + for (k, v) in extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:preferred_rust_version,:preferred_go_version)) + get_platform_compiler_version!(compiler_args, k, platform, kwargs[k]) + end + # We build in a platform-specific directory build_path = joinpath(dir, "build", triplet(platform)) mkpath(build_path) - shards = choose_shards(platform; extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:preferred_rust_version,:preferred_go_version,:bootstrap_list,:compilers))...) + shards = choose_shards(platform; compiler_args..., extract_kwargs(kwargs, (:bootstrap_list,:compilers))...) concrete_platform = get_concrete_platform(platform, shards) prefix = setup_workspace( @@ -854,7 +903,8 @@ function autobuild(dir::AbstractString, compiler_wrapper_dir = joinpath(prefix, "compiler_wrappers"), src_name = src_name, shards = shards, - extract_kwargs(kwargs, (:preferred_gcc_version,:preferred_llvm_version,:compilers,:allow_unsafe_flags,:lock_microarchitecture,:clang_use_lld))..., + compiler_args..., + extract_kwargs(kwargs, (:compilers,:allow_unsafe_flags,:lock_microarchitecture,:clang_use_lld))..., ) # Set up some bash traps diff --git a/test/building.jl b/test/building.jl index 4bd08d46..cc415640 100644 --- a/test/building.jl +++ b/test/building.jl @@ -479,3 +479,53 @@ end ) end end + +@testset "Different compiler versions" begin + platforms = [Platform("x86_64", "linux"), + Platform("x86_64", "freebsd")] + + # Use GCC 5 on FreeBSD and GCC 10 on Linux + c = preferred_platform_compiler(platforms, v"5") + set_preferred_compiler_version!(c, v"10", p ->Sys.islinux(p)) + + mktempdir() do build_path + build_output_meta = autobuild( + build_path, + "libfoo", + v"1.0.0", + [DirectorySource(build_tests_dir)], + # Include a GCC version check before we build + raw""" + export ver=$(cc -dumpversion) + export shortver=${ver%.*.*} + + if [[ "${target}" == *-linux-* ]]; then + if [[ $shortver == "10" ]]; then + # This is what we want + echo "Success! GCC version correct" + else + # This is not what we want + exit 1 + fi + else + if [[ $shortver == "5" ]]; then + # This is what we want + echo "Success! GCC version correct" + else + # This is not what we want + exit 1 + fi + fi + """*libfoo_make_script, + platforms, + libfoo_products, + # No dependencies + Dependency[]; + # Don't do audit passes + skip_audit=true, + # Specify the per-platform GCC versions + preferred_gcc_version=c, + ) + @test haskey(build_output_meta, p) + end +end diff --git a/test/compilers.jl b/test/compilers.jl new file mode 100644 index 00000000..f9ae90ec --- /dev/null +++ b/test/compilers.jl @@ -0,0 +1,35 @@ +using BinaryBuilder, Test + +@testset "Compiler dictionary creation" begin + platforms = supported_platforms() + + pd = preferred_platform_compiler(platforms) + + # Make sure everything is nothing + @test length(platforms) == length(pd) + @test all(isnothing.(values(pd))) + + # Try the expanded platform list + platforms = supported_platforms() + platforms = expand_gfortran_versions() + pd = preferred_platform_compiler(platforms) + @test length(platforms) == length(pd) + @test all(isnothing.(values(pd))) + + # Set everything to a version + pd = preferred_platform_compiler(platforms, v"5") + @test length(platforms) == length(pd) + @test all(values(pd) .== v"5") + + # Assign some compiler values + pd = preferred_platform_compiler(platforms, v"5") + set_preferred_compiler_version!(pd, v"10", p ->Sys.islinux(p)) + set_preferred_compiler_version!(pd, v"6", p ->Sys.isfreebsd(p)) + set_preferred_compiler_version!(pd, v"12", [Platform("x86_64", "macos")]) + set_preferred_compiler_version!(pd, v"13", [Platform("aarch64", "macos")]) + + @test pd[Platform("x86_64", "linux")] == v"10" + @test pd[Platform("x86_64", "freebsd")] == v"6" + @test pd[Platform("x86_64", "macos")] == v"12" + @test pd[Platform("aarch64", "macos")] == v"13" +end diff --git a/test/runtests.jl b/test/runtests.jl index e29bb570..d31bdd0d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -67,6 +67,7 @@ install_license ${WORKSPACE}/srcdir/libfoo/LICENSE.md # Run all our tests include("basic.jl") +include("compilers.jl") include("building.jl") include("auditing.jl") include("jll.jl") From c1812e8d6f0063ba7479e07c537f48735cc27ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= <765740+giordano@users.noreply.github.com> Date: Fri, 3 Jan 2025 21:14:23 +0000 Subject: [PATCH 3/7] Update test/compilers.jl --- test/compilers.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/compilers.jl b/test/compilers.jl index f9ae90ec..9a8a8ac6 100644 --- a/test/compilers.jl +++ b/test/compilers.jl @@ -11,7 +11,7 @@ using BinaryBuilder, Test # Try the expanded platform list platforms = supported_platforms() - platforms = expand_gfortran_versions() + platforms = expand_gfortran_versions(platforms) pd = preferred_platform_compiler(platforms) @test length(platforms) == length(pd) @test all(isnothing.(values(pd))) From ac1fa23b84c7a06e1aea4cf3c134bb783be24d27 Mon Sep 17 00:00:00 2001 From: Ian McInerney Date: Fri, 3 Jan 2025 22:42:50 +0000 Subject: [PATCH 4/7] Update test/compilers.jl --- test/compilers.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/test/compilers.jl b/test/compilers.jl index 9a8a8ac6..eaf53942 100644 --- a/test/compilers.jl +++ b/test/compilers.jl @@ -22,6 +22,7 @@ using BinaryBuilder, Test @test all(values(pd) .== v"5") # Assign some compiler values + platforms = supported_platforms() pd = preferred_platform_compiler(platforms, v"5") set_preferred_compiler_version!(pd, v"10", p ->Sys.islinux(p)) set_preferred_compiler_version!(pd, v"6", p ->Sys.isfreebsd(p)) From 1fac9eb8ea216bd7001be8cedd0f977b8ae5fd13 Mon Sep 17 00:00:00 2001 From: Ian McInerney Date: Fri, 3 Jan 2025 23:30:20 +0000 Subject: [PATCH 5/7] Minor changes --- test/building.jl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/building.jl b/test/building.jl index cc415640..9956d716 100644 --- a/test/building.jl +++ b/test/building.jl @@ -499,6 +499,8 @@ end export ver=$(cc -dumpversion) export shortver=${ver%.*.*} + echo "Compiler version: $shortver" + if [[ "${target}" == *-linux-* ]]; then if [[ $shortver == "10" ]]; then # This is what we want @@ -523,6 +525,8 @@ end Dependency[]; # Don't do audit passes skip_audit=true, + # Make it verbose to see the results + verbose=true, # Specify the per-platform GCC versions preferred_gcc_version=c, ) From dd3658ba54a38a451acf53b33380374a93e15a42 Mon Sep 17 00:00:00 2001 From: Ian McInerney Date: Fri, 3 Jan 2025 23:56:54 +0000 Subject: [PATCH 6/7] Switch to linux for both, freebsd uses clang not gcc --- test/building.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/building.jl b/test/building.jl index 9956d716..6d0b1f67 100644 --- a/test/building.jl +++ b/test/building.jl @@ -482,11 +482,11 @@ end @testset "Different compiler versions" begin platforms = [Platform("x86_64", "linux"), - Platform("x86_64", "freebsd")] + Platform("aarch64", "linux")] # Use GCC 5 on FreeBSD and GCC 10 on Linux c = preferred_platform_compiler(platforms, v"5") - set_preferred_compiler_version!(c, v"10", p ->Sys.islinux(p)) + set_preferred_compiler_version!(c, v"10", p -> arch(p) == "aarch64" ) mktempdir() do build_path build_output_meta = autobuild( @@ -501,7 +501,7 @@ end echo "Compiler version: $shortver" - if [[ "${target}" == *-linux-* ]]; then + if [[ "${target}" == aarch64-* ]]; then if [[ $shortver == "10" ]]; then # This is what we want echo "Success! GCC version correct" From d23ffdb745fafdb63eacb904d1b8229c7255f459 Mon Sep 17 00:00:00 2001 From: Ian McInerney Date: Sat, 4 Jan 2025 00:22:16 +0000 Subject: [PATCH 7/7] Update test/building.jl --- test/building.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/building.jl b/test/building.jl index 6d0b1f67..96926eb0 100644 --- a/test/building.jl +++ b/test/building.jl @@ -530,6 +530,6 @@ end # Specify the per-platform GCC versions preferred_gcc_version=c, ) - @test haskey(build_output_meta, p) + @test length(keys(build_output_meta)) == length(platforms) end end