From 73b875d35b9c4fd1d8404b73f371cf4caf3a9776 Mon Sep 17 00:00:00 2001 From: Venkateshprasad Date: Thu, 13 Jan 2022 14:40:18 +0530 Subject: [PATCH 1/7] Make BBBase relocatable on sysimages - create a const artifacts_toml with @path - use envcache to determine the version instead of parsing the project.toml: Using @path here doesn't work as - a: Project.toml which at DIR*.. - b: the entire `src` would be stored in the Path object (and not Project.toml) - While creating storage_cache check if "deps_path" is valid else create one at dev/BBBase.jl/deps: Even here using @path directly is of no use as you are creating a new folder and don't intend any file to survive relocation So now it checks if the path is valid; ow creates one at .julia/dev Leaving this storage_cache untouched will result in creating a folder identical to original-depot-path failing all the RelocatableFolder's `getpath` efforts --- Project.toml | 2 +- src/BinaryBuilderBase.jl | 15 ++++++++++++--- src/DockerRunner.jl | 1 - src/Rootfs.jl | 6 ------ 4 files changed, 13 insertions(+), 11 deletions(-) diff --git a/Project.toml b/Project.toml index f62e8f13..28d9a1d6 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "BinaryBuilderBase" uuid = "7f725544-6523-48cd-82d1-3fa08ff4056e" authors = ["Elliot Saba "] -version = "1.3.0" +version = "1.3.1" [deps] CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" diff --git a/src/BinaryBuilderBase.jl b/src/BinaryBuilderBase.jl index 748760cf..48ff599a 100644 --- a/src/BinaryBuilderBase.jl +++ b/src/BinaryBuilderBase.jl @@ -3,7 +3,7 @@ module BinaryBuilderBase using Pkg, Pkg.Artifacts, Random, Libdl, InteractiveUtils using Base.BinaryPlatforms using Downloads -using JSON, OutputCollectors, Scratch +using JSON, OutputCollectors, Scratch, RelocatableFolders # Re-export useful stuff from Base.BinaryPlatforms: export HostPlatform, platform_dlext, valid_dl_path, arch, libc, @@ -70,9 +70,13 @@ const allow_ecryptfs = Ref(false) const use_ccache = Ref(false) const bootstrap_list = Symbol[] +const artifacts_toml = @path Pkg.Artifacts.find_artifacts_toml(@__FILE__) + function get_bbb_version(dir=@__DIR__, uuid="7f725544-6523-48cd-82d1-3fa08ff4056e") # Get BinaryBuilder.jl's version and git sha - version = Pkg.TOML.parsefile(joinpath(dir, "..", "Project.toml"))["version"] + version = Pkg.activate() do + Pkg.Types.EnvCache().project.version + end try # get the gitsha if we can repo = LibGit2.GitRepo(dirname(@__DIR__)) @@ -170,9 +174,14 @@ function __init__() global runner_override, use_squashfs, allow_ecryptfs global use_ccache, storage_cache + deps_path = joinpath(@__DIR__, "..", "deps") + ispath(deps_path) || begin + deps_path = joinpath(Base.DEPOT_PATH, "BinaryBuilderBase.jl", "deps") + @info "Creating a storage cache at $deps_path" + end # Allow the user to override the default value for `storage_dir` storage_cache[] = get(ENV, "BINARYBUILDER_STORAGE_DIR", - abspath(joinpath(@__DIR__, "..", "deps"))) + abspath(deps_path)) # If the user has signalled that they really want us to automatically # accept apple EULAs, do that. diff --git a/src/DockerRunner.jl b/src/DockerRunner.jl index 3488cd5d..64241ede 100644 --- a/src/DockerRunner.jl +++ b/src/DockerRunner.jl @@ -16,7 +16,6 @@ end docker_image(version::VersionNumber) = "julia_binarybuilder_rootfs:v$(version)" function docker_image(rootfs::CompilerShard) name = artifact_name(rootfs) - artifacts_toml = joinpath(@__DIR__, "..", "Artifacts.toml") hash = artifact_hash(name, artifacts_toml; platform=rootfs.host) return string( "julia_binarybuilder_rootfs:", diff --git a/src/Rootfs.jl b/src/Rootfs.jl index 856a36de..e56cdbca 100644 --- a/src/Rootfs.jl +++ b/src/Rootfs.jl @@ -94,7 +94,6 @@ end const ALL_SHARDS = Ref{Union{Vector{CompilerShard},Nothing}}(nothing) function all_compiler_shards()::Vector{CompilerShard} if ALL_SHARDS[] === nothing - artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml") artifact_dict = load_artifacts_toml(artifacts_toml) ALL_SHARDS[] = CompilerShard[] @@ -124,7 +123,6 @@ function all_compiler_shards()::Vector{CompilerShard} end function shard_source_artifact_hash(cs::CompilerShard) - artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml") name = artifact_name(cs) hash = artifact_hash( name, @@ -179,7 +177,6 @@ function mount_path(cs::CompilerShard, build_prefix::AbstractString) return joinpath(build_prefix, ".mounts", artifact_name(cs)) else name = artifact_name(cs) - artifacts_toml = joinpath(@__DIR__, "..", "Artifacts.toml") hash = artifact_hash(name, artifacts_toml; platform=cs.host) if hash === nothing error("Unable to find artifact $(name) within $(artifacts_toml)") @@ -254,7 +251,6 @@ function mount(cs::CompilerShard, build_prefix::AbstractString; verbose::Bool = end # Ensure this artifact is on-disk; hard to mount it if it's not installed - artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml") ensure_artifact_installed(artifact_name(cs), artifacts_toml; platform=cs.host, verbose=true) # Easy out if we're not Linux with a UserNSRunner trying to use a .squashfs @@ -345,7 +341,6 @@ function macos_sdk_already_installed() css = all_compiler_shards() macos_artifact_names = artifact_name.(filter(cs -> cs.target !== nothing && Sys.isapple(cs.target::Platform), css)) - artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml") macos_artifact_hashes = artifact_hash.(macos_artifact_names, artifacts_toml; platform=default_host_platform) # Return `true` if _any_ of these artifacts exist on-disk: @@ -913,7 +908,6 @@ happens, you don't need an internet connection to build your precious, precious binaries. """ function download_all_artifacts(; verbose::Bool = false) - artifacts_toml = joinpath(dirname(@__DIR__), "Artifacts.toml") ensure_all_artifacts_installed( artifacts_toml; include_lazy=true, From 1218a09020603997f9fea0e1ea6d01e9018656d4 Mon Sep 17 00:00:00 2001 From: Venkateshprasad Date: Mon, 17 Jan 2022 22:16:09 +0530 Subject: [PATCH 2/7] use depots1 --- src/BinaryBuilderBase.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/BinaryBuilderBase.jl b/src/BinaryBuilderBase.jl index 48ff599a..07c4f161 100644 --- a/src/BinaryBuilderBase.jl +++ b/src/BinaryBuilderBase.jl @@ -176,7 +176,7 @@ function __init__() deps_path = joinpath(@__DIR__, "..", "deps") ispath(deps_path) || begin - deps_path = joinpath(Base.DEPOT_PATH, "BinaryBuilderBase.jl", "deps") + deps_path = joinpath(Pkg.depots1(), "dev", "BinaryBuilderBase.jl", "deps") @info "Creating a storage cache at $deps_path" end # Allow the user to override the default value for `storage_dir` From 377c30ed6941a70d14ef621b27722cbbf9d5e509 Mon Sep 17 00:00:00 2001 From: Venkateshprasad Date: Tue, 18 Jan 2022 13:49:43 +0530 Subject: [PATCH 3/7] Use String explicitly --- src/BinaryBuilderBase.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/BinaryBuilderBase.jl b/src/BinaryBuilderBase.jl index 07c4f161..d1963c47 100644 --- a/src/BinaryBuilderBase.jl +++ b/src/BinaryBuilderBase.jl @@ -70,7 +70,8 @@ const allow_ecryptfs = Ref(false) const use_ccache = Ref(false) const bootstrap_list = Symbol[] -const artifacts_toml = @path Pkg.Artifacts.find_artifacts_toml(@__FILE__) +const _artifacts_toml = @path Pkg.Artifacts.find_artifacts_toml(@__FILE__) +const artifacts_toml = String(_artifacts_toml) function get_bbb_version(dir=@__DIR__, uuid="7f725544-6523-48cd-82d1-3fa08ff4056e") # Get BinaryBuilder.jl's version and git sha From 107498f73ae1f2ca600814ce6ff1e16321242396 Mon Sep 17 00:00:00 2001 From: Venkateshprasad Date: Thu, 20 Jan 2022 10:04:24 +0530 Subject: [PATCH 4/7] getpath of art_toml locally --- src/BinaryBuilderBase.jl | 1 - src/DockerRunner.jl | 1 + src/Rootfs.jl | 6 ++++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/BinaryBuilderBase.jl b/src/BinaryBuilderBase.jl index d1963c47..405e6003 100644 --- a/src/BinaryBuilderBase.jl +++ b/src/BinaryBuilderBase.jl @@ -71,7 +71,6 @@ const use_ccache = Ref(false) const bootstrap_list = Symbol[] const _artifacts_toml = @path Pkg.Artifacts.find_artifacts_toml(@__FILE__) -const artifacts_toml = String(_artifacts_toml) function get_bbb_version(dir=@__DIR__, uuid="7f725544-6523-48cd-82d1-3fa08ff4056e") # Get BinaryBuilder.jl's version and git sha diff --git a/src/DockerRunner.jl b/src/DockerRunner.jl index 64241ede..f83a2e97 100644 --- a/src/DockerRunner.jl +++ b/src/DockerRunner.jl @@ -16,6 +16,7 @@ end docker_image(version::VersionNumber) = "julia_binarybuilder_rootfs:v$(version)" function docker_image(rootfs::CompilerShard) name = artifact_name(rootfs) + artifacts_toml = String(_artifacts_toml) hash = artifact_hash(name, artifacts_toml; platform=rootfs.host) return string( "julia_binarybuilder_rootfs:", diff --git a/src/Rootfs.jl b/src/Rootfs.jl index e56cdbca..1f0f7ee0 100644 --- a/src/Rootfs.jl +++ b/src/Rootfs.jl @@ -94,6 +94,7 @@ end const ALL_SHARDS = Ref{Union{Vector{CompilerShard},Nothing}}(nothing) function all_compiler_shards()::Vector{CompilerShard} if ALL_SHARDS[] === nothing + artifacts_toml = String(_artifacts_toml) artifact_dict = load_artifacts_toml(artifacts_toml) ALL_SHARDS[] = CompilerShard[] @@ -124,6 +125,7 @@ end function shard_source_artifact_hash(cs::CompilerShard) name = artifact_name(cs) + artifacts_toml = String(_artifacts_toml) hash = artifact_hash( name, artifacts_toml; @@ -177,6 +179,7 @@ function mount_path(cs::CompilerShard, build_prefix::AbstractString) return joinpath(build_prefix, ".mounts", artifact_name(cs)) else name = artifact_name(cs) + artifacts_toml = String(_artifacts_toml) hash = artifact_hash(name, artifacts_toml; platform=cs.host) if hash === nothing error("Unable to find artifact $(name) within $(artifacts_toml)") @@ -251,6 +254,7 @@ function mount(cs::CompilerShard, build_prefix::AbstractString; verbose::Bool = end # Ensure this artifact is on-disk; hard to mount it if it's not installed + artifacts_toml = String(_artifacts_toml) ensure_artifact_installed(artifact_name(cs), artifacts_toml; platform=cs.host, verbose=true) # Easy out if we're not Linux with a UserNSRunner trying to use a .squashfs @@ -341,6 +345,7 @@ function macos_sdk_already_installed() css = all_compiler_shards() macos_artifact_names = artifact_name.(filter(cs -> cs.target !== nothing && Sys.isapple(cs.target::Platform), css)) + artifacts_toml = String(_artifacts_toml) macos_artifact_hashes = artifact_hash.(macos_artifact_names, artifacts_toml; platform=default_host_platform) # Return `true` if _any_ of these artifacts exist on-disk: @@ -908,6 +913,7 @@ happens, you don't need an internet connection to build your precious, precious binaries. """ function download_all_artifacts(; verbose::Bool = false) + artifacts_toml = String(_artifacts_toml) ensure_all_artifacts_installed( artifacts_toml; include_lazy=true, From 8f19edc6a586fa32c49e52c4b806e397c6d41ab4 Mon Sep 17 00:00:00 2001 From: Venkateshprasad Date: Tue, 25 Jan 2022 15:42:31 +0530 Subject: [PATCH 5/7] activate . while fetching the version --- src/BinaryBuilderBase.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/BinaryBuilderBase.jl b/src/BinaryBuilderBase.jl index 405e6003..7a6d479d 100644 --- a/src/BinaryBuilderBase.jl +++ b/src/BinaryBuilderBase.jl @@ -74,7 +74,7 @@ const _artifacts_toml = @path Pkg.Artifacts.find_artifacts_toml(@__FILE__) function get_bbb_version(dir=@__DIR__, uuid="7f725544-6523-48cd-82d1-3fa08ff4056e") # Get BinaryBuilder.jl's version and git sha - version = Pkg.activate() do + version = Pkg.activate(".") do Pkg.Types.EnvCache().project.version end try @@ -176,7 +176,7 @@ function __init__() deps_path = joinpath(@__DIR__, "..", "deps") ispath(deps_path) || begin - deps_path = joinpath(Pkg.depots1(), "dev", "BinaryBuilderBase.jl", "deps") + deps_path = joinpath(Pkg.depots1(), "binary_builder_deps") @info "Creating a storage cache at $deps_path" end # Allow the user to override the default value for `storage_dir` From fd1786d0a03f0ce69f6d3ddc65ba63febe80ada5 Mon Sep 17 00:00:00 2001 From: Venkateshprasad Date: Mon, 31 Jan 2022 19:19:57 +0530 Subject: [PATCH 6/7] return project.version after verifying project.name --- src/BinaryBuilderBase.jl | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/BinaryBuilderBase.jl b/src/BinaryBuilderBase.jl index 350880b5..d2fce81d 100644 --- a/src/BinaryBuilderBase.jl +++ b/src/BinaryBuilderBase.jl @@ -75,7 +75,13 @@ const _artifacts_toml = @path Pkg.Artifacts.find_artifacts_toml(@__FILE__) function get_bbb_version(dir=@__DIR__, uuid="7f725544-6523-48cd-82d1-3fa08ff4056e") # Get BinaryBuilder.jl's version and git sha version = Pkg.activate(".") do - Pkg.Types.EnvCache().project.version + Pkg.Types.EnvCache().project.name == "BinaryBuilderBase" ? + Pkg.Types.EnvCache().project.version : + try + Pkg.TOML.parsefile(normpath(Base.find_package("BinaryBuilderBase.jl"), "..", "..", "Project.toml"))["version"] + catch e + nothing + end end try # get the gitsha if we can From 7d70fb6c2ca73503b3f6505a45ceccc7135dbd7b Mon Sep 17 00:00:00 2001 From: Venkateshprasad Date: Wed, 16 Feb 2022 19:12:48 +0530 Subject: [PATCH 7/7] use dir --- src/BinaryBuilderBase.jl | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/src/BinaryBuilderBase.jl b/src/BinaryBuilderBase.jl index d2fce81d..3f6fff36 100644 --- a/src/BinaryBuilderBase.jl +++ b/src/BinaryBuilderBase.jl @@ -74,15 +74,17 @@ const _artifacts_toml = @path Pkg.Artifacts.find_artifacts_toml(@__FILE__) function get_bbb_version(dir=@__DIR__, uuid="7f725544-6523-48cd-82d1-3fa08ff4056e") # Get BinaryBuilder.jl's version and git sha - version = Pkg.activate(".") do - Pkg.Types.EnvCache().project.name == "BinaryBuilderBase" ? - Pkg.Types.EnvCache().project.version : - try - Pkg.TOML.parsefile(normpath(Base.find_package("BinaryBuilderBase.jl"), "..", "..", "Project.toml"))["version"] - catch e - nothing - end - end + version = isdir(dir) ? Pkg.TOML.parsefile(joinpath(dir, "..", "Project.toml"))["version"] : + Pkg.activate(".") do + pkgname = basename(dirname(dirname(dir))) + Pkg.Types.EnvCache().project.name == pkgname ? + Pkg.Types.EnvCache().project.version : + try + Pkg.TOML.parsefile(normpath(Base.find_package(pkgname), "..", "..", "Project.toml"))["version"] + catch e + @info "Couldn't find $pkgname in the current environment" + end + end try # get the gitsha if we can repo = LibGit2.GitRepo(dirname(@__DIR__))