From 70825b877b591332efa34d6431dd29fe1b905086 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 10 Apr 2024 20:08:10 +0200 Subject: [PATCH] [backports-release-1.10] allow extensions to trigger from packages in [deps] (#54009) There is a use case where you have a weak dependency (for one of your extensions) that is misbehaving and you quickly want to try debug that issue. A workflow that feels reasonable for this could be: ``` pkg> dev WeakDependency julia> using Package, WeakDependency ``` This doesn't work right now for two reasons: 1. Doing the `dev WeakDependency` will add the dependency to `[deps]` but not remove it from `[weakdeps]` which means you all of a sudden are in the scenario described in https://pkgdocs.julialang.org/v1/creating-packages/#Transition-from-normal-dependency-to-extension which is not what is desired. 2. The extension will not actually load because you can right now only trigger extensions from weak deps getting loaded, not from deps getting loaded. Point 1. is fixed by https://github.com/JuliaLang/Pkg.jl/pull/3865 Point 2. is fixed by this PR. (cherry picked from commit f46cb4c67cd2ce7f88da778f5c0355c1100b5ccd) --- base/loading.jl | 53 +++++++++++-------- doc/src/manual/code-loading.md | 12 ++--- test/loading.jl | 5 ++ .../EnvWithHasExtensions/Manifest.toml | 6 ++- .../Extensions/ExtDep3.jl/Project.toml | 4 ++ .../Extensions/ExtDep3.jl/src/ExtDep3.jl | 5 ++ .../HasDepWithExtensions.jl/Manifest.toml | 9 +++- .../HasDepWithExtensions.jl/Project.toml | 1 + .../Extensions/HasExtensions.jl/Manifest.toml | 9 ++-- .../Extensions/HasExtensions.jl/Project.toml | 4 ++ .../HasExtensions.jl/ext/ExtensionDep.jl | 9 ++++ .../HasExtensions.jl/src/HasExtensions.jl | 1 + 12 files changed, 84 insertions(+), 34 deletions(-) create mode 100644 test/project/Extensions/ExtDep3.jl/Project.toml create mode 100644 test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl create mode 100644 test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl diff --git a/base/loading.jl b/base/loading.jl index c271babe57484..57080c4b5598e 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1211,12 +1211,13 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi proj_pkg = project_file_name_uuid(project_file, pkg.name) if pkg == proj_pkg d_proj = parsed_toml(project_file) - weakdeps = get(d_proj, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}} extensions = get(d_proj, "extensions", nothing)::Union{Nothing, Dict{String, Any}} extensions === nothing && return - weakdeps === nothing && return - if weakdeps isa Dict{String, Any} - return _insert_extension_triggers(pkg, extensions, weakdeps) + weakdeps = get(Dict{String, Any}, d_proj, "weakdeps")::Union{Vector{String}, Dict{String,Any}} + deps = get(Dict{String, Any}, d_proj, "deps")::Union{Vector{String}, Dict{String,Any}} + if weakdeps isa Dict{String,Any} && deps isa Dict{String,Any} + total_deps = merge(weakdeps, deps) + return _insert_extension_triggers(pkg, extensions, total_deps) end end @@ -1231,27 +1232,35 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi uuid = get(entry, "uuid", nothing)::Union{String, Nothing} uuid === nothing && continue if UUID(uuid) == pkg.uuid - weakdeps = get(entry, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}} extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}} extensions === nothing && return - weakdeps === nothing && return - if weakdeps isa Dict{String, Any} - return _insert_extension_triggers(pkg, extensions, weakdeps) + weakdeps = get(Dict{String, Any}, entry, "weakdeps")::Union{Vector{String}, Dict{String,Any}} + deps = get(Dict{String, Any}, entry, "deps")::Union{Vector{String}, Dict{String,Any}} + + function expand_deps_list(deps′::Vector{String}) + deps′_expanded = Dict{String, Any}() + for (dep_name, entries) in d + dep_name in deps′ || continue + entries::Vector{Any} + if length(entries) != 1 + error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))") + end + entry = first(entries)::Dict{String, Any} + uuid = entry["uuid"]::String + deps′_expanded[dep_name] = uuid + end + return deps′_expanded end - d_weakdeps = Dict{String, Any}() - for (dep_name, entries) in d - dep_name in weakdeps || continue - entries::Vector{Any} - if length(entries) != 1 - error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))") - end - entry = first(entries)::Dict{String, Any} - uuid = entry["uuid"]::String - d_weakdeps[dep_name] = uuid + if weakdeps isa Vector{String} + weakdeps = expand_deps_list(weakdeps) end - @assert length(d_weakdeps) == length(weakdeps) - return _insert_extension_triggers(pkg, extensions, d_weakdeps) + if deps isa Vector{String} + deps = expand_deps_list(deps) + end + + total_deps = merge(weakdeps, deps) + return _insert_extension_triggers(pkg, extensions, total_deps) end end end @@ -1259,7 +1268,7 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi return nothing end -function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, weakdeps::Dict{String, Any}) +function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, totaldeps::Dict{String, Any}) for (ext, triggers) in extensions triggers = triggers::Union{String, Vector{String}} triggers isa String && (triggers = [triggers]) @@ -1273,7 +1282,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any} push!(trigger1, gid) for trigger in triggers # TODO: Better error message if this lookup fails? - uuid_trigger = UUID(weakdeps[trigger]::String) + uuid_trigger = UUID(totaldeps[trigger]::String) trigger_id = PkgId(uuid_trigger, trigger) if !haskey(Base.loaded_modules, trigger_id) || haskey(package_locks, trigger_id) trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id) diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 743ee83c333a4..16b1b563f7f35 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -351,7 +351,7 @@ Since the primary environment is typically the environment of a project you're w ### [Package Extensions](@id man-extensions) -A package "extension" is a module that is automatically loaded when a specified set of other packages (its "extension dependencies") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The extension dependencies of an extension are a subset of those packages listed under the `[weakdeps]` section of the project file. Those packages can have compat entries like other packages. +A package "extension" is a module that is automatically loaded when a specified set of other packages (its "triggers") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The triggers of an extension are a subset of those packages listed under the `[weakdeps]` (and possibly, but uncommonly the `[deps]`) section of the project file. Those packages can have compat entries like other packages. ```toml name = "MyPackage" @@ -371,8 +371,8 @@ FooExt = "ExtDep" ``` The keys under `extensions` are the names of the extensions. -They are loaded when all the packages on the right hand side (the extension dependencies) of that extension are loaded. -If an extension only has one extension dependency the list of extension dependencies can be written as just a string for brevity. +They are loaded when all the packages on the right hand side (the triggers) of that extension are loaded. +If an extension only has one trigger the list of triggers can be written as just a string for brevity. The location for the entry point of the extension is either in `ext/FooExt.jl` or `ext/FooExt/FooExt.jl` for extension `FooExt`. The content of an extension is often structured as: @@ -380,10 +380,10 @@ The content of an extension is often structured as: ``` module FooExt -# Load main package and extension dependencies +# Load main package and triggers using MyPackage, ExtDep -# Extend functionality in main package with types from the extension dependencies +# Extend functionality in main package with types from the triggers MyPackage.func(x::ExtDep.SomeStruct) = ... end @@ -391,7 +391,7 @@ end When a package with extensions is added to an environment, the `weakdeps` and `extensions` sections are stored in the manifest file in the section for that package. The dependency lookup rules for -a package are the same as for its "parent" except that the listed extension dependencies are also considered as +a package are the same as for its "parent" except that the listed triggers are also considered as dependencies. ### [Package/Environment Preferences](@id preferences) diff --git a/test/loading.jl b/test/loading.jl index 292eabe28a878..dde13da066c80 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1028,6 +1028,9 @@ end using ExtDep2 $ew using ExtDep2 $ew HasExtensions.ext_folder_loaded || error("ext_folder_loaded not set") + using ExtDep3 + $ew using ExtDep3 + $ew HasExtensions.ext_dep_loaded || error("ext_dep_loaded not set") end """ return `$(Base.julia_cmd()) $compile --startup-file=no -e $cmd` @@ -1075,6 +1078,8 @@ end Base.get_extension(HasExtensions, :Extension) isa Module || error("expected extension to load") using ExtDep2 Base.get_extension(HasExtensions, :ExtensionFolder) isa Module || error("expected extension to load") + using ExtDep3 + Base.get_extension(HasExtensions, :ExtensionDep) isa Module || error("expected extension to load") end """ for compile in (`--compiled-modules=no`, ``) diff --git a/test/project/Extensions/EnvWithHasExtensions/Manifest.toml b/test/project/Extensions/EnvWithHasExtensions/Manifest.toml index 8ac961fa1a9a9..004ef7892c173 100644 --- a/test/project/Extensions/EnvWithHasExtensions/Manifest.toml +++ b/test/project/Extensions/EnvWithHasExtensions/Manifest.toml @@ -1,8 +1,8 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.0-beta4" +julia_version = "1.12.0-DEV" manifest_format = "2.0" -project_hash = "caa716752e6dff3d77c3de929ebbb5d2024d04ef" +project_hash = "a4c480cfa7da9610333d5c42623bf746bd286c5f" [[deps.ExtDep]] deps = ["SomePackage"] @@ -18,10 +18,12 @@ version = "0.1.0" [deps.HasExtensions.extensions] Extension = "ExtDep" ExtensionFolder = ["ExtDep", "ExtDep2"] + LinearAlgebraExt = "LinearAlgebra" [deps.HasExtensions.weakdeps] ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" + LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" [[deps.SomePackage]] path = "../SomePackage" diff --git a/test/project/Extensions/ExtDep3.jl/Project.toml b/test/project/Extensions/ExtDep3.jl/Project.toml new file mode 100644 index 0000000000000..690b2f1cffff4 --- /dev/null +++ b/test/project/Extensions/ExtDep3.jl/Project.toml @@ -0,0 +1,4 @@ +name = "ExtDep3" +uuid = "a5541f1e-a556-4fdc-af15-097880d743a1" +version = "0.1.0" +authors = ["Kristoffer "] diff --git a/test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl b/test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl new file mode 100644 index 0000000000000..96a0b472d06c5 --- /dev/null +++ b/test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl @@ -0,0 +1,5 @@ +module ExtDep3 + +greet() = print("Hello World!") + +end # module ExtDep3 diff --git a/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml b/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml index 52542fc822094..27de581913b57 100644 --- a/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml +++ b/test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml @@ -2,7 +2,7 @@ julia_version = "1.10.0-DEV" manifest_format = "2.0" -project_hash = "d523b3401f72a1ed34b7b43749fd2655c6b78542" +project_hash = "4e196b07f2ee7adc48ac9d528d42b3cf3737c7a0" [[deps.ExtDep]] deps = ["SomePackage"] @@ -15,7 +15,13 @@ path = "../ExtDep2" uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" version = "0.1.0" +[[deps.ExtDep3]] +path = "../ExtDep3.jl" +uuid = "a5541f1e-a556-4fdc-af15-097880d743a1" +version = "0.1.0" + [[deps.HasExtensions]] +deps = ["ExtDep3"] path = "../HasExtensions.jl" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" version = "0.1.0" @@ -23,6 +29,7 @@ weakdeps = ["ExtDep", "ExtDep2"] [deps.HasExtensions.extensions] Extension = "ExtDep" + ExtensionDep = "ExtDep3" ExtensionFolder = ["ExtDep", "ExtDep2"] [[deps.SomePackage]] diff --git a/test/project/Extensions/HasDepWithExtensions.jl/Project.toml b/test/project/Extensions/HasDepWithExtensions.jl/Project.toml index 8f308a9fbee72..aa4956caada74 100644 --- a/test/project/Extensions/HasDepWithExtensions.jl/Project.toml +++ b/test/project/Extensions/HasDepWithExtensions.jl/Project.toml @@ -5,4 +5,5 @@ version = "0.1.0" [deps] ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" +ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1" HasExtensions = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" diff --git a/test/project/Extensions/HasExtensions.jl/Manifest.toml b/test/project/Extensions/HasExtensions.jl/Manifest.toml index 55f7958701a75..429c6598fc4f4 100644 --- a/test/project/Extensions/HasExtensions.jl/Manifest.toml +++ b/test/project/Extensions/HasExtensions.jl/Manifest.toml @@ -1,7 +1,10 @@ # This file is machine-generated - editing it directly is not advised -julia_version = "1.10.0-DEV" +julia_version = "1.12.0-DEV" manifest_format = "2.0" -project_hash = "c87947f1f1f070eea848950c304d668a112dec3d" +project_hash = "c0bb526b75939a74a6195ee4819e598918a22ad7" -[deps] +[[deps.ExtDep3]] +path = "../ExtDep3.jl" +uuid = "a5541f1e-a556-4fdc-af15-097880d743a1" +version = "0.1.0" diff --git a/test/project/Extensions/HasExtensions.jl/Project.toml b/test/project/Extensions/HasExtensions.jl/Project.toml index 72577de36d65d..a02f5662d602d 100644 --- a/test/project/Extensions/HasExtensions.jl/Project.toml +++ b/test/project/Extensions/HasExtensions.jl/Project.toml @@ -2,10 +2,14 @@ name = "HasExtensions" uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8" version = "0.1.0" +[deps] +ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1" + [weakdeps] ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c" ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d" [extensions] Extension = "ExtDep" +ExtensionDep = "ExtDep3" ExtensionFolder = ["ExtDep", "ExtDep2"] diff --git a/test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl b/test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl new file mode 100644 index 0000000000000..e2710d4d89bbb --- /dev/null +++ b/test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl @@ -0,0 +1,9 @@ +module ExtensionDep + +using HasExtensions, ExtDep3 + +function __init__() + HasExtensions.ext_dep_loaded = true +end + +end diff --git a/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl b/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl index dbfaeec4f8812..9d9785f87f790 100644 --- a/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl +++ b/test/project/Extensions/HasExtensions.jl/src/HasExtensions.jl @@ -6,5 +6,6 @@ foo(::HasExtensionsStruct) = 1 ext_loaded = false ext_folder_loaded = false +ext_dep_loaded = false end # module