Skip to content

Commit

Permalink
allow extensions to trigger from packages in [deps] (#54009)
Browse files Browse the repository at this point in the history
There is a use case where you have a weak dependency (for one of your
extensions) that is misbehaving and you quickly want to try debug that
issue. A workflow that feels reasonable for this could be:

```
pkg> dev WeakDependency

# do some fixes in this dependency

julia> using Package, WeakDependency
# this loads the extension of Package triggered by loading WeakDependency

# check that things work ok now
```

This doesn't work right now for two reasons:

1. Doing the `dev WeakDependency` will add the dependency to `[deps]`
but not remove it from `[weakdeps]` which means you all of a sudden are
in the scenario described in
https://pkgdocs.julialang.org/v1/creating-packages/#Transition-from-normal-dependency-to-extension
which is not what is desired.
2. The extension will not actually load because you can right now only
trigger extensions from weak deps getting loaded, not from deps getting
loaded.

Point 1. is fixed by JuliaLang/Pkg.jl#3865
Point 2. is fixed by this PR.
  • Loading branch information
KristofferC authored Apr 10, 2024
1 parent e9a24d4 commit f46cb4c
Show file tree
Hide file tree
Showing 13 changed files with 85 additions and 38 deletions.
53 changes: 30 additions & 23 deletions base/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1388,13 +1388,12 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi
proj_pkg = project_file_name_uuid(implicit_project_file, pkg.name)
if pkg == proj_pkg
d_proj = parsed_toml(implicit_project_file)
weakdeps = get(d_proj, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}}
extensions = get(d_proj, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
extensions === nothing && return
weakdeps === nothing && return
if weakdeps isa Dict{String, Any}
return _insert_extension_triggers(pkg, extensions, weakdeps)
end
weakdeps = get(Dict{String, Any}, d_proj, "weakdeps")::Dict{String,Any}
deps = get(Dict{String, Any}, d_proj, "deps")::Dict{String,Any}
total_deps = merge(weakdeps, deps)
return _insert_extension_triggers(pkg, extensions, total_deps)
end

# Now look in manifest
Expand All @@ -1409,35 +1408,43 @@ function insert_extension_triggers(env::String, pkg::PkgId)::Union{Nothing,Missi
uuid = get(entry, "uuid", nothing)::Union{String, Nothing}
uuid === nothing && continue
if UUID(uuid) == pkg.uuid
weakdeps = get(entry, "weakdeps", nothing)::Union{Nothing, Vector{String}, Dict{String,Any}}
extensions = get(entry, "extensions", nothing)::Union{Nothing, Dict{String, Any}}
extensions === nothing && return
weakdeps === nothing && return
if weakdeps isa Dict{String, Any}
return _insert_extension_triggers(pkg, extensions, weakdeps)
weakdeps = get(Dict{String, Any}, entry, "weakdeps")::Union{Vector{String}, Dict{String,Any}}
deps = get(Dict{String, Any}, entry, "deps")::Union{Vector{String}, Dict{String,Any}}

function expand_deps_list(deps′::Vector{String})
deps′_expanded = Dict{String, Any}()
for (dep_name, entries) in d
dep_name in deps′ || continue
entries::Vector{Any}
if length(entries) != 1
error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))")
end
entry = first(entries)::Dict{String, Any}
uuid = entry["uuid"]::String
deps′_expanded[dep_name] = uuid
end
return deps′_expanded
end

d_weakdeps = Dict{String, Any}()
for (dep_name, entries) in d
dep_name in weakdeps || continue
entries::Vector{Any}
if length(entries) != 1
error("expected a single entry for $(repr(dep_name)) in $(repr(project_file))")
end
entry = first(entries)::Dict{String, Any}
uuid = entry["uuid"]::String
d_weakdeps[dep_name] = uuid
if weakdeps isa Vector{String}
weakdeps = expand_deps_list(weakdeps)
end
if deps isa Vector{String}
deps = expand_deps_list(deps)
end
@assert length(d_weakdeps) == length(weakdeps)
return _insert_extension_triggers(pkg, extensions, d_weakdeps)

total_deps = merge(weakdeps, deps)
return _insert_extension_triggers(pkg, extensions, total_deps)
end
end
end
end
return nothing
end

function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, weakdeps::Dict{String, Any})
function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}, totaldeps::Dict{String, Any})
for (ext, triggers) in extensions
triggers = triggers::Union{String, Vector{String}}
triggers isa String && (triggers = [triggers])
Expand All @@ -1451,7 +1458,7 @@ function _insert_extension_triggers(parent::PkgId, extensions::Dict{String, Any}
push!(trigger1, gid)
for trigger in triggers
# TODO: Better error message if this lookup fails?
uuid_trigger = UUID(weakdeps[trigger]::String)
uuid_trigger = UUID(totaldeps[trigger]::String)
trigger_id = PkgId(uuid_trigger, trigger)
if !haskey(explicit_loaded_modules, trigger_id) || haskey(package_locks, trigger_id)
trigger1 = get!(Vector{ExtensionId}, EXT_DORMITORY, trigger_id)
Expand Down
2 changes: 1 addition & 1 deletion base/precompilation.jl
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ function precompilepkgs(pkgs::Vector{String}=String[];
all_extdeps_available = true
for extdep_uuid in extdep_uuids
extdep_name = env.names[extdep_uuid]
if extdep_uuid in keys(env.deps) || Base.in_sysimage(Base.PkgId(extdep_uuid, extdep_name))
if extdep_uuid in keys(env.deps)
push!(ext_deps, Base.PkgId(extdep_uuid, extdep_name))
else
all_extdeps_available = false
Expand Down
12 changes: 6 additions & 6 deletions doc/src/manual/code-loading.md
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,7 @@ Since the primary environment is typically the environment of a project you're w

### [Package Extensions](@id man-extensions)

A package "extension" is a module that is automatically loaded when a specified set of other packages (its "extension dependencies") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The extension dependencies of an extension are a subset of those packages listed under the `[weakdeps]` section of the project file. Those packages can have compat entries like other packages.
A package "extension" is a module that is automatically loaded when a specified set of other packages (its "triggers") are loaded in the current Julia session. Extensions are defined under the `[extensions]` section in the project file. The triggers of an extension are a subset of those packages listed under the `[weakdeps]` (and possibly, but uncommonly the `[deps]`) section of the project file. Those packages can have compat entries like other packages.

```toml
name = "MyPackage"
Expand All @@ -371,27 +371,27 @@ FooExt = "ExtDep"
```

The keys under `extensions` are the names of the extensions.
They are loaded when all the packages on the right hand side (the extension dependencies) of that extension are loaded.
If an extension only has one extension dependency the list of extension dependencies can be written as just a string for brevity.
They are loaded when all the packages on the right hand side (the triggers) of that extension are loaded.
If an extension only has one trigger the list of triggers can be written as just a string for brevity.
The location for the entry point of the extension is either in `ext/FooExt.jl` or `ext/FooExt/FooExt.jl` for
extension `FooExt`.
The content of an extension is often structured as:

```
module FooExt
# Load main package and extension dependencies
# Load main package and triggers
using MyPackage, ExtDep
# Extend functionality in main package with types from the extension dependencies
# Extend functionality in main package with types from the triggers
MyPackage.func(x::ExtDep.SomeStruct) = ...
end
```

When a package with extensions is added to an environment, the `weakdeps` and `extensions` sections
are stored in the manifest file in the section for that package. The dependency lookup rules for
a package are the same as for its "parent" except that the listed extension dependencies are also considered as
a package are the same as for its "parent" except that the listed triggers are also considered as
dependencies.

### [Workspaces](@id workspaces)
Expand Down
6 changes: 5 additions & 1 deletion test/loading.jl
Original file line number Diff line number Diff line change
Expand Up @@ -1057,7 +1057,9 @@ end
$ew HasDepWithExtensions.do_something() || error("do_something errored")
using ExtDep2
$ew using ExtDep2
$ew HasExtensions.ext_folder_loaded || error("ext_folder_loaded not set")
using ExtDep3
$ew using ExtDep3
$ew HasExtensions.ext_dep_loaded || error("ext_dep_loaded not set")
end
"""
return `$(Base.julia_cmd()) $compile --startup-file=no -e $cmd`
Expand Down Expand Up @@ -1105,6 +1107,8 @@ end
Base.get_extension(HasExtensions, :Extension) isa Module || error("expected extension to load")
using ExtDep2
Base.get_extension(HasExtensions, :ExtensionFolder) isa Module || error("expected extension to load")
using ExtDep3
Base.get_extension(HasExtensions, :ExtensionDep) isa Module || error("expected extension to load")
end
"""
for compile in (`--compiled-modules=no`, ``)
Expand Down
6 changes: 4 additions & 2 deletions test/project/Extensions/EnvWithHasExtensions/Manifest.toml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# This file is machine-generated - editing it directly is not advised

julia_version = "1.9.0-beta4"
julia_version = "1.12.0-DEV"
manifest_format = "2.0"
project_hash = "caa716752e6dff3d77c3de929ebbb5d2024d04ef"
project_hash = "a4c480cfa7da9610333d5c42623bf746bd286c5f"

[[deps.ExtDep]]
deps = ["SomePackage"]
Expand All @@ -18,10 +18,12 @@ version = "0.1.0"
[deps.HasExtensions.extensions]
Extension = "ExtDep"
ExtensionFolder = ["ExtDep", "ExtDep2"]
LinearAlgebraExt = "LinearAlgebra"

[deps.HasExtensions.weakdeps]
ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[[deps.SomePackage]]
path = "../SomePackage"
Expand Down
4 changes: 4 additions & 0 deletions test/project/Extensions/ExtDep3.jl/Project.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
name = "ExtDep3"
uuid = "a5541f1e-a556-4fdc-af15-097880d743a1"
version = "0.1.0"
authors = ["Kristoffer <kcarlsson89@gmail.com>"]
5 changes: 5 additions & 0 deletions test/project/Extensions/ExtDep3.jl/src/ExtDep3.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
module ExtDep3

greet() = print("Hello World!")

end # module ExtDep3
11 changes: 9 additions & 2 deletions test/project/Extensions/HasDepWithExtensions.jl/Manifest.toml
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
# This file is machine-generated - editing it directly is not advised

julia_version = "1.10.0"
julia_version = "1.12.0-DEV"
manifest_format = "2.0"
project_hash = "d523b3401f72a1ed34b7b43749fd2655c6b78542"
project_hash = "4e196b07f2ee7adc48ac9d528d42b3cf3737c7a0"

[[deps.ExtDep]]
deps = ["SomePackage"]
Expand All @@ -15,13 +15,20 @@ path = "../ExtDep2"
uuid = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
version = "0.1.0"

[[deps.ExtDep3]]
path = "../ExtDep3.jl"
uuid = "a5541f1e-a556-4fdc-af15-097880d743a1"
version = "0.1.0"

[[deps.HasExtensions]]
deps = ["ExtDep3"]
path = "../HasExtensions.jl"
uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"
version = "0.1.0"

[deps.HasExtensions.extensions]
Extension = "ExtDep"
ExtensionDep = "ExtDep3"
ExtensionFolder = ["ExtDep", "ExtDep2"]
LinearAlgebraExt = "LinearAlgebra"

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,5 @@ version = "0.1.0"
[deps]
ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1"
HasExtensions = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"
9 changes: 6 additions & 3 deletions test/project/Extensions/HasExtensions.jl/Manifest.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,10 @@
# This file is machine-generated - editing it directly is not advised

julia_version = "1.10.0-DEV"
julia_version = "1.12.0-DEV"
manifest_format = "2.0"
project_hash = "c87947f1f1f070eea848950c304d668a112dec3d"
project_hash = "c0bb526b75939a74a6195ee4819e598918a22ad7"

[deps]
[[deps.ExtDep3]]
path = "../ExtDep3.jl"
uuid = "a5541f1e-a556-4fdc-af15-097880d743a1"
version = "0.1.0"
4 changes: 4 additions & 0 deletions test/project/Extensions/HasExtensions.jl/Project.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,16 @@ name = "HasExtensions"
uuid = "4d3288b3-3afc-4bb6-85f3-489fffe514c8"
version = "0.1.0"

[deps]
ExtDep3 = "a5541f1e-a556-4fdc-af15-097880d743a1"

[weakdeps]
ExtDep = "fa069be4-f60b-4d4c-8b95-f8008775090c"
ExtDep2 = "55982ee5-2ad5-4c40-8cfe-5e9e1b01500d"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"

[extensions]
Extension = "ExtDep"
ExtensionDep = "ExtDep3"
ExtensionFolder = ["ExtDep", "ExtDep2"]
LinearAlgebraExt = "LinearAlgebra"
9 changes: 9 additions & 0 deletions test/project/Extensions/HasExtensions.jl/ext/ExtensionDep.jl
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
module ExtensionDep

using HasExtensions, ExtDep3

function __init__()
HasExtensions.ext_dep_loaded = true
end

end
Original file line number Diff line number Diff line change
Expand Up @@ -6,5 +6,6 @@ foo(::HasExtensionsStruct) = 1

ext_loaded = false
ext_folder_loaded = false
ext_dep_loaded = false

end # module

0 comments on commit f46cb4c

Please sign in to comment.