Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve consistency of Dict types & construction #1968

Merged
merged 1 commit into from
Aug 19, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 16 additions & 13 deletions src/API.jl
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ using ..Artifacts: artifact_paths
include("generate.jl")

dependencies() = dependencies(Context())
function dependencies(ctx::Context)::Dict{UUID, PackageInfo}
function dependencies(ctx::Context)
pkgs = Operations.load_all_deps(ctx)
return Dict(pkg.uuid => Operations.package_info(ctx, pkg) for pkg in pkgs)
return Dict(pkg.uuid::UUID => Operations.package_info(ctx, pkg) for pkg in pkgs)
end
function dependencies(fn::Function, uuid::UUID)
dep = get(dependencies(), uuid, nothing)
Expand Down Expand Up @@ -329,6 +329,9 @@ function test(ctx::Context, pkgs::Vector{PackageSpec};
return
end

const UsageDict = Dict{String,DateTime}
const UsageByDepotDict = Dict{String,UsageDict}

"""
gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
Expand All @@ -351,11 +354,11 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
# versions that are only ever subsets of what we read out of them in the first place.

# Collect last known usage dates of manifest and artifacts toml files, split by depot
manifest_usage_by_depot = Dict{String, Dict{String, DateTime}}()
artifact_usage_by_depot = Dict{String, Dict{String, DateTime}}()
manifest_usage_by_depot = UsageByDepotDict()
artifact_usage_by_depot = UsageByDepotDict()

# Collect both last known usage dates, as well as parent projects for each scratch space
scratch_usage_by_depot = Dict{String, Dict{String, DateTime}}()
scratch_usage_by_depot = UsageByDepotDict()
scratch_parents_by_depot = Dict{String, Dict{String, Set{String}}}()

# Load manifest files from all depots
Expand All @@ -376,22 +379,22 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
# tracked manifest/artifact.toml), then merge the usage values from each file
# into the overall list across depots to create a single, coherent view across
# all depots.
usage = Dict{String, DateTime}()
usage = UsageDict()
reduce_usage!(joinpath(logdir(depot), "manifest_usage.toml")) do filename, info
# For Manifest usage, store only the last DateTime for each filename found
usage[filename] = max(get(usage, filename, DateTime(0)), DateTime(info["time"]))
end
manifest_usage_by_depot[depot] = usage

usage = Dict{String, DateTime}()
usage = UsageDict()
reduce_usage!(joinpath(logdir(depot), "artifact_usage.toml")) do filename, info
# For Artifact usage, store only the last DateTime for each filename found
usage[filename] = max(get(usage, filename, DateTime(0)), DateTime(info["time"]))
end
artifact_usage_by_depot[depot] = usage

# track last-used
usage = Dict{String, DateTime}()
usage = UsageDict()
parents = Dict{String, Set{String}}()
reduce_usage!(joinpath(logdir(depot), "scratch_usage.toml")) do filename, info
# For Artifact usage, store only the last DateTime for each filename found
Expand Down Expand Up @@ -576,7 +579,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
end

gc_time = now()
function merge_orphanages!(new_orphanage, paths, deletion_list, old_orphanage = Dict())
function merge_orphanages!(new_orphanage, paths, deletion_list, old_orphanage = UsageDict())
for path in paths
free_time = something(
get(old_orphanage, path, nothing),
Expand Down Expand Up @@ -625,7 +628,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
end
end
end
merge_orphanages!(Dict(), depot_orphaned_packages, packages_to_delete)
merge_orphanages!(UsageDict(), depot_orphaned_packages, packages_to_delete)
end


Expand Down Expand Up @@ -711,11 +714,11 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)

# Read in this depot's `orphaned.toml` file:
orphanage_file = joinpath(logdir(depot), "orphaned.toml")
new_orphanage = Dict{String, DateTime}()
new_orphanage = UsageDict()
old_orphanage = try
TOML.parse(String(read(orphanage_file)))
catch
Dict{String, DateTime}()
UsageDict()
end

# Update the package and artifact lists of things to delete, and
Expand Down Expand Up @@ -905,7 +908,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
Context!(ctx; kwargs...)
if !isfile(ctx.env.project_file) && isfile(ctx.env.manifest_file)
_manifest = Pkg.Types.read_manifest(ctx.env.manifest_file)
deps = Dict()
deps = Dict{String,String}()
for (uuid, pkg) in _manifest
if pkg.name in keys(deps)
# TODO, query what package to put in Project when in interactive mode?
Expand Down
25 changes: 13 additions & 12 deletions src/Artifacts.jl
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ within a particular package's UUID. In both cases, there are two different targ
the override: overriding to an on-disk location through an absolutet path, and
overriding to another artifact by its content-hash.
"""
const ARTIFACT_OVERRIDES = Ref{Union{Dict,Nothing}}(nothing)
const ARTIFACT_OVERRIDES = Ref{Union{Dict{Symbol,Any},Nothing}}(nothing)
function load_overrides(;force::Bool = false)
if ARTIFACT_OVERRIDES[] !== nothing && !force
return ARTIFACT_OVERRIDES[]
Expand All @@ -85,7 +85,7 @@ function load_overrides(;force::Bool = false)
# Overrides per UUID/bound name are intercepted upon Artifacts.toml load, and new
# entries within the "hash" overrides are generated on-the-fly. Thus, all redirects
# mechanisticly happen through the "hash" overrides.
overrides = Dict(
overrides = Dict{Symbol,Any}(
# Overrides by UUID
:UUID => Dict{Base.UUID,Dict{String,Union{String,SHA1}}}(),

Expand Down Expand Up @@ -149,18 +149,19 @@ function load_overrides(;force::Bool = false)
end

# If this mapping is itself a dict, store it as a set of UUID/artifact name overrides
if !haskey(overrides[:UUID], uuid)
overrides[:UUID][uuid] = Dict{String,Union{String,SHA1}}()
ovruuid = overrides[:UUID]::Dict{Base.UUID,Dict{String,Union{String,SHA1}}}
if !haskey(ovruuid, uuid)
ovruuid[uuid] = Dict{String,Union{String,SHA1}}()
end

# For each name in the mapping, update appropriately
for name in keys(mapping)
# If the mapping for this name is the empty string, un-override it
if mapping[name] == ""
delete!(overrides[:UUID][uuid], name)
delete!(ovruuid[uuid], name)
else
# Otherwise, store it!
overrides[:UUID][uuid][name] = mapping[name]
ovruuid[uuid][name] = mapping[name]
end
end
end
Expand Down Expand Up @@ -434,14 +435,14 @@ function unpack_platform(entry::Dict, name::String, artifacts_toml::String)::Uni
end
end

const os_map = IdDict{Type{<:Platform},String}(
Windows => "windows",
MacOS => "macos",
FreeBSD => "freebsd",
Linux => "linux",
)
function pack_platform!(meta::Dict, p::Platform)
@nospecialize meta p
os_map = Dict(
Windows => "windows",
MacOS => "macos",
FreeBSD => "freebsd",
Linux => "linux",
)
meta["os"] = os_map[typeof(p)]
meta["arch"] = string(arch(p))
if libc(p) !== nothing
Expand Down
8 changes: 4 additions & 4 deletions src/BinaryPlatforms.jl
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,7 @@ function platform_key_abi(machine::String)
:libgfortran4 => "(-libgfortran4)|(-gcc7)",
:libgfortran5 => "(-libgfortran5)|(-gcc8)",
)
libstdcxx_version_mapping = Dict(
libstdcxx_version_mapping = Dict{Symbol,String}(
:libstdcxx_nothing => "",
# This is sadly easier than parsing out the digit directly
(Symbol("libstdcxx$(idx)") => "-libstdcxx$(idx)" for idx in 18:26)...,
Expand Down Expand Up @@ -494,15 +494,15 @@ function platform_key_abi(machine::String)
# First, figure out what platform we're dealing with, then sub that off
# to the appropriate constructor. If a constructor runs into trouble,
# catch the error and return `UnknownPlatform()` here to be nicer to client code.
ctors = Dict(:darwin => MacOS, :mingw32 => Windows, :freebsd => FreeBSD, :linux => Linux)
ctors = Dict{Symbol,Type{<:Platform}}(:darwin => MacOS, :mingw32 => Windows, :freebsd => FreeBSD, :linux => Linux)
try
T = ctors[platform]
compiler_abi = CompilerABI(;
libgfortran_version=libgfortran_version,
libstdcxx_version=libstdcxx_version,
cxxstring_abi=cxxstring_abi
)
return T(arch, libc=libc, call_abi=call_abi, compiler_abi=compiler_abi)
return T(arch, libc=libc, call_abi=call_abi, compiler_abi=compiler_abi)::Platform
catch err
if isa(err, ArgumentError)
msg = " ($(err.msg))"
Expand Down Expand Up @@ -731,7 +731,7 @@ default_platkey = platform_key_abi(string(
))
function platform_key_abi()
global default_platkey
return default_platkey
return default_platkey::Platform
end

function platforms_match(a::Platform, b::Platform)
Expand Down
10 changes: 5 additions & 5 deletions src/Operations.jl
Original file line number Diff line number Diff line change
Expand Up @@ -413,14 +413,14 @@ function deps_graph(ctx::Context, uuid_to_name::Dict{UUID,String}, reqs::Resolve
uuids = collect(union(keys(reqs), keys(fixed), map(fx->keys(fx.requires), values(fixed))...))
seen = UUID[]

all_versions = Dict{UUID,Set{VersionNumber}}()
all_deps = Dict{UUID,Dict{VersionNumber,Dict{String,UUID}}}()
all_compat = Dict{UUID,Dict{VersionNumber,Dict{String,VersionSpec}}}()
all_versions = VersionsDict()
all_deps = DepsDict()
all_compat = CompatDict()

for (fp, fx) in fixed
all_versions[fp] = Set([fx.version])
all_deps[fp] = Dict(fx.version => Dict())
all_compat[fp] = Dict(fx.version => Dict())
all_deps[fp] = Dict(fx.version => valtype(DepsValDict)())
all_compat[fp] = Dict(fx.version => valtype(CompatValDict)())
end

while true
Expand Down
12 changes: 6 additions & 6 deletions src/REPLMode/REPLMode.jl
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ function OptionSpec(;name::String,
return OptionSpec(name, short_name, api, takes_arg)
end

function OptionSpecs(decs::Vector{OptionDeclaration})::Dict{String, OptionSpec}
specs = Dict()
function OptionSpecs(decs::Vector{OptionDeclaration})
specs = Dict{String, OptionSpec}()
for x in decs
opt_spec = OptionSpec(;x...)
@assert !haskey(specs, opt_spec.name) # don't overwrite
Expand Down Expand Up @@ -97,8 +97,8 @@ function CommandSpec(;name::Union{Nothing,String} = nothing,
OptionSpecs(option_spec), completions, description, help)
end

function CommandSpecs(declarations::Vector{CommandDeclaration})::Dict{String,CommandSpec}
specs = Dict()
function CommandSpecs(declarations::Vector{CommandDeclaration})
specs = Dict{String,CommandSpec}()
for dec in declarations
spec = CommandSpec(;dec...)
@assert !haskey(specs, spec.canonical_name) "duplicate spec entry"
Expand All @@ -111,8 +111,8 @@ function CommandSpecs(declarations::Vector{CommandDeclaration})::Dict{String,Com
return specs
end

function CompoundSpecs(compound_declarations)::Dict{String,Dict{String,CommandSpec}}
compound_specs = Dict()
function CompoundSpecs(compound_declarations)
compound_specs = Dict{String,Dict{String,CommandSpec}}()
for (name, command_declarations) in compound_declarations
specs = CommandSpecs(command_declarations)
@assert !haskey(compound_specs, name) "duplicate super spec entry"
Expand Down
28 changes: 13 additions & 15 deletions src/Resolve/graphtype.jl
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ mutable struct ResolveLog
journal = ResolveJournal()
init = ResolveLogEntry(journal, UUID0, "")
globals = ResolveLogEntry(journal, UUID0, "Global events:")
return new(init, globals, Dict(), journal, true, verbose, uuid_to_name)
return new(init, globals, Dict{UUID,ResolveLogEntry}(), journal, true, verbose, uuid_to_name)
end
end

Expand Down Expand Up @@ -235,49 +235,47 @@ mutable struct Graph
cavfld::Vector{FieldValue}

function Graph(
versions::Dict{UUID,Set{VersionNumber}},
deps::Dict{UUID,Dict{VersionNumber,Dict{String,UUID}}},
compat::Dict{UUID,Dict{VersionNumber,Dict{String,VersionSpec}}},
versions::VersionsDict,
deps::DepsDict,
compat::CompatDict,
uuid_to_name::Dict{UUID,String},
reqs::Requires,
fixed::Dict{UUID,Fixed},
verbose::Bool = false
)
# make sure all versions of all packages know about julia uuid
for (uuid,vnmap) in deps, vn in versions[uuid]
get!(Dict, vnmap, vn)["julia"] = uuid_julia
get!(valtype(vnmap), vnmap, vn)["julia"] = uuid_julia
end

# Tell the resolver about julia itself
fixed[uuid_julia] = Fixed(VERSION)
uuid_to_name[uuid_julia] = "julia"
versions[uuid_julia] = Set([VERSION])
deps[uuid_julia] = Dict(VERSION => Dict())
compat[uuid_julia] = Dict(VERSION => Dict())
deps[uuid_julia] = DepsValDict(VERSION => valtype(DepsValDict)())
compat[uuid_julia] = CompatValDict(VERSION => valtype(CompatValDict)())

extra_uuids = union(keys(reqs), keys(fixed), map(fx->keys(fx.requires), values(fixed))...)
extra_uuids = union(keys(reqs), union(keys(fixed), map(fx->keys(fx.requires), values(fixed))...))
extra_uuids ⊆ keys(versions) || error("unknown UUID found in reqs/fixed") # TODO?

# Type assert below due to https://github.com/JuliaLang/julia/issues/25918
data = GraphData(versions, uuid_to_name, verbose)::GraphData
data = GraphData(versions, uuid_to_name, verbose)
pkgs, np, spp, pdict, pvers, vdict, rlog = data.pkgs, data.np, data.spp, data.pdict, data.pvers, data.vdict, data.rlog

local extended_deps
let spp = spp # Due to https://github.com/JuliaLang/julia/issues/15276
# Type assert below to help inference
extended_deps = [Vector{Dict{Int,BitVector}}(undef, spp[p0]-1) for p0 = 1:np]::Vector{Vector{Dict{Int,BitVector}}}
extended_deps = [Vector{Dict{Int,BitVector}}(undef, spp[p0]-1) for p0 = 1:np]
end
for p0 = 1:np, v0 = 1:(spp[p0]-1)
n2u = Dict{String,UUID}()
vn = pvers[p0][v0]
vnmap = get(Dict, deps[pkgs[p0]], vn)
vnmap = get(Dict{String,UUID}, deps[pkgs[p0]], vn)
for (name, uuid) in vnmap
# check conflicts ??
n2u[name] = uuid
end
req = Dict{Int,VersionSpec}()
uuid = pkgs[p0]
vnmap = get(Dict, compat[pkgs[p0]], vn)
vnmap = get(Dict{String,VersionSpec}, compat[pkgs[p0]], vn)
for (name,vs) in vnmap
haskey(n2u, name) || error("Unknown package $name found in the compatibility requirements of $(pkgID(pkgs[p0], uuid_to_name))")
uuid = n2u[name]
Expand Down Expand Up @@ -882,7 +880,7 @@ function showlog(io::IO, rlog::ResolveLog, p::UUID; view::Symbol = :tree)
view ∈ [:plain, :tree] || throw(ArgumentError("the view argument should be `:plain` or `:tree`"))
entry = rlog.pool[p]
if view === :tree
_show(io, rlog, entry, _logindent, IdDict(entry=>true), true)
_show(io, rlog, entry, _logindent, IdDict{Any,Any}(entry=>true), true)
else
entries = ResolveLogEntry[entry]
function getentries(entry)
Expand Down
7 changes: 7 additions & 0 deletions src/Types.jl
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,16 @@ export UUID, SHA1, VersionRange, VersionSpec,
printpkgstyle, isurl,
projectfile_path, manifestfile_path,
RegistrySpec
export DepsValDict, CompatValDict, VersionsDict, DepsDict, CompatDict

include("versions.jl")

const DepsValDict = Dict{VersionNumber,Dict{String,UUID}}
const CompatValDict = Dict{VersionNumber,Dict{String,VersionSpec}}
const VersionsDict = Dict{UUID,Set{VersionNumber}}
const DepsDict = Dict{UUID,DepsValDict}
const CompatDict = Dict{UUID,CompatValDict}

const URL_regex = r"((file|git|ssh|http(s)?)|(git@[\w\-\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)?(/)?"x

#################
Expand Down
2 changes: 1 addition & 1 deletion test/artifacts.jl
Original file line number Diff line number Diff line change
Expand Up @@ -551,7 +551,7 @@ end
end

# Force Pkg to reload what it knows about artifact overrides
Pkg.Artifacts.load_overrides(;force=true)
@inferred Union{Nothing,Dict{Symbol,Any}} Pkg.Artifacts.load_overrides(;force=true)

# Verify that the hash-based override worked
@test artifact_path(baz_hash) == artifact_path(bar_hash)
Expand Down
4 changes: 2 additions & 2 deletions test/binaryplatforms.jl
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ using Test, Pkg.BinaryPlatforms
import Pkg.BinaryPlatforms: platform_name

# The platform we're running on
const platform = platform_key_abi()
const platform = @inferred Platform platform_key_abi()

@testset "PlatformNames" begin
# Ensure the platform type constructors are well behaved
Expand Down Expand Up @@ -114,7 +114,7 @@ const platform = platform_key_abi()

@testset "platform_key_abi parsing" begin
# Make sure the platform_key_abi() with explicit triplet works
@test platform_key_abi("x86_64-linux-gnu") == Linux(:x86_64)
@test @inferred(Platform, platform_key_abi("x86_64-linux-gnu")) == Linux(:x86_64)
@test platform_key_abi("x86_64-linux-musl") == Linux(:x86_64, libc=:musl)
@test platform_key_abi("i686-unknown-linux-gnu") == Linux(:i686)
@test platform_key_abi("x86_64-apple-darwin14") == MacOS()
Expand Down
2 changes: 1 addition & 1 deletion test/pkg.jl
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ temp_pkg_dir() do project_path
@testset "adding and upgrading different versions" begin
# VersionNumber
Pkg.add(PackageSpec(TEST_PKG.name, v"0.3"))
@test Pkg.dependencies()[TEST_PKG.uuid].version == v"0.3"
@test @inferred(Pkg.dependencies())[TEST_PKG.uuid].version == v"0.3"
Pkg.add(PackageSpec(TEST_PKG.name, v"0.3.1"))
@test Pkg.dependencies()[TEST_PKG.uuid].version == v"0.3.1"
Pkg.rm(TEST_PKG.name)
Expand Down
6 changes: 6 additions & 0 deletions test/repl.jl
Original file line number Diff line number Diff line change
Expand Up @@ -637,4 +637,10 @@ end
end end
end

@testset "Inference" begin
@inferred Pkg.REPLMode.OptionSpecs(Pkg.REPLMode.OptionDeclaration[])
@inferred Pkg.REPLMode.CommandSpecs(Pkg.REPLMode.CommandDeclaration[])
@inferred Pkg.REPLMode.CompoundSpecs(Pair{String,Vector{Pkg.REPLMode.CommandDeclaration}}[])
end

end # module
Loading