Skip to content

Commit

Permalink
Improve consistency of Dict types & construction
Browse files Browse the repository at this point in the history
  • Loading branch information
timholy committed Aug 18, 2020
1 parent 4a4621b commit 8174ecd
Show file tree
Hide file tree
Showing 12 changed files with 68 additions and 54 deletions.
29 changes: 16 additions & 13 deletions src/API.jl
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,9 @@ using ..Artifacts: artifact_paths
include("generate.jl")

dependencies() = dependencies(Context())
function dependencies(ctx::Context)::Dict{UUID, PackageInfo}
function dependencies(ctx::Context)
pkgs = Operations.load_all_deps(ctx)
return Dict(pkg.uuid => Operations.package_info(ctx, pkg) for pkg in pkgs)
return Dict(pkg.uuid::UUID => Operations.package_info(ctx, pkg) for pkg in pkgs)
end
function dependencies(fn::Function, uuid::UUID)
dep = get(dependencies(), uuid, nothing)
Expand Down Expand Up @@ -329,6 +329,9 @@ function test(ctx::Context, pkgs::Vector{PackageSpec};
return
end

const UsageDict = Dict{String,DateTime}
const UsageByDepotDict = Dict{String,UsageDict}

"""
gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
Expand All @@ -351,11 +354,11 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
# versions that are only ever subsets of what we read out of them in the first place.

# Collect last known usage dates of manifest and artifacts toml files, split by depot
manifest_usage_by_depot = Dict{String, Dict{String, DateTime}}()
artifact_usage_by_depot = Dict{String, Dict{String, DateTime}}()
manifest_usage_by_depot = UsageByDepotDict()
artifact_usage_by_depot = UsageByDepotDict()

# Collect both last known usage dates, as well as parent projects for each scratch space
scratch_usage_by_depot = Dict{String, Dict{String, DateTime}}()
scratch_usage_by_depot = UsageByDepotDict()
scratch_parents_by_depot = Dict{String, Dict{String, Set{String}}}()

# Load manifest files from all depots
Expand All @@ -376,22 +379,22 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
# tracked manifest/artifact.toml), then merge the usage values from each file
# into the overall list across depots to create a single, coherent view across
# all depots.
usage = Dict{String, DateTime}()
usage = UsageDict()
reduce_usage!(joinpath(logdir(depot), "manifest_usage.toml")) do filename, info
# For Manifest usage, store only the last DateTime for each filename found
usage[filename] = max(get(usage, filename, DateTime(0)), DateTime(info["time"]))
end
manifest_usage_by_depot[depot] = usage

usage = Dict{String, DateTime}()
usage = UsageDict()
reduce_usage!(joinpath(logdir(depot), "artifact_usage.toml")) do filename, info
# For Artifact usage, store only the last DateTime for each filename found
usage[filename] = max(get(usage, filename, DateTime(0)), DateTime(info["time"]))
end
artifact_usage_by_depot[depot] = usage

# track last-used
usage = Dict{String, DateTime}()
usage = UsageDict()
parents = Dict{String, Set{String}}()
reduce_usage!(joinpath(logdir(depot), "scratch_usage.toml")) do filename, info
# For Artifact usage, store only the last DateTime for each filename found
Expand Down Expand Up @@ -576,7 +579,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
end

gc_time = now()
function merge_orphanages!(new_orphanage, paths, deletion_list, old_orphanage = Dict())
function merge_orphanages!(new_orphanage, paths, deletion_list, old_orphanage = UsageDict())
for path in paths
free_time = something(
get(old_orphanage, path, nothing),
Expand Down Expand Up @@ -625,7 +628,7 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)
end
end
end
merge_orphanages!(Dict(), depot_orphaned_packages, packages_to_delete)
merge_orphanages!(UsageDict(), depot_orphaned_packages, packages_to_delete)
end


Expand Down Expand Up @@ -711,11 +714,11 @@ function gc(ctx::Context=Context(); collect_delay::Period=Day(7), kwargs...)

# Read in this depot's `orphaned.toml` file:
orphanage_file = joinpath(logdir(depot), "orphaned.toml")
new_orphanage = Dict{String, DateTime}()
new_orphanage = UsageDict()
old_orphanage = try
TOML.parse(String(read(orphanage_file)))
catch
Dict{String, DateTime}()
UsageDict()
end

# Update the package and artifact lists of things to delete, and
Expand Down Expand Up @@ -905,7 +908,7 @@ function instantiate(ctx::Context; manifest::Union{Bool, Nothing}=nothing,
Context!(ctx; kwargs...)
if !isfile(ctx.env.project_file) && isfile(ctx.env.manifest_file)
_manifest = Pkg.Types.read_manifest(ctx.env.manifest_file)
deps = Dict()
deps = Dict{String,String}()
for (uuid, pkg) in _manifest
if pkg.name in keys(deps)
# TODO, query what package to put in Project when in interactive mode?
Expand Down
14 changes: 7 additions & 7 deletions src/Artifacts.jl
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ function load_overrides(;force::Bool = false)
# Overrides per UUID/bound name are intercepted upon Artifacts.toml load, and new
# entries within the "hash" overrides are generated on-the-fly. Thus, all redirects
# mechanisticly happen through the "hash" overrides.
overrides = Dict(
overrides = Dict{Symbol,Any}(
# Overrides by UUID
:UUID => Dict{Base.UUID,Dict{String,Union{String,SHA1}}}(),

Expand Down Expand Up @@ -434,14 +434,14 @@ function unpack_platform(entry::Dict, name::String, artifacts_toml::String)::Uni
end
end

const os_map = IdDict{Type{<:Platform},String}(
Windows => "windows",
MacOS => "macos",
FreeBSD => "freebsd",
Linux => "linux",
)
function pack_platform!(meta::Dict, p::Platform)
@nospecialize meta p
os_map = Dict(
Windows => "windows",
MacOS => "macos",
FreeBSD => "freebsd",
Linux => "linux",
)
meta["os"] = os_map[typeof(p)]
meta["arch"] = string(arch(p))
if libc(p) !== nothing
Expand Down
6 changes: 3 additions & 3 deletions src/BinaryPlatforms.jl
Original file line number Diff line number Diff line change
Expand Up @@ -430,7 +430,7 @@ function platform_key_abi(machine::AbstractString)
:libgfortran4 => "(-libgfortran4)|(-gcc7)",
:libgfortran5 => "(-libgfortran5)|(-gcc8)",
)
libstdcxx_version_mapping = Dict(
libstdcxx_version_mapping = Dict{Symbol,String}(
:libstdcxx_nothing => "",
# This is sadly easier than parsing out the digit directly
(Symbol("libstdcxx$(idx)") => "-libstdcxx$(idx)" for idx in 18:26)...,
Expand Down Expand Up @@ -494,15 +494,15 @@ function platform_key_abi(machine::AbstractString)
# First, figure out what platform we're dealing with, then sub that off
# to the appropriate constructor. If a constructor runs into trouble,
# catch the error and return `UnknownPlatform()` here to be nicer to client code.
ctors = Dict(:darwin => MacOS, :mingw32 => Windows, :freebsd => FreeBSD, :linux => Linux)
ctors = Dict{Symbol,Type{<:Platform}}(:darwin => MacOS, :mingw32 => Windows, :freebsd => FreeBSD, :linux => Linux)
try
T = ctors[platform]
compiler_abi = CompilerABI(;
libgfortran_version=libgfortran_version,
libstdcxx_version=libstdcxx_version,
cxxstring_abi=cxxstring_abi
)
return T(arch, libc=libc, call_abi=call_abi, compiler_abi=compiler_abi)
return T(arch, libc=libc, call_abi=call_abi, compiler_abi=compiler_abi)::Platform
catch err
if isa(err, ArgumentError)
msg = " ($(err.msg))"
Expand Down
10 changes: 5 additions & 5 deletions src/Operations.jl
Original file line number Diff line number Diff line change
Expand Up @@ -406,14 +406,14 @@ function deps_graph(ctx::Context, uuid_to_name::Dict{UUID,String}, reqs::Resolve
uuids = collect(union(keys(reqs), keys(fixed), map(fx->keys(fx.requires), values(fixed))...))
seen = UUID[]

all_versions = Dict{UUID,Set{VersionNumber}}()
all_deps = Dict{UUID,Dict{VersionNumber,Dict{String,UUID}}}()
all_compat = Dict{UUID,Dict{VersionNumber,Dict{String,VersionSpec}}}()
all_versions = VersionsDict()
all_deps = DepsDict()
all_compat = CompatDict()

for (fp, fx) in fixed
all_versions[fp] = Set([fx.version])
all_deps[fp] = Dict(fx.version => Dict())
all_compat[fp] = Dict(fx.version => Dict())
all_deps[fp] = Dict(fx.version => valtype(DepsValDict)())
all_compat[fp] = Dict(fx.version => valtype(CompatValDict)())
end

while true
Expand Down
12 changes: 6 additions & 6 deletions src/REPLMode/REPLMode.jl
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ function OptionSpec(;name::String,
return OptionSpec(name, short_name, api, takes_arg)
end

function OptionSpecs(decs::Vector{OptionDeclaration})::Dict{String, OptionSpec}
specs = Dict()
function OptionSpecs(decs::Vector{OptionDeclaration})
specs = Dict{String, OptionSpec}()
for x in decs
opt_spec = OptionSpec(;x...)
@assert !haskey(specs, opt_spec.name) # don't overwrite
Expand Down Expand Up @@ -97,8 +97,8 @@ function CommandSpec(;name::Union{Nothing,String} = nothing,
OptionSpecs(option_spec), completions, description, help)
end

function CommandSpecs(declarations::Vector{CommandDeclaration})::Dict{String,CommandSpec}
specs = Dict()
function CommandSpecs(declarations::Vector{CommandDeclaration})
specs = Dict{String,CommandSpec}()
for dec in declarations
spec = CommandSpec(;dec...)
@assert !haskey(specs, spec.canonical_name) "duplicate spec entry"
Expand All @@ -111,8 +111,8 @@ function CommandSpecs(declarations::Vector{CommandDeclaration})::Dict{String,Com
return specs
end

function CompoundSpecs(compound_declarations)::Dict{String,Dict{String,CommandSpec}}
compound_specs = Dict()
function CompoundSpecs(compound_declarations)
compound_specs = Dict{String,Dict{String,CommandSpec}}()
for (name, command_declarations) in compound_declarations
specs = CommandSpecs(command_declarations)
@assert !haskey(compound_specs, name) "duplicate super spec entry"
Expand Down
28 changes: 13 additions & 15 deletions src/Resolve/graphtype.jl
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ mutable struct ResolveLog
journal = ResolveJournal()
init = ResolveLogEntry(journal, UUID0, "")
globals = ResolveLogEntry(journal, UUID0, "Global events:")
return new(init, globals, Dict(), journal, true, verbose, uuid_to_name)
return new(init, globals, Dict{UUID,ResolveLogEntry}(), journal, true, verbose, uuid_to_name)
end
end

Expand Down Expand Up @@ -235,49 +235,47 @@ mutable struct Graph
cavfld::Vector{FieldValue}

function Graph(
versions::Dict{UUID,Set{VersionNumber}},
deps::Dict{UUID,Dict{VersionNumber,Dict{String,UUID}}},
compat::Dict{UUID,Dict{VersionNumber,Dict{String,VersionSpec}}},
versions::VersionsDict,
deps::DepsDict,
compat::CompatDict,
uuid_to_name::Dict{UUID,String},
reqs::Requires,
fixed::Dict{UUID,Fixed},
verbose::Bool = false
)
# make sure all versions of all packages know about julia uuid
for (uuid,vnmap) in deps, vn in versions[uuid]
get!(Dict, vnmap, vn)["julia"] = uuid_julia
get!(valtype(vnmap), vnmap, vn)["julia"] = uuid_julia
end

# Tell the resolver about julia itself
fixed[uuid_julia] = Fixed(VERSION)
uuid_to_name[uuid_julia] = "julia"
versions[uuid_julia] = Set([VERSION])
deps[uuid_julia] = Dict(VERSION => Dict())
compat[uuid_julia] = Dict(VERSION => Dict())
deps[uuid_julia] = DepsValDict(VERSION => valtype(DepsValDict)())
compat[uuid_julia] = CompatValDict(VERSION => valtype(CompatValDict)())

extra_uuids = union(keys(reqs), keys(fixed), map(fx->keys(fx.requires), values(fixed))...)
extra_uuids = union(keys(reqs), union(keys(fixed), map(fx->keys(fx.requires), values(fixed))...))
extra_uuids keys(versions) || error("unknown UUID found in reqs/fixed") # TODO?

# Type assert below due to https://github.com/JuliaLang/julia/issues/25918
data = GraphData(versions, uuid_to_name, verbose)::GraphData
data = GraphData(versions, uuid_to_name, verbose)
pkgs, np, spp, pdict, pvers, vdict, rlog = data.pkgs, data.np, data.spp, data.pdict, data.pvers, data.vdict, data.rlog

local extended_deps
let spp = spp # Due to https://github.com/JuliaLang/julia/issues/15276
# Type assert below to help inference
extended_deps = [Vector{Dict{Int,BitVector}}(undef, spp[p0]-1) for p0 = 1:np]::Vector{Vector{Dict{Int,BitVector}}}
extended_deps = [Vector{Dict{Int,BitVector}}(undef, spp[p0]-1) for p0 = 1:np]
end
for p0 = 1:np, v0 = 1:(spp[p0]-1)
n2u = Dict{String,UUID}()
vn = pvers[p0][v0]
vnmap = get(Dict, deps[pkgs[p0]], vn)
vnmap = get(Dict{String,UUID}, deps[pkgs[p0]], vn)
for (name, uuid) in vnmap
# check conflicts ??
n2u[name] = uuid
end
req = Dict{Int,VersionSpec}()
uuid = pkgs[p0]
vnmap = get(Dict, compat[pkgs[p0]], vn)
vnmap = get(Dict{String,VersionSpec}, compat[pkgs[p0]], vn)
for (name,vs) in vnmap
haskey(n2u, name) || error("Unknown package $name found in the compatibility requirements of $(pkgID(pkgs[p0], uuid_to_name))")
uuid = n2u[name]
Expand Down Expand Up @@ -882,7 +880,7 @@ function showlog(io::IO, rlog::ResolveLog, p::UUID; view::Symbol = :tree)
view [:plain, :tree] || throw(ArgumentError("the view argument should be `:plain` or `:tree`"))
entry = rlog.pool[p]
if view === :tree
_show(io, rlog, entry, _logindent, IdDict(entry=>true), true)
_show(io, rlog, entry, _logindent, IdDict{Any,Any}(entry=>true), true)
else
entries = ResolveLogEntry[entry]
function getentries(entry)
Expand Down
7 changes: 7 additions & 0 deletions src/Types.jl
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,16 @@ export UUID, SHA1, VersionRange, VersionSpec,
printpkgstyle, isurl,
projectfile_path, manifestfile_path,
RegistrySpec
export DepsValDict, CompatValDict, VersionsDict, DepsDict, CompatDict

include("versions.jl")

const DepsValDict = Dict{VersionNumber,Dict{String,UUID}}
const CompatValDict = Dict{VersionNumber,Dict{String,VersionSpec}}
const VersionsDict = Dict{UUID,Set{VersionNumber}}
const DepsDict = Dict{UUID,DepsValDict}
const CompatDict = Dict{UUID,CompatValDict}

const URL_regex = r"((file|git|ssh|http(s)?)|(git@[\w\-\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git)?(/)?"x

#################
Expand Down
2 changes: 1 addition & 1 deletion test/artifacts.jl
Original file line number Diff line number Diff line change
Expand Up @@ -551,7 +551,7 @@ end
end

# Force Pkg to reload what it knows about artifact overrides
Pkg.Artifacts.load_overrides(;force=true)
@inferred(Pkg.Artifacts.load_overrides(;force=true))

# Verify that the hash-based override worked
@test artifact_path(baz_hash) == artifact_path(bar_hash)
Expand Down
4 changes: 2 additions & 2 deletions test/binaryplatforms.jl
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ using Test, Pkg.BinaryPlatforms
import Pkg.BinaryPlatforms: platform_name

# The platform we're running on
const platform = platform_key_abi()
const platform = @inferred Platform platform_key_abi()

@testset "PlatformNames" begin
# Ensure the platform type constructors are well behaved
Expand Down Expand Up @@ -114,7 +114,7 @@ const platform = platform_key_abi()

@testset "platform_key_abi parsing" begin
# Make sure the platform_key_abi() with explicit triplet works
@test platform_key_abi("x86_64-linux-gnu") == Linux(:x86_64)
@test @inferred(Platform, platform_key_abi("x86_64-linux-gnu")) == Linux(:x86_64)
@test platform_key_abi("x86_64-linux-musl") == Linux(:x86_64, libc=:musl)
@test platform_key_abi("i686-unknown-linux-gnu") == Linux(:i686)
@test platform_key_abi("x86_64-apple-darwin14") == MacOS()
Expand Down
2 changes: 1 addition & 1 deletion test/pkg.jl
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@ temp_pkg_dir() do project_path
@testset "adding and upgrading different versions" begin
# VersionNumber
Pkg.add(PackageSpec(TEST_PKG.name, v"0.3"))
@test Pkg.dependencies()[TEST_PKG.uuid].version == v"0.3"
@test @inferred(Pkg.dependencies())[TEST_PKG.uuid].version == v"0.3"
Pkg.add(PackageSpec(TEST_PKG.name, v"0.3.1"))
@test Pkg.dependencies()[TEST_PKG.uuid].version == v"0.3.1"
Pkg.rm(TEST_PKG.name)
Expand Down
6 changes: 6 additions & 0 deletions test/repl.jl
Original file line number Diff line number Diff line change
Expand Up @@ -637,4 +637,10 @@ end
end end
end

@testset "Inference" begin
@test @inferred Pkg.REPLMode.OptionSpecs(Pkg.REPLMode.OptionDeclaration[])
@test @inferred Pkg.REPLMode.CommandSpecs(Pkg.REPLMode.CommandDeclaration[])
@test @inferred Pkg.REPLMode.CompoundSpecs(Pair{String,Vector{Pkg.REPLMode.CommandDeclaration}}[])
end

end # module
2 changes: 1 addition & 1 deletion test/resolve_utils.jl
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ function graph_from_data(deps_data)
uuid(p) = storeuuid(p, uuid_to_name)
fixed = Dict{UUID,Fixed}()
all_versions = Dict{UUID,Set{VersionNumber}}()
all_deps = Dict{UUID,Dict{VersionNumber,Dict{String,UUID}}}()
all_deps = DepsDict()
all_compat = Dict{UUID,Dict{VersionNumber,Dict{String,VersionSpec}}}()

deps = Dict{String,Dict{VersionNumber,Dict{String,VersionSpec}}}()
Expand Down

0 comments on commit 8174ecd

Please sign in to comment.