Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

add an LRU cache to precompile files #32651

Merged
merged 1 commit into from
Aug 16, 2019
Merged

add an LRU cache to precompile files #32651

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
47 changes: 34 additions & 13 deletions base/loading.jl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -649,15 +649,22 @@ end
cache_file_entry(pkg::PkgId) = joinpath(
"compiled",
"v$(VERSION.major).$(VERSION.minor)",
pkg.uuid === nothing ? "$(pkg.name).ji" : joinpath(pkg.name, "$(package_slug(pkg.uuid)).ji")
)
pkg.uuid === nothing ? "" : pkg.name),
pkg.uuid === nothing ? pkg.name : package_slug(pkg.uuid)

function find_all_in_cache_path(pkg::PkgId)
paths = String[]
entry = cache_file_entry(pkg)
for depot in DEPOT_PATH
path = joinpath(depot, entry)
isfile_casesensitive(path) && push!(paths, path)
entrypath, entryfile = cache_file_entry(pkg)
for path in joinpath.(DEPOT_PATH, entrypath)
isdir(path) || continue
for file in readdir(path)
if !((pkg.uuid === nothing && file == entryfile * ".ji") ||
(pkg.uuid !== nothing && startswith(file, entryfile * "_")))
continue
end
filepath = joinpath(path, file)
isfile_casesensitive(filepath) && push!(paths, filepath)
end
end
return paths
end
Expand Down Expand Up @@ -746,6 +753,10 @@ function _require_search_from_serialized(pkg::PkgId, sourcepath::String)
if staledeps === true
continue
end
try
touch(path_to_try) # update timestamp of precompilation file
catch # file might be read-only and then we fail to update timestamp, which is fine
end
# finish loading module graph into staledeps
for i in 1:length(staledeps)
dep = staledeps[i]
Expand Down Expand Up @@ -1222,11 +1233,25 @@ function compilecache(pkg::PkgId)
path === nothing && throw(ArgumentError("$pkg not found during precompilation"))
return compilecache(pkg, path)
end

const MAX_NUM_PRECOMPILE_FILES = 10

function compilecache(pkg::PkgId, path::String)
# decide where to put the resulting cache file
cachefile = abspath(DEPOT_PATH[1], cache_file_entry(pkg))
cachepath = dirname(cachefile)
entrypath, entryfile = cache_file_entry(pkg)
cachepath = joinpath(DEPOT_PATH[1], entrypath)
isdir(cachepath) || mkpath(cachepath)
if pkg.uuid === nothing
cachefile = abspath(cachepath, entryfile) * ".ji"
else
candidates = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath))
if length(candidates) >= MAX_NUM_PRECOMPILE_FILES
idx = findmin(mtime.(joinpath.(cachepath, candidates)))[2]
rm(joinpath(cachepath, candidates[idx]))
end
project_precompile_slug = slug(_crc32c(something(Base.active_project(), "")), 5)
cachefile = abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji"))
end
# build up the list of modules that we want the precompile process to preserve
concrete_deps = copy(_concrete_dependencies)
for (key, mod) in loaded_modules
Expand All @@ -1236,11 +1261,7 @@ function compilecache(pkg::PkgId, path::String)
end
# run the expression and cache the result
verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug
if isfile(cachefile)
@logmsg verbosity "Recompiling stale cache file $cachefile for $pkg"
else
@logmsg verbosity "Precompiling $pkg"
end
@logmsg verbosity "Precompiling $pkg"
p = create_expr_cache(path, cachefile, concrete_deps, pkg.uuid)
if success(p)
# append checksum to the end of the .ji file:
Expand Down