From b4e432497c6362f07a6c4f7b38cc935800f02f64 Mon Sep 17 00:00:00 2001 From: Elliot Saba Date: Mon, 28 Sep 2020 06:54:12 +0000 Subject: [PATCH] [loading]: Add serialization/checking of `Preferences.jl` hash This adds the calculation, serialization and verification of preferences hashes at code loading time. Preferences, as stored by the forthcoming `Preferences.jl` package within a top-level `Project.toml` file, are parsed by the `dump.c` and `loading.jl` code loading machinery and used to provide a compile-time preferences machinery. --- base/loading.jl | 90 +++++++++++++++++++++++++++++++++++++++------- base/util.jl | 2 ++ src/dump.c | 25 +++++++++++++ test/precompile.jl | 2 +- 4 files changed, 105 insertions(+), 14 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 33d8b4b1f7325..776f06202e0b4 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -322,6 +322,31 @@ function manifest_deps_get(env::String, where::PkgId, name::String, cache::TOMLC return nothing end +function uuid_in_environment(project_file::String, uuid::UUID, cache::TOMLCache) + # First, check to see if we're looking for the environment itself + proj_uuid = get(parsed_toml(cache, project_file), "uuid", nothing) + if proj_uuid !== nothing && UUID(proj_uuid) == uuid + return true + end + + # Check to see if there's a Manifest.toml associated with this project + manifest_file = project_file_manifest_path(project_file, cache) + if manifest_file === nothing + return false + end + manifest = parsed_toml(cache, manifest_file) + for (dep_name, entries) in manifest + for entry in entries + entry_uuid = get(entry, "uuid", nothing)::Union{String, Nothing} + if uuid !== nothing && UUID(entry_uuid) == uuid + return true + end + end + end + # If all else fails, return `false` + return false +end + function manifest_uuid_path(env::String, pkg::PkgId, cache::TOMLCache)::Union{Nothing,String} project_file = env_project_file(env) if project_file isa String @@ -950,7 +975,7 @@ function _require(pkg::PkgId, cache::TOMLCache) if (0 == ccall(:jl_generating_output, Cint, ())) || (JLOptions().incremental != 0) # spawn off a new incremental pre-compile task for recursive `require` calls # or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable) - cachefile = compilecache(pkg, path) + cachefile = compilecache(pkg, path, cache) if isa(cachefile, Exception) if precompilableerror(cachefile) verbosity = isinteractive() ? CoreLogging.Info : CoreLogging.Debug @@ -1195,7 +1220,7 @@ end @assert precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), Bool)) @assert precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), Bool)) -function compilecache_path(pkg::PkgId)::String +function compilecache_path(pkg::PkgId, cache::TOMLCache)::String entrypath, entryfile = cache_file_entry(pkg) cachepath = joinpath(DEPOT_PATH[1], entrypath) isdir(cachepath) || mkpath(cachepath) @@ -1205,6 +1230,7 @@ function compilecache_path(pkg::PkgId)::String crc = _crc32c(something(Base.active_project(), "")) crc = _crc32c(unsafe_string(JLOptions().image_file), crc) crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) + crc = _crc32c(get_preferences_hash(pkg.uuid, cache), crc) project_precompile_slug = slug(crc, 5) abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji")) end @@ -1218,18 +1244,17 @@ This can be used to reduce package load times. Cache files are stored in `DEPOT_PATH[1]/compiled`. See [Module initialization and precompilation](@ref) for important notes. """ -function compilecache(pkg::PkgId, cache::TOMLCache = TOMLCache()) +function compilecache(pkg::PkgId, cache::TOMLCache = TOMLCache(), show_errors::Bool = true) path = locate_package(pkg, cache) path === nothing && throw(ArgumentError("$pkg not found during precompilation")) - return compilecache(pkg, path) + return compilecache(pkg, path, cache, show_errors) end const MAX_NUM_PRECOMPILE_FILES = 10 -# `show_errors` is an "internal" interface for Pkg.precompile -function compilecache(pkg::PkgId, path::String, show_errors::Bool = true) +function compilecache(pkg::PkgId, path::String, cache::TOMLCache = TOMLCache(), show_errors::Bool = true) # decide where to put the resulting cache file - cachefile = compilecache_path(pkg) + cachefile = compilecache_path(pkg, cache) cachepath = dirname(cachefile) # prune the directory with cache files if pkg.uuid !== nothing @@ -1333,6 +1358,8 @@ function parse_cache_header(f::IO) end totbytes -= 4 + 4 + n2 + 8 end + prefs_hash = read(f, UInt64) + totbytes -= 8 @assert totbytes == 12 "header of cache file appears to be corrupt" srctextpos = read(f, Int64) # read the list of modules that are required to be present during loading @@ -1345,7 +1372,7 @@ function parse_cache_header(f::IO) build_id = read(f, UInt64) # build id push!(required_modules, PkgId(uuid, sym) => build_id) end - return modules, (includes, requires), required_modules, srctextpos + return modules, (includes, requires), required_modules, srctextpos, prefs_hash end function parse_cache_header(cachefile::String; srcfiles_only::Bool=false) @@ -1354,21 +1381,21 @@ function parse_cache_header(cachefile::String; srcfiles_only::Bool=false) !isvalid_cache_header(io) && throw(ArgumentError("Invalid header in cache file $cachefile.")) ret = parse_cache_header(io) srcfiles_only || return ret - modules, (includes, requires), required_modules, srctextpos = ret + modules, (includes, requires), required_modules, srctextpos, prefs_hash = ret srcfiles = srctext_files(io, srctextpos) delidx = Int[] for (i, chi) in enumerate(includes) chi.filename ∈ srcfiles || push!(delidx, i) end deleteat!(includes, delidx) - return modules, (includes, requires), required_modules, srctextpos + return modules, (includes, requires), required_modules, srctextpos, prefs_hash finally close(io) end end function cache_dependencies(f::IO) - defs, (includes, requires), modules = parse_cache_header(f) + defs, (includes, requires), modules, srctextpos, prefs_hash = parse_cache_header(f) return modules, map(chi -> (chi.filename, chi.mtime), includes) # return just filename and mtime end @@ -1383,7 +1410,7 @@ function cache_dependencies(cachefile::String) end function read_dependency_src(io::IO, filename::AbstractString) - modules, (includes, requires), required_modules, srctextpos = parse_cache_header(io) + modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io) srctextpos == 0 && error("no source-text stored in cache file") seek(io, srctextpos) return _read_dependency_src(io, filename) @@ -1428,6 +1455,37 @@ function srctext_files(f::IO, srctextpos::Int64) return files end +# Find the Project.toml that we should load/store to for Preferences +function get_preferences_project_path(uuid::UUID, cache::TOMLCache = TOMLCache()) + for env in load_path() + project_file = env_project_file(env) + if !isa(project_file, String) + continue + end + if uuid_in_environment(project_file, uuid, cache) + return project_file + end + end + return nothing +end + +function get_preferences(uuid::UUID, cache::TOMLCache = TOMLCache(); + prefs_key::String = "compile-preferences") + project_path = get_preferences_project_path(uuid, cache) + if project_path !== nothing + preferences = get(parsed_toml(cache, project_path), prefs_key, Dict{String,Any}()) + if haskey(preferences, string(uuid)) + return preferences[string(uuid)] + end + end + # Fall back to default value of "no preferences". + return Dict{String,Any}() +end +get_preferences_hash(uuid::UUID, cache::TOMLCache = TOMLCache()) = UInt64(hash(get_preferences(uuid, cache))) +get_preferences_hash(m::Module, cache::TOMLCache = TOMLCache()) = get_preferences_hash(PkgId(m).uuid, cache) +get_preferences_hash(::Nothing, cache::TOMLCache = TOMLCache()) = UInt64(hash(Dict{String,Any}())) + + # returns true if it "cachefile.ji" is stale relative to "modpath.jl" # otherwise returns the list of dependencies to also check stale_cachefile(modpath::String, cachefile::String) = stale_cachefile(modpath, cachefile, TOMLCache()) @@ -1438,7 +1496,7 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) @debug "Rejecting cache file $cachefile due to it containing an invalid cache header" return true # invalid cache file end - (modules, (includes, requires), required_modules) = parse_cache_header(io) + modules, (includes, requires), required_modules, srctextpos, prefs_hash = parse_cache_header(io) id = isempty(modules) ? nothing : first(modules).first modules = Dict{PkgId, UInt64}(modules) @@ -1514,6 +1572,12 @@ function stale_cachefile(modpath::String, cachefile::String, cache::TOMLCache) end if isa(id, PkgId) + curr_prefs_hash = get_preferences_hash(id.uuid, cache) + if prefs_hash != curr_prefs_hash + @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))" + return true + end + get!(PkgOrigin, pkgorigins, id).cachepath = cachefile end diff --git a/base/util.jl b/base/util.jl index e9db6af3150b7..e8c68d6f1ab29 100644 --- a/base/util.jl +++ b/base/util.jl @@ -385,6 +385,8 @@ _crc32c(io::IO, crc::UInt32=0x00000000) = _crc32c(io, typemax(Int64), crc) _crc32c(io::IOStream, crc::UInt32=0x00000000) = _crc32c(io, filesize(io)-position(io), crc) _crc32c(uuid::UUID, crc::UInt32=0x00000000) = ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt128}, Csize_t), crc, uuid.value, 16) +_crc32c(x::Integer, crc::UInt32=0x00000000) = + ccall(:jl_crc32c, UInt32, (UInt32, Vector{UInt8}, Csize_t), crc, reinterpret(UInt8, [x]), sizeof(x)) """ @kwdef typedef diff --git a/src/dump.c b/src/dump.c index e07b01121d031..287bf2d44701b 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1123,6 +1123,31 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t **udepsp, jl_array_t * write_int32(s, 0); } write_int32(s, 0); // terminator, for ease of reading + + // Calculate Preferences hash for current package. + jl_value_t *prefs_hash = NULL; + if (jl_base_module) { + // Toplevel module is the module we're currently compiling, use it to get our preferences hash + jl_value_t * toplevel = (jl_value_t*)jl_get_global(jl_base_module, jl_symbol("__toplevel__")); + jl_value_t * prefs_hash_func = jl_get_global(jl_base_module, jl_symbol("get_preferences_hash")); + + if (toplevel && prefs_hash_func) { + // call get_preferences_hash(__toplevel__) + jl_value_t *prefs_hash_args[2] = {prefs_hash_func, (jl_value_t*)toplevel}; + size_t last_age = jl_get_ptls_states()->world_age; + jl_get_ptls_states()->world_age = jl_world_counter; + prefs_hash = (jl_value_t*)jl_apply(prefs_hash_args, 2); + jl_get_ptls_states()->world_age = last_age; + } + } + + // If we successfully got the preferences, write it out, otherwise write `0` for this `.ji` file. + if (prefs_hash != NULL) { + write_uint64(s, jl_unbox_uint64(prefs_hash)); + } else { + write_uint64(s, 0); + } + // write a dummy file position to indicate the beginning of the source-text pos = ios_pos(s); ios_seek(s, initial_pos); diff --git a/test/precompile.jl b/test/precompile.jl index d5449d507c3b6..d92cee18101c1 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -416,7 +416,7 @@ try """) cachefile = Base.compilecache(Base.PkgId("FooBar")) - @test cachefile == Base.compilecache_path(Base.PkgId("FooBar")) + @test cachefile == Base.compilecache_path(Base.PkgId("FooBar"), Base.TOMLCache()) @test isfile(joinpath(cachedir, "FooBar.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Vector @test !isdefined(Main, :FooBar)