Merge pull request #12559 from stevengj/cache_mtime

cache dependency mtimes and check equality, not mtime(dep) <= mtime(cache)

base/loading.jl

@@ -116,13 +116,14 @@ const package_locks = Dict{Symbol,Condition}()
 const package_loaded = Set{Symbol}()
 
 # used to optionally track dependencies when requiring a module:
-const _require_dependencies = ByteString[]
+const _require_dependencies = Tuple{ByteString,Float64}[]
 const _track_dependencies = [false]
 function _include_dependency(_path::AbstractString)
     prev = source_path(nothing)
     path = (prev === nothing) ? abspath(_path) : joinpath(dirname(prev),_path)
-    if _track_dependencies[1]
-        push!(_require_dependencies, abspath(path))
+    if myid() == 1 && _track_dependencies[1]
+        apath = abspath(path)
+        push!(_require_dependencies, (apath, mtime(apath)))
     end
     return path, prev
 end
@@ -343,7 +344,7 @@ isvalid_cache_header(f::IOStream) = 0 != ccall(:jl_deserialize_verify_header, Ci
 
 function cache_dependencies(f::IO)
     modules = Tuple{Symbol,UInt64}[]
-    files = ByteString[]
+    files = Tuple{ByteString,Float64}[]
     while true
         n = ntoh(read(f, Int32))
         n == 0 && break
@@ -355,7 +356,7 @@ function cache_dependencies(f::IO)
     while true
         n = ntoh(read(f, Int32))
         n == 0 && break
-        push!(files, bytestring(readbytes(f, n)))
+        push!(files, (bytestring(readbytes(f, n)), ntoh(read(f, Float64))))
     end
     return modules, files
 end
@@ -370,15 +371,15 @@ function cache_dependencies(cachefile::AbstractString)
     end
 end
 
-function stale_cachefile(cachefile::AbstractString, cachefile_mtime::Real=mtime(cachefile))
+function stale_cachefile(cachefile::AbstractString)
     io = open(cachefile, "r")
     try
         if !isvalid_cache_header(io)
             return true # invalid cache file
         end
         modules, files = cache_dependencies(io)
-        for f in files
-            if mtime(f) > cachefile_mtime
+        for (f,ftime) in files
+            if mtime(f) != ftime
                 return true
             end
         end
@@ -399,7 +400,7 @@ end
 
 function recompile_stale(mod, cachefile)
     cachestat = stat(cachefile)
-    if iswritable(cachestat) && stale_cachefile(cachefile, cachestat.mtime)
+    if iswritable(cachestat) && stale_cachefile(cachefile)
         info("Recompiling stale cache file $cachefile for module $mod.")
         create_expr_cache(find_in_path(string(mod)), cachefile)
     end

src/dump.c

@@ -190,6 +190,11 @@ static void write_as_tag(ios_t *s, uint8_t tag)
     write_uint8(s, tag);
 }
 
+static void write_float64(ios_t *s, double x)
+{
+    write_uint64(s, *((uint64_t*)&x));
+}
+
 // --- Static Compile ---
 
 #define jl_serialize_value(s, v) jl_serialize_value_(s,(jl_value_t*)(v))
@@ -952,7 +957,7 @@ void jl_serialize_mod_list(ios_t *s)
 }
 
 // "magic" string and version header of .ji file
-static const int JI_FORMAT_VERSION = 1;
+static const int JI_FORMAT_VERSION = 2;
 static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature
 static const uint16_t BOM = 0xFEFF; // byte-order marker
 static void jl_serialize_header(ios_t *s)
@@ -988,10 +993,10 @@ void jl_serialize_dependency_list(ios_t *s)
         size_t l = jl_array_len(deps);
         jl_value_t *prev = NULL;
         for (size_t i=0; i < l; i++) {
-            jl_value_t *dep = jl_cellref(deps, i);
+            jl_value_t *dep = jl_fieldref(jl_cellref(deps, i), 0);
             size_t slen = jl_string_len(dep);
             if (!prev || memcmp(jl_string_data(dep), jl_string_data(prev), slen)) {
-                total_size += 4 + slen;
+                total_size += 4 + slen + 8;
             }
             prev = dep;
         }
@@ -1004,12 +1009,14 @@ void jl_serialize_dependency_list(ios_t *s)
         size_t l = jl_array_len(deps);
         jl_value_t *prev = NULL;
         for (size_t i=0; i < l; i++) {
-            jl_value_t *dep = jl_cellref(deps, i);
+            jl_value_t *deptuple = jl_cellref(deps, i);
+            jl_value_t *dep = jl_fieldref(deptuple, 0);
             size_t slen = jl_string_len(dep);
             if (!prev || memcmp(jl_string_data(dep), jl_string_data(prev), slen)) {
                 write_int32(s, slen);
                 ios_write(s, jl_string_data(dep), slen);
             }
+            write_float64(s, jl_unbox_float64(jl_fieldref(deptuple, 1)));
             prev = dep;
         }
         write_int32(s, 0); // terminator, for ease of reading

test/compile.jl

@@ -47,7 +47,7 @@ try
         deps = Base.cache_dependencies(cachefile)
         @test sort(deps[1]) == map(s -> (s, Base.module_uuid(eval(s))),
                                    [:Base,:Core,:Main])
-        @test sort(deps[2]) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
+        @test map(x -> x[1], sort(deps[2])) == [Foo_file,joinpath(dir,"bar.jl"),joinpath(dir,"foo.jl")]
     end
 
     Baz_file = joinpath(dir, "Baz.jl")