Emit split ji and so pkgimage

base/loading.jl

@@ -898,8 +898,14 @@ function _include_from_serialized(pkg::PkgId, path::String, depmods::Vector{Any}
         t_comp_before = cumulative_compile_time_ns()
     end
 
-    @debug "Loading cache file $path for $pkg"
-    sv = ccall(:jl_restore_incremental, Any, (Cstring, Any), path, depmods)
+    opath = string(chopsuffix(path, ".ji"), ".", Base.Libc.dlext)
+    if ispath(opath)
+        @debug "Loading object cache file $opath for $pkg"
+        sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any), opath, depmods)
+    else
+        @debug "Loading cache file $path for $pkg"
+        sv = ccall(:jl_restore_incremental, Any, (Cstring, Any), path, depmods)
+    end
     if isa(sv, Exception)
         return sv
     end
@@ -1658,9 +1664,11 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto
 end
 
 const PRECOMPILE_TRACE_COMPILE = Ref{String}()
-function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout)
+function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String},
+                           concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout)
     @nospecialize internal_stderr internal_stdout
     rm(output, force=true)   # Remove file if it exists
+    rm(output_o, force=true)
     depot_path = map(abspath, DEPOT_PATH)
     dl_load_path = map(abspath, DL_LOAD_PATH)
     load_path = map(abspath, Base.load_path())
@@ -1679,11 +1687,17 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_d
     for (pkg, build_id) in concrete_deps
         push!(deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))")
     end
+
+    if output_o !== nothing
+        opts = `--output-o $(output_o) --output-ji $(output) --output-incremental=yes`
+    else
+        opts = `--output-ji $(output) --output-incremental=yes`
+    end
+
     deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing)
     deps = deps_eltype * "[" * join(deps_strs, ",") * "]"
     trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : ``
-    io = open(pipeline(addenv(`$(julia_cmd()::Cmd) -O0
-                              --output-ji $output --output-incremental=yes
+    io = open(pipeline(addenv(`$(julia_cmd()::Cmd) -O0 $(opts)
                               --startup-file=no --history-file=no --warn-overwrite=yes
                               --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no")
                               $trace
@@ -1738,6 +1752,58 @@ end
 
 const MAX_NUM_PRECOMPILE_FILES = Ref(10)
 
+module Linking
+
+const lld_path = Ref{String}()
+if Sys.iswindows()
+    const lld_exe = "lld.exe"
+else
+    const lld_exe = "lld"
+end
+
+function __init__()
+    # Prefer our own bundled lld, but if we don't have one, pick it up off of the PATH
+    # If this is an in-tree build, `lld` will live in `tools`.  Otherwise, it'll be in `libexec`
+    for bundled_lld_path in (joinpath(Sys.BINDIR, Base.LIBEXECDIR, lld_exe),
+                             joinpath(Sys.BINDIR, "..", "tools", lld_exe),
+                             joinpath(Sys.BINDIR, lld_exe))
+        if isfile(bundled_lld_path)
+            lld_path[] = abspath(bundled_lld_path)
+            return
+        end
+    end
+    lld_path[] = something(Sys.which(lld_exe), lld_exe)
+    return
+end
+
+function lld()
+    return Cmd([lld_path[]])
+end
+
+
+function ld()
+    @static if Sys.iswindows()
+        flavor = "link"
+    elseif Sys.isapple()
+        flavor = "darwin"
+    else
+        flavor = "gnu"
+    end
+    `$(lld()) -flavor $flavor`
+end
+
+is_debug() = ccall(:jl_is_debugbuild, Cint, ()) == 1
+
+function link_jilib(path, out, args=``)
+    LIBDIR = joinpath(Sys.BINDIR, "..", "lib")
+    LIBS = is_debug() ? `-ljulia-debug -ljulia-internal-debug` : `-ljulia -ljulia-internal`
+    WHOLE_ARCHIVE = Sys.isapple() ? `-all_load` : `--whole-archive`
+    NO_WHOLE_ARCHIVE = Sys.isapple() ? `` : `--no-whole-archive`
+
+    run(`$(ld()) --shared --output=$out $WHOLE_ARCHIVE $path $NO_WHOLE_ARCHIVE -L$(LIBDIR) $LIBS $args`, stdin, stdout, stderr)
+end
+end
+
 function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, internal_stdout::IO = stdout,
                       keep_loaded_modules::Bool = true)
 
@@ -1762,23 +1828,32 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in
     # write the checksum, _and then_ atomically move the file to `cachefile`.
     mkpath(cachepath)
     tmppath, tmpio = mktemp(cachepath)
+    tmppath_o, tmpio_o = mktemp(cachepath)
+    tmppath_so, tmpio_so = mktemp(cachepath)
     local p
     try
         close(tmpio)
-        p = create_expr_cache(pkg, path, tmppath, concrete_deps, internal_stderr, internal_stdout)
+        close(tmpio_o)
+        close(tmpio_so)
+        p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, internal_stderr, internal_stdout)
         if success(p)
+            # Run linker over tmppath_o
+            Linking.link_jilib(tmppath_o, tmppath_so)
+
+            # Read preferences hash back from .ji file (we can't precompute because
+            # we don't actually know what the list of compile-time preferences are without compiling)
+            prefs_hash = preferences_hash(tmppath)
+            cachefile = compilecache_path(pkg, prefs_hash)
+            ocachefile = string(chopsuffix(cachefile, ".ji"), ".", Base.Libc.dlext)
+
             # append checksum to the end of the .ji file:
             open(tmppath, "a+") do f
+                # TODO write path and checksum of ocachefile
                 write(f, _crc32c(seekstart(f)))
             end
             # inherit permission from the source file (and make them writable)
             chmod(tmppath, filemode(path) & 0o777 | 0o200)
 
-            # Read preferences hash back from .ji file (we can't precompute because
-            # we don't actually know what the list of compile-time preferences are without compiling)
-            prefs_hash = preferences_hash(tmppath)
-            cachefile = compilecache_path(pkg, prefs_hash)
-
             # prune the directory with cache files
             if pkg.uuid !== nothing
                 entrypath, entryfile = cache_file_entry(pkg)
@@ -1791,10 +1866,13 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in
 
             # this is atomic according to POSIX:
             rename(tmppath, cachefile; force=true)
+            rename(tmppath_so, ocachefile; force=true)
             return cachefile
         end
     finally
         rm(tmppath, force=true)
+        rm(tmppath_o, force=true)
+        rm(tmppath_so, force=true)
     end
     if p.exitcode == 125
         return PrecompilableError()
@@ -2336,4 +2414,5 @@ end
 
 precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing))
 precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String))
-precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), IO, IO))
+precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO))
+precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO))

src/julia.h

@@ -1762,7 +1762,7 @@ JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void);
 JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s);
 JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname);
 JL_DLLEXPORT void jl_set_sysimg_so(void *handle);
-JL_DLLEXPORT ios_t *jl_create_system_image(void *, jl_array_t *worklist);
+JL_DLLEXPORT void jl_create_system_image(void *, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z);
 JL_DLLEXPORT void jl_restore_system_image(const char *fname);
 JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len);
 JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods);

src/precompile.c

@@ -63,19 +63,17 @@ JL_DLLEXPORT void jl_write_compiler_output(void)
     if (jl_options.incremental)
         jl_precompile_toplevel_module = NULL;
 
-    if (jl_options.incremental) {
-        if (jl_options.outputbc || jl_options.outputunoptbc)
-            jl_printf(JL_STDERR, "WARNING: incremental output to a .bc file is not implemented\n");
-        if (jl_options.outputasm)
-            jl_printf(JL_STDERR, "WARNING: incremental output to a .s file is not implemented\n");
-        if (jl_options.outputo) {
-            jl_printf(JL_STDERR, "WARNING: incremental output to a .o file is not implemented\n");
-        }
-    }
+    bool_t emit_native = jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm;
+
+    bool_t emit_split = jl_options.outputji && emit_native;
 
     ios_t *s = NULL;
-    if (jl_options.outputji || jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm)
-        s = jl_create_system_image(native_code, jl_options.incremental ? worklist : NULL);
+    ios_t *z = NULL;
+    if (jl_options.outputji || emit_native)
+        jl_create_system_image(native_code, jl_options.incremental ? worklist : NULL, emit_split, &s, &z);
+
+    if (!emit_split)
+        z = s; 
 
     if (jl_options.outputji) {
         ios_t f;
@@ -91,7 +89,7 @@ JL_DLLEXPORT void jl_write_compiler_output(void)
                         jl_options.outputunoptbc,
                         jl_options.outputo,
                         jl_options.outputasm,
-                        (const char*)s->buf, (size_t)s->size);
+                        (const char*)z->buf, (size_t)z->size);
         jl_postoutput_hook();
     }
 
@@ -100,6 +98,11 @@ JL_DLLEXPORT void jl_write_compiler_output(void)
         free(s);
     }
 
+    if (emit_split) {
+        ios_close(z);
+        free(z);
+    }
+
     for (size_t i = 0; i < jl_current_modules.size; i += 2) {
         if (jl_current_modules.table[i + 1] != HT_NOTFOUND) {
             jl_printf(JL_STDERR, "\nWARNING: detected unclosed module: ");

src/staticdata.c

@@ -2430,7 +2430,7 @@ static void jl_save_system_image_to_stream(ios_t *f,
     jl_gc_enable(en);
 }
 
-static int64_t jl_incremental_header_stuff(ios_t *f, jl_array_t *worklist, jl_array_t **mod_array, jl_array_t **udeps)
+static int64_t jl_incremental_write_header(ios_t *f, jl_array_t *worklist, jl_array_t **mod_array, jl_array_t **udeps)
 {
     *mod_array = jl_get_loaded_modules();  // __toplevel__ modules loaded in this session (from Base.loaded_modules_array)
     assert(jl_precompile_toplevel_module == NULL);
@@ -2450,28 +2450,46 @@ static int64_t jl_incremental_header_stuff(ios_t *f, jl_array_t *worklist, jl_ar
     return srctextpos;
 }
 
-
-JL_DLLEXPORT ios_t *jl_create_system_image(void *_native_data, jl_array_t *worklist)
+JL_DLLEXPORT jl_create_system_image(void *_native_data, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z)
 {
     jl_gc_collect(JL_GC_FULL);
     jl_gc_collect(JL_GC_INCREMENTAL);   // sweep finalizers
     JL_TIMING(SYSIMG_DUMP);
 
+    // iff emit_split
+    // write header and src_text to one file f/s
+    // write systemimg to a second file ff/z
     jl_task_t *ct = jl_current_task;
     ios_t *f = (ios_t*)malloc_s(sizeof(ios_t));
     ios_mem(f, 0);
+
+    ios_t *ff = NULL;
+    if (emit_split) {
+        ff = (ios_t*)malloc_s(sizeof(ios_t));
+        ios_mem(ff, 0);
+    } else {
+        ff = f;
+    }
+
     jl_array_t *mod_array = NULL, *udeps = NULL, *extext_methods = NULL, *new_specializations = NULL;
     jl_array_t *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL;
     JL_GC_PUSH7(&mod_array, &udeps, &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges);
     int64_t srctextpos = 0;
     if (worklist) {
-        srctextpos = jl_incremental_header_stuff(f, worklist, &mod_array, &udeps);
+        srctextpos = jl_incremental_write_header(f, worklist, &mod_array, &udeps);
+        if (emit_split) {
+            write_header(ff);
+            write_mod_list(ff, mod_array);
+        }
         jl_gc_enable_finalizers(ct, 0); // make sure we don't run any Julia code concurrently after this point
         jl_prepare_serialization_data(mod_array, newly_inferred, jl_worklist_key(worklist), &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges);
+
         write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f));
+        if (emit_split)
+            write_padding(ff, LLT_ALIGN(ios_pos(ff), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(ff));
     }
     native_functions = _native_data;
-    jl_save_system_image_to_stream(f, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges);
+    jl_save_system_image_to_stream(ff, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges);
     native_functions = NULL;
     if (worklist) {
         jl_gc_enable_finalizers(ct, 1); // make sure we don't run any Julia code concurrently before this point
@@ -2524,7 +2542,10 @@ JL_DLLEXPORT ios_t *jl_create_system_image(void *_native_data, jl_array_t *workl
     }
 
     JL_GC_POP();
-    return f;
+    *s = f;
+    if (emit_split)
+        *z = ff;
+    return;
 }
 
 JL_DLLEXPORT size_t ios_write_direct(ios_t *dest, ios_t *src);
@@ -2924,15 +2945,16 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_array_t *de
         return jl_get_exceptionf(jl_errorexception_type,
                 "Precompile file header verification checks failed.");
     }
-    { // skip past the mod list
-        size_t len;
-        while ((len = read_int32(f)))
-            ios_skip(f, len + 3 * sizeof(uint64_t));
-    }
-    { // skip past the dependency list
-        size_t deplen = read_uint64(f);
-        ios_skip(f, deplen);
-    }
+    // mod list and dependency list stored in .ji
+    // { // skip past the mod list
+    //     size_t len;
+    //     while ((len = read_int32(f)))
+    //         ios_skip(f, len + 3 * sizeof(uint64_t));
+    // }
+    // { // skip past the dependency list
+    //     size_t deplen = read_uint64(f);
+    //     ios_skip(f, deplen);
+    // }
 
     // verify that the system state is valid
     jl_value_t *verify_fail = read_verify_mod_list(f, depmods);