From 0a8e0636e5b3db2a5dceb53d1a5d75ac920f8951 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Wed, 12 Oct 2022 22:20:49 -0400 Subject: [PATCH 01/25] Cache objectcode in pkgimages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Handle external functions in other images When we find an edge to an external function (already cached in an loaded pkgimage), we emit a global variable which we will patch during loading with the address of the function to call. 2. Split `.ji` and `.so` Validation headers and srctext goes into `.ji`, data and objectcode goes into .so. 3. Implement Base.Linking using lld to link on user machines On MacOS we are not gurantueed to have a usable `-lSystem` we can link against, so we use `-undefined dynamic_lookup` 4. Implement logic for caching and selection Co-authored-by: Mosè Giordano Co-authored-by: Tim Holy --- Makefile | 13 ++ base/Base.jl | 1 + base/Makefile | 4 + base/linking.jl | 147 +++++++++++++ base/loading.jl | 221 ++++++++++++++++---- base/options.jl | 1 + base/util.jl | 31 ++- contrib/cache_stdlibs.jl | 49 +++++ contrib/print_sorted_stdlibs.jl | 11 +- doc/make.jl | 1 + doc/src/devdocs/pkgimg.md | 47 +++++ doc/src/devdocs/sysimg.md | 2 +- doc/src/manual/code-loading.md | 9 +- doc/src/manual/environment-variables.md | 4 + doc/src/manual/methods.md | 36 +++- doc/src/manual/modules.md | 24 ++- doc/src/manual/performance-tips.md | 2 +- src/aotcompile.cpp | 119 +++++++++-- src/codegen-stubs.c | 9 +- src/codegen.cpp | 48 ++++- src/coverage.cpp | 2 +- src/debug-registry.h | 16 +- src/debuginfo.cpp | 94 +++++---- src/debuginfo.h | 2 +- src/disasm.cpp | 2 +- src/jitlayers.h | 4 +- src/jl_exported_data.inc | 2 + src/jl_exported_funcs.inc | 4 + src/jloptions.c | 22 ++ src/jloptions.h | 1 + src/julia.h | 5 +- src/julia_internal.h | 11 +- src/llvm-multiversioning.cpp | 78 +++---- src/precompile.c | 108 ++++++++-- src/processor.cpp | 4 +- src/processor.h | 9 +- src/processor_arm.cpp | 9 +- src/processor_fallback.cpp | 9 +- src/processor_x86.cpp | 9 +- src/staticdata.c | 265 +++++++++++++++--------- src/staticdata_utils.c | 44 +++- test/compiler/contextual.jl | 6 +- test/loading.jl | 6 +- test/precompile.jl | 51 +++-- 44 files changed, 1205 insertions(+), 337 deletions(-) create mode 100644 base/linking.jl create mode 100644 contrib/cache_stdlibs.jl create mode 100644 doc/src/devdocs/pkgimg.md diff --git a/Makefile b/Makefile index eb6704a1fa1e0..f085d005402ba 100644 --- a/Makefile +++ b/Makefile @@ -260,13 +260,21 @@ ifeq ($(OS),WINNT) -$(INSTALL_M) $(wildcard $(build_bindir)/*.dll) $(DESTDIR)$(bindir)/ ifeq ($(JULIA_BUILD_MODE),release) -$(INSTALL_M) $(build_libdir)/libjulia.dll.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libjulia-internal.dll.a $(DESTDIR)$(libdir)/ else ifeq ($(JULIA_BUILD_MODE),debug) -$(INSTALL_M) $(build_libdir)/libjulia-debug.dll.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libjulia-internal-debug.dll.a $(DESTDIR)$(libdir)/ endif # We have a single exception; we want 7z.dll to live in libexec, not bin, so that 7z.exe can find it. -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(libexecdir)/ -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libssp.a $(DESTDIR)$(libdir)/ # TODO use implib + # The rest are compiler dependencies, as an example memcpy is exported by msvcrt + # These are files from mingw32 and required for creating shared libraries like our caches. + -$(INSTALL_M) $(build_libdir)/libgcc_s.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libgcc.a $(DESTDIR)$(libdir)/ + -$(INSTALL_M) $(build_libdir)/libmsvcrt.a $(DESTDIR)$(libdir)/ else # Copy over .dSYM directories directly for Darwin @@ -331,6 +339,11 @@ else ifeq ($(JULIA_BUILD_MODE),debug) $(INSTALL_M) $(build_private_libdir)/sys-debug.$(SHLIB_EXT) $(DESTDIR)$(private_libdir) endif + # Cache stdlibs + @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no $(JULIAHOME)/contrib/cache_stdlibs.jl) + # CI uses `--check-bounds=yes` which impacts the cache flags + @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no --check-bounds=yes $(JULIAHOME)/contrib/cache_stdlibs.jl) + # Copy in all .jl sources as well mkdir -p $(DESTDIR)$(datarootdir)/julia/base $(DESTDIR)$(datarootdir)/julia/test cp -R -L $(JULIAHOME)/base/* $(DESTDIR)$(datarootdir)/julia/base diff --git a/base/Base.jl b/base/Base.jl index 0c53a8bc9124b..c2a1dc4de96b5 100644 --- a/base/Base.jl +++ b/base/Base.jl @@ -417,6 +417,7 @@ include("threadcall.jl") include("uuid.jl") include("pkgid.jl") include("toml_parser.jl") +include("linking.jl") include("loading.jl") # misc useful functions & macros diff --git a/base/Makefile b/base/Makefile index bb79549aeea2e..e4b28f1380705 100644 --- a/base/Makefile +++ b/base/Makefile @@ -81,6 +81,10 @@ ifeq ($(DARWIN_FRAMEWORK), 1) @echo "const DARWIN_FRAMEWORK_NAME = \"$(FRAMEWORK_NAME)\"" >> $@ else @echo "const DARWIN_FRAMEWORK = false" >> $@ +endif +ifeq ($(OS), Darwin) + @echo "const MACOS_PRODUCT_VERSION = \"$(shell sw_vers -productVersion)\"" >> $@ + @echo "const MACOS_PLATFORM_VERSION = \"$(shell xcrun --show-sdk-version)\"" >> $@ endif @echo "const BUILD_TRIPLET = \"$(BB_TRIPLET_LIBGFORTRAN_CXXABI)\"" >> $@ diff --git a/base/linking.jl b/base/linking.jl new file mode 100644 index 0000000000000..288279347f1c5 --- /dev/null +++ b/base/linking.jl @@ -0,0 +1,147 @@ +# This file is a part of Julia. License is MIT: https://julialang.org/license +module Linking + +import Base.Libc: Libdl + +# inlined LLD_jll +# These get calculated in __init__() +const PATH = Ref("") +const LIBPATH = Ref("") +const PATH_list = String[] +const LIBPATH_list = String[] +const lld_path = Ref{String}() +const lld_exe = Sys.iswindows() ? "lld.exe" : "lld" + +if Sys.iswindows() + const LIBPATH_env = "PATH" + const LIBPATH_default = "" + const pathsep = ';' +elseif Sys.isapple() + const LIBPATH_env = "DYLD_FALLBACK_LIBRARY_PATH" + const LIBPATH_default = "~/lib:/usr/local/lib:/lib:/usr/lib" + const pathsep = ':' +else + const LIBPATH_env = "LD_LIBRARY_PATH" + const LIBPATH_default = "" + const pathsep = ':' +end + +function adjust_ENV!(env::Dict, PATH::String, LIBPATH::String, adjust_PATH::Bool, adjust_LIBPATH::Bool) + if adjust_LIBPATH + LIBPATH_base = get(env, LIBPATH_env, expanduser(LIBPATH_default)) + if !isempty(LIBPATH_base) + env[LIBPATH_env] = string(LIBPATH, pathsep, LIBPATH_base) + else + env[LIBPATH_env] = LIBPATH + end + end + if adjust_PATH && (LIBPATH_env != "PATH" || !adjust_LIBPATH) + if !isempty(get(env, "PATH", "")) + env["PATH"] = string(PATH, pathsep, env["PATH"]) + else + env["PATH"] = PATH + end + end + return env +end + +function __init_lld_path() + # Prefer our own bundled lld, but if we don't have one, pick it up off of the PATH + # If this is an in-tree build, `lld` will live in `tools`. Otherwise, it'll be in `libexec` + for bundled_lld_path in (joinpath(Sys.BINDIR, Base.LIBEXECDIR, lld_exe), + joinpath(Sys.BINDIR, "..", "tools", lld_exe), + joinpath(Sys.BINDIR, lld_exe)) + if isfile(bundled_lld_path) + lld_path[] = abspath(bundled_lld_path) + return + end + end + lld_path[] = something(Sys.which(lld_exe), lld_exe) + return +end + +const VERBOSE = Ref{Bool}(false) + +function __init__() + VERBOSE[] = parse(Bool, get(ENV, "JULIA_VERBOSE_LINKING", "false")) + + __init_lld_path() + PATH[] = dirname(lld_path[]) + if Sys.iswindows() + # On windows, the dynamic libraries (.dll) are in Sys.BINDIR ("usr\\bin") + append!(LIBPATH_list, [abspath(Sys.BINDIR, Base.LIBDIR, "julia"), Sys.BINDIR]) + else + append!(LIBPATH_list, [abspath(Sys.BINDIR, Base.LIBDIR, "julia"), abspath(Sys.BINDIR, Base.LIBDIR)]) + end + LIBPATH[] = join(LIBPATH_list, pathsep) + return +end + +function lld(; adjust_PATH::Bool = true, adjust_LIBPATH::Bool = true) + env = adjust_ENV!(copy(ENV), PATH[], LIBPATH[], adjust_PATH, adjust_LIBPATH) + return Cmd(Cmd([lld_path[]]); env) +end + +function ld() + default_args = `` + @static if Sys.iswindows() + # LLD supports mingw style linking + flavor = "gnu" + m = Sys.ARCH == :x86_64 ? "i386pep" : "i386pe" + default_args = `-m $m -Bdynamic --enable-auto-image-base --allow-multiple-definition` + elseif Sys.isapple() + flavor = "darwin" + arch = Sys.ARCH == :aarch64 ? :arm64 : Sys.ARCH + default_args = `-arch $arch -undefined dynamic_lookup -platform_version macos $(Base.MACOS_PRODUCT_VERSION) $(Base.MACOS_PLATFORM_VERSION)` + else + flavor = "gnu" + end + + `$(lld()) -flavor $flavor $default_args` +end + +const WHOLE_ARCHIVE = if Sys.isapple() + "-all_load" +else + "--whole-archive" +end + +const NO_WHOLE_ARCHIVE = if Sys.isapple() + "" +else + "--no-whole-archive" +end + +const SHARED = if Sys.isapple() + "-dylib" +else + "-shared" +end + +is_debug() = ccall(:jl_is_debugbuild, Cint, ()) == 1 +libdir() = abspath(Sys.BINDIR, Base.LIBDIR) +private_libdir() = abspath(Sys.BINDIR, Base.PRIVATE_LIBDIR) +if Sys.iswindows() + shlibdir() = Sys.BINDIR +else + shlibdir() = libdir() +end + +function link_image_cmd(path, out) + LIBDIR = "-L$(libdir())" + PRIVATE_LIBDIR = "-L$(private_libdir())" + SHLIBDIR = "-L$(shlibdir())" + LIBS = is_debug() ? ("-ljulia-debug", "-ljulia-internal-debug") : ("-ljulia", "-ljulia-internal") + @static if Sys.iswindows() + LIBS = (LIBS..., "-lopenlibm", "-lssp", "-lgcc_s", "-lgcc", "-lmsvcrt") + end + + V = VERBOSE[] ? "--verbose" : "" + `$(ld()) $V $SHARED -o $out $WHOLE_ARCHIVE $path $NO_WHOLE_ARCHIVE $LIBDIR $PRIVATE_LIBDIR $SHLIBDIR $LIBS` +end + +function link_image(path, out, internal_stderr::IO = stderr, internal_stdout::IO = stdout) + run(link_image_cmd(path, out), Base.DevNull(), stderr, stdout) +end + +end # module Linking diff --git a/base/loading.jl b/base/loading.jl index ea350ff72d960..587353ff3b820 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -907,7 +907,8 @@ function find_all_in_cache_path(pkg::PkgId) isdir(path) || continue for file in readdir(path, sort = false) # no sort given we sort later if !((pkg.uuid === nothing && file == entryfile * ".ji") || - (pkg.uuid !== nothing && startswith(file, entryfile * "_"))) + (pkg.uuid !== nothing && startswith(file, entryfile * "_") && + endswith(file, ".ji"))) continue end filepath = joinpath(path, file) @@ -924,13 +925,15 @@ function find_all_in_cache_path(pkg::PkgId) end end +ocachefile_from_cachefile(cachefile) = string(chopsuffix(cachefile, ".ji"), ".", Base.Libc.dlext) + # use an Int counter so that nested @time_imports calls all remain open const TIMING_IMPORTS = Threads.Atomic{Int}(0) # these return either the array of modules loaded from the path / content given # or an Exception that describes why it couldn't be loaded # and it reconnects the Base.Docs.META -function _include_from_serialized(pkg::PkgId, path::String, depmods::Vector{Any}) +function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}, depmods::Vector{Any}) assert_havelock(require_lock) timing_imports = TIMING_IMPORTS[] > 0 try @@ -940,8 +943,13 @@ function _include_from_serialized(pkg::PkgId, path::String, depmods::Vector{Any} t_comp_before = cumulative_compile_time_ns() end - @debug "Loading cache file $path for $pkg" - sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint), path, depmods, false) + if ocachepath !== nothing + @debug "Loading object cache file $ocachepath for $pkg" + sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any), ocachepath, depmods) + else + @debug "Loading cache file $path for $pkg" + sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint), path, depmods, false) + end if isa(sv, Exception) return sv end @@ -1200,7 +1208,7 @@ end # loads a precompile cache file, ignoring stale_cachefile tests # assuming all depmods are already loaded and everything is valid -function _tryrequire_from_serialized(modkey::PkgId, path::String, sourcepath::String, depmods::Vector{Any}) +function _tryrequire_from_serialized(modkey::PkgId, path::String, ocachepath::Union{Nothing, String}, sourcepath::String, depmods::Vector{Any}) assert_havelock(require_lock) loaded = nothing if root_module_exists(modkey) @@ -1222,7 +1230,7 @@ function _tryrequire_from_serialized(modkey::PkgId, path::String, sourcepath::St package_locks[modkey] = Threads.Condition(require_lock) try set_pkgorigin_version_path(modkey, sourcepath) - loaded = _include_from_serialized(modkey, path, depmods) + loaded = _include_from_serialized(modkey, path, ocachepath, depmods) finally loading = pop!(package_locks, modkey) notify(loading, loaded, all=true) @@ -1241,13 +1249,23 @@ end # loads a precompile cache file, ignoring stale_cachefile tests # load the best available (non-stale) version of all dependent modules first -function _tryrequire_from_serialized(pkg::PkgId, path::String) +function _tryrequire_from_serialized(pkg::PkgId, path::String, ocachepath::Union{Nothing, String}) assert_havelock(require_lock) local depmodnames io = open(path, "r") try iszero(isvalid_cache_header(io)) && return ArgumentError("Invalid header in cache file $path.") - depmodnames = parse_cache_header(io)[3] + _, _, depmodnames, _, _, _, clone_targets, _ = parse_cache_header(io) + pkgimage = !isempty(clone_targets) + if pkgimage + ocachepath !== nothing || return ArgumentError("Expected ocachepath to be provided") + isfile(ocachepath) || return ArgumentError("Ocachepath $ocachpath is not a file.") + ocachepath == ocachefile_from_cachefile(path) || return ArgumentError("$ocachepath is not the expected ocachefile") + # TODO: Check for valid clone_targets? + isvalid_pkgimage_crc(io, ocachepath) || return ArgumentError("Invalid checksum in cache file $ocachepath.") + else + @assert ocachepath === nothing + end isvalid_file_crc(io) || return ArgumentError("Invalid checksum in cache file $path.") finally close(io) @@ -1263,7 +1281,7 @@ function _tryrequire_from_serialized(pkg::PkgId, path::String) depmods[i] = dep end # then load the file - return _include_from_serialized(pkg, path, depmods) + return _include_from_serialized(pkg, path, ocachepath, depmods) end # returns `nothing` if require found a precompile cache for this sourcepath, but couldn't load it @@ -1271,12 +1289,13 @@ end @constprop :none function _require_search_from_serialized(pkg::PkgId, sourcepath::String, build_id::UInt128) assert_havelock(require_lock) paths = find_all_in_cache_path(pkg) + ocachefile = nothing for path_to_try in paths::Vector{String} staledeps = stale_cachefile(pkg, build_id, sourcepath, path_to_try) if staledeps === true continue end - staledeps = staledeps::Vector{Any} + staledeps, ocachefile = staledeps::Tuple{Vector{Any}, Union{Nothing, String}} # finish checking staledeps module graph for i in 1:length(staledeps) dep = staledeps[i] @@ -1289,8 +1308,8 @@ end if modstaledeps === true continue end - modstaledeps = modstaledeps::Vector{Any} - staledeps[i] = (modpath, modkey, modpath_to_try, modstaledeps) + modstaledeps, modocachepath = modstaledeps::Tuple{Vector{Any}, Union{Nothing, String}} + staledeps[i] = (modpath, modkey, modpath_to_try, modstaledeps, modocachepath) modfound = true break end @@ -1301,6 +1320,7 @@ end end end if staledeps === true + ocachefile = nothing continue end try @@ -1312,19 +1332,20 @@ end for i in 1:length(staledeps) dep = staledeps[i] dep isa Module && continue - modpath, modkey, modpath_to_try, modstaledeps = dep::Tuple{String, PkgId, String, Vector{Any}} - dep = _tryrequire_from_serialized(modkey, modpath_to_try, modpath, modstaledeps) + modpath, modkey, modcachepath, modstaledeps, modocachepath = dep::Tuple{String, PkgId, String, Vector{Any}, Union{Nothing, String}} + dep = _tryrequire_from_serialized(modkey, modcachepath, modocachepath, modpath, modstaledeps) if !isa(dep, Module) - @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modpath." exception=dep + @debug "Rejecting cache file $path_to_try because required dependency $modkey failed to load from cache file for $modcachepath." exception=dep staledeps = true break end staledeps[i] = dep end if staledeps === true + ocachefile = nothing continue end - restored = _include_from_serialized(pkg, path_to_try, staledeps) + restored = _include_from_serialized(pkg, path_to_try, ocachefile, staledeps) if !isa(restored, Module) @debug "Deserialization checks failed while attempting to load cache from $path_to_try" exception=restored else @@ -1633,7 +1654,8 @@ function _require(pkg::PkgId, env=nothing) end # fall-through to loading the file locally else - m = _tryrequire_from_serialized(pkg, cachefile) + cachefile, ocachefile = cachefile::Tuple{String, Union{Nothing, String}} + m = _tryrequire_from_serialized(pkg, cachefile, ocachefile) if !isa(m, Module) @warn "The call to compilecache failed to create a usable precompiled cache file for $pkg" exception=m else @@ -1669,10 +1691,11 @@ function _require(pkg::PkgId, env=nothing) return loaded end -function _require_from_serialized(uuidkey::PkgId, path::String) +# Only used from test/precompile.jl +function _require_from_serialized(uuidkey::PkgId, path::String, ocachepath::Union{String, Nothing}) @lock require_lock begin set_pkgorigin_version_path(uuidkey, nothing) - newm = _tryrequire_from_serialized(uuidkey, path) + newm = _tryrequire_from_serialized(uuidkey, path, ocachepath) newm isa Module || throw(newm) insert_extension_triggers(uuidkey) # After successfully loading, notify downstream consumers @@ -1872,9 +1895,11 @@ function include_package_for_output(pkg::PkgId, input::String, depot_path::Vecto end const PRECOMPILE_TRACE_COMPILE = Ref{String}() -function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout) +function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String}, + concrete_deps::typeof(_concrete_dependencies), internal_stderr::IO = stderr, internal_stdout::IO = stdout) @nospecialize internal_stderr internal_stdout rm(output, force=true) # Remove file if it exists + output_o === nothing || rm(output_o, force=true) depot_path = map(abspath, DEPOT_PATH) dl_load_path = map(abspath, DL_LOAD_PATH) load_path = map(abspath, Base.load_path()) @@ -1893,11 +1918,20 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, concrete_d for (pkg, build_id) in concrete_deps push!(deps_strs, "$(pkg_str(pkg)) => $(repr(build_id))") end + + if output_o !== nothing + cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) + opt_level = Base.JLOptions().opt_level + opts = `-O$(opt_level) --output-o $(output_o) --output-ji $(output) --output-incremental=yes` + else + cpu_target = nothing + opts = `-O0 --output-ji $(output) --output-incremental=yes` + end + deps_eltype = sprint(show, eltype(concrete_deps); context = :module=>nothing) deps = deps_eltype * "[" * join(deps_strs, ",") * "]" trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])` : `` - io = open(pipeline(addenv(`$(julia_cmd()::Cmd) -O0 - --output-ji $output --output-incremental=yes + io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) $(opts) --startup-file=no --history-file=no --warn-overwrite=yes --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no") $trace @@ -1930,6 +1964,8 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64)::String crc = _crc32c(something(Base.active_project(), "")) crc = _crc32c(unsafe_string(JLOptions().image_file), crc) crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) + crc = _crc32c(ccall(:jl_cache_flags, UInt8, ()), crc) + crc = _crc32c(prefs_hash, crc) project_precompile_slug = slug(crc, 5) abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji")) @@ -1976,27 +2012,61 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in # create a temporary file in `cachepath` directory, write the cache in it, # write the checksum, _and then_ atomically move the file to `cachefile`. mkpath(cachepath) + cache_objects = JLOptions().use_pkgimage_native_code != 0 tmppath, tmpio = mktemp(cachepath) + + if cache_objects + tmppath_o, tmpio_o = mktemp(cachepath) + tmppath_so, tmpio_so = mktemp(cachepath) + else + tmppath_o = nothing + end local p try close(tmpio) - p = create_expr_cache(pkg, path, tmppath, concrete_deps, internal_stderr, internal_stdout) + if cache_objects + close(tmpio_o) + close(tmpio_so) + end + p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, internal_stderr, internal_stdout) + if success(p) + if cache_objects + # Run linker over tmppath_o + Linking.link_image(tmppath_o, tmppath_so) + end + + # Read preferences hash back from .ji file (we can't precompute because + # we don't actually know what the list of compile-time preferences are without compiling) + prefs_hash = preferences_hash(tmppath) + cachefile = compilecache_path(pkg, prefs_hash) + ocachefile = cache_objects ? ocachefile_from_cachefile(cachefile) : nothing + + # append checksum for so to the end of the .ji file: + crc_so = UInt32(0) + if cache_objects + crc_so = open(_crc32c, tmppath_so, "r") + end + # append extra crc to the end of the .ji file: open(tmppath, "r+") do f if iszero(isvalid_cache_header(f)) error("Invalid header for $pkg in new cache file $(repr(tmppath)).") end + seekend(f) + write(f, crc_so) seekstart(f) write(f, _crc32c(f)) end + # inherit permission from the source file (and make them writable) chmod(tmppath, filemode(path) & 0o777 | 0o200) - - # Read preferences hash back from .ji file (we can't precompute because - # we don't actually know what the list of compile-time preferences are without compiling) - prefs_hash = preferences_hash(tmppath) - cachefile = compilecache_path(pkg, prefs_hash) + if cache_objects + # Ensure that the user can execute the `.so` we're generating + # Note that on windows, `filemode(path)` typically returns `0o666`, so this + # addition of the execute bit for the user is doubly needed. + chmod(tmppath_so, filemode(path) & 0o777 | 0o300) + end # prune the directory with cache files if pkg.uuid !== nothing @@ -2010,10 +2080,17 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in # this is atomic according to POSIX (not Win32): rename(tmppath, cachefile; force=true) - return cachefile + if cache_objects + rename(tmppath_so, ocachefile::String; force=true) + end + return cachefile, ocachefile end finally rm(tmppath, force=true) + if cache_objects + rm(tmppath_o, force=true) + rm(tmppath_so, force=true) + end end if p.exitcode == 125 return PrecompilableError() @@ -2027,9 +2104,26 @@ function module_build_id(m::Module) return (UInt128(hi) << 64) | lo end -isvalid_cache_header(f::IOStream) = ccall(:jl_read_verify_header, UInt64, (Ptr{Cvoid},), f.ios) # returns checksum id or zero +function isvalid_cache_header(f::IOStream) + pkgimage = Ref{UInt8}() + checksum = ccall(:jl_read_verify_header, UInt64, (Ptr{Cvoid}, Ptr{UInt8}, Ptr{Int64}, Ptr{Int64}), f.ios, pkgimage, Ref{Int64}(), Ref{Int64}()) # returns checksum id or zero + + if !iszero(checksum) && pkgimage[] != 0 + @debug "Cache header was for pkgimage" + return UInt64(0) # We somehow read the header for a pkgimage and not a ji + end + return checksum +end isvalid_file_crc(f::IOStream) = (_crc32c(seekstart(f), filesize(f) - 4) == read(f, UInt32)) +function isvalid_pkgimage_crc(f::IOStream, ocachefile::String) + seekstart(f) # TODO necessary + seek(f, filesize(f) - 8) + expected_crc_so = read(f, UInt32) + crc_so = open(_crc32c, ocachefile, "r") + expected_crc_so == crc_so +end + struct CacheHeaderIncludes id::PkgId filename::String @@ -2038,6 +2132,7 @@ struct CacheHeaderIncludes end function parse_cache_header(f::IO) + flags = read(f, UInt8) modules = Vector{Pair{PkgId, UInt64}}() while true n = read(f, Int32) @@ -2111,7 +2206,10 @@ function parse_cache_header(f::IO) build_id |= read(f, UInt64) push!(required_modules, PkgId(uuid, sym) => build_id) end - return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash + l = read(f, Int32) + clone_targets = read(f, l) + + return modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags end function parse_cache_header(cachefile::String; srcfiles_only::Bool=false) @@ -2133,8 +2231,6 @@ function parse_cache_header(cachefile::String; srcfiles_only::Bool=false) end end - - preferences_hash(f::IO) = parse_cache_header(f)[6] function preferences_hash(cachefile::String) io = open(cachefile, "r") @@ -2148,7 +2244,6 @@ function preferences_hash(cachefile::String) end end - function cache_dependencies(f::IO) _, (includes, _), modules, _... = parse_cache_header(f) return modules, map(chi -> (chi.filename, chi.mtime), includes) # return just filename and mtime @@ -2375,6 +2470,15 @@ get_compiletime_preferences(uuid::UUID) = collect(get(Vector{String}, COMPILETIM get_compiletime_preferences(m::Module) = get_compiletime_preferences(PkgId(m).uuid) get_compiletime_preferences(::Nothing) = String[] +function check_clone_targets(clone_targets) + try + ccall(:jl_check_pkgimage_clones, Cvoid, (Ptr{Cchar},), clone_targets) + return true + catch + return false + end +end + # returns true if it "cachefile.ji" is stale relative to "modpath.jl" and build_id for modkey # otherwise returns the list of dependencies to also check @constprop :none function stale_cachefile(modpath::String, cachefile::String; ignore_loaded::Bool = false) @@ -2388,10 +2492,33 @@ end @debug "Rejecting cache file $cachefile due to it containing an invalid cache header" return true # invalid cache file end - modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash = parse_cache_header(io) + modules, (includes, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags = parse_cache_header(io) if isempty(modules) return true # ignore empty file end + pkgimage = !isempty(clone_targets) + if pkgimage + if ccall(:jl_match_cache_flags, UInt8, (UInt8,), flags) == 0 + @debug "Rejecting cache file $cachefile for $modkey since the flags are mismatched" cachefile_flags=flags current_flags=ccall(:jl_cache_flags, UInt8, ()) + return true + end + ocachefile = ocachefile_from_cachefile(cachefile) + if JLOptions().use_pkgimage_native_code == 0 + # presence of clone_targets means native code cache + @debug "Rejecting cache file $cachefile for $modkey since it would require usage of pkgimage" + return true + end + if !check_clone_targets(clone_targets) + @debug "Rejecting cache file $cachefile for $modkey since pkgimage can't be loaded on this target" + return true + end + if !isfile(ocachefile) + @debug "Rejecting cache file $cachefile for $modkey since pkgimage $ocachefile was not found" + return true + end + else + ocachefile = nothing + end id = first(modules) if id.first != modkey && modkey != PkgId("") @debug "Rejecting cache file $cachefile for $modkey since it is for $id instead" @@ -2454,7 +2581,7 @@ end # now check if this file is fresh relative to its source files if !skip_timecheck - if !samefile(includes[1].filename, modpath) + if !samefile(includes[1].filename, modpath) && !samefile(fixup_stdlib_path(includes[1].filename), modpath) @debug "Rejecting cache file $cachefile because it is for file $(includes[1].filename) not file $modpath" return true # cache file was compiled from a different path end @@ -2467,6 +2594,16 @@ end end for chi in includes f, ftime_req = chi.filename, chi.mtime + if !isfile(f) + _f = fixup_stdlib_path(f) + if isfile(_f) && startswith(_f, Sys.STDLIB) + # mtime is changed by extraction + @debug "Skipping mtime check for file $f used by $cachefile, since it is a stdlib" + continue + end + @debug "Rejecting stale cache file $cachefile because file $f does not exist" + return true + end ftime = mtime(f) is_stale = ( ftime != ftime_req ) && ( ftime != floor(ftime_req) ) && # Issue #13606, PR #13613: compensate for Docker images rounding mtimes @@ -2486,13 +2623,20 @@ end return true end + if pkgimage + if !isvalid_pkgimage_crc(io, ocachefile::String) + @debug "Rejecting cache file $cachefile because $ocachefile has an invalid checksum" + return true + end + end + curr_prefs_hash = get_preferences_hash(id.uuid, prefs) if prefs_hash != curr_prefs_hash @debug "Rejecting cache file $cachefile because preferences hash does not match 0x$(string(prefs_hash, base=16)) != 0x$(string(curr_prefs_hash, base=16))" return true end - return depmods # fresh cachefile + return depmods, ocachefile # fresh cachefile finally close(io) end @@ -2565,4 +2709,5 @@ end precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) -precompile(create_expr_cache, (PkgId, String, String, typeof(_concrete_dependencies), IO, IO)) +precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), IO, IO)) +precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), IO, IO)) diff --git a/base/options.jl b/base/options.jl index 48a8f7ff59d38..8b580f6c900b0 100644 --- a/base/options.jl +++ b/base/options.jl @@ -37,6 +37,7 @@ struct JLOptions cookie::Ptr{UInt8} handle_signals::Int8 use_sysimage_native_code::Int8 + use_pkgimage_native_code::Int8 use_compiled_modules::Int8 bindto::Ptr{UInt8} outputbc::Ptr{UInt8} diff --git a/base/util.jl b/base/util.jl index 3ae75a7f58e28..9209ed19fb647 100644 --- a/base/util.jl +++ b/base/util.jl @@ -133,7 +133,7 @@ See also [`print`](@ref), [`println`](@ref), [`show`](@ref). printstyled(stdout, msg...; bold=bold, underline=underline, blink=blink, reverse=reverse, hidden=hidden, color=color) """ - Base.julia_cmd(juliapath=joinpath(Sys.BINDIR, julia_exename())) + Base.julia_cmd(juliapath=joinpath(Sys.BINDIR, julia_exename()); cpu_target) Return a julia command similar to the one of the running process. Propagates any of the `--cpu-target`, `--sysimage`, `--compile`, `--sysimage-native-code`, @@ -148,10 +148,15 @@ Among others, `--math-mode`, `--warn-overwrite`, and `--trace-compile` are notab !!! compat "Julia 1.5" The flags `--color` and `--startup-file` were added in Julia 1.5. + +!!! compat "Julia 1.9" + The keyword argument `cpu_target` was added. """ -function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename())) +function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Union{Nothing,String} = nothing) opts = JLOptions() - cpu_target = unsafe_string(opts.cpu_target) + if cpu_target === nothing + cpu_target = unsafe_string(opts.cpu_target) + end image_file = unsafe_string(opts.image_file) addflags = String[] let compile = if opts.compile_enabled == 0 @@ -220,6 +225,12 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename())) if opts.use_sysimage_native_code == 0 push!(addflags, "--sysimage-native-code=no") end + if opts.use_pkgimage_native_code == 0 + push!(addflags, "--pkgimage-native-code=no") + else + # If pkgimage is set, malloc_log and code_coverage should not + @assert opts.malloc_log == 0 && opts.code_coverage == 0 + end return `$julia -C$cpu_target -J$image_file $addflags` end @@ -487,10 +498,17 @@ function _crc32c(io::IO, nb::Integer, crc::UInt32=0x00000000) end _crc32c(io::IO, crc::UInt32=0x00000000) = _crc32c(io, typemax(Int64), crc) _crc32c(io::IOStream, crc::UInt32=0x00000000) = _crc32c(io, filesize(io)-position(io), crc) -_crc32c(uuid::UUID, crc::UInt32=0x00000000) = - ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt128}, Csize_t), crc, uuid.value, 16) +_crc32c(uuid::UUID, crc::UInt32=0x00000000) = _crc32c(uuid.value, crc) +_crc32c(x::UInt128, crc::UInt32=0x00000000) = + ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt128}, Csize_t), crc, x, 16) _crc32c(x::UInt64, crc::UInt32=0x00000000) = ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt64}, Csize_t), crc, x, 8) +_crc32c(x::UInt32, crc::UInt32=0x00000000) = + ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt32}, Csize_t), crc, x, 4) +_crc32c(x::UInt16, crc::UInt32=0x00000000) = + ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt16}, Csize_t), crc, x, 2) +_crc32c(x::UInt8, crc::UInt32=0x00000000) = + ccall(:jl_crc32c, UInt32, (UInt32, Ref{UInt8}, Csize_t), crc, x, 1) """ @kwdef typedef @@ -652,7 +670,8 @@ function runtests(tests = ["all"]; ncores::Int = ceil(Int, Sys.CPU_THREADS / 2), seed !== nothing && push!(tests, "--seed=0x$(string(seed % UInt128, base=16))") # cast to UInt128 to avoid a minus sign ENV2 = copy(ENV) ENV2["JULIA_CPU_THREADS"] = "$ncores" - ENV2["JULIA_DEPOT_PATH"] = mktempdir(; cleanup = true) + pathsep = Sys.iswindows() ? ";" : ":" + ENV2["JULIA_DEPOT_PATH"] = string(mktempdir(; cleanup = true), pathsep) # make sure the default depots can be loaded delete!(ENV2, "JULIA_LOAD_PATH") delete!(ENV2, "JULIA_PROJECT") try diff --git a/contrib/cache_stdlibs.jl b/contrib/cache_stdlibs.jl new file mode 100644 index 0000000000000..bdcc3d9535fa4 --- /dev/null +++ b/contrib/cache_stdlibs.jl @@ -0,0 +1,49 @@ +# Stdlibs sorted in dependency, then alphabetical, order by contrib/print_sorted_stdlibs.jl +# Run with the `--exclude-sysimage` option to filter out all packages included in the system image +stdlibs = [ + # No dependencies + + # 1-depth packages + :GMP_jll, + :LLVMLibUnwind_jll, + :LibUV_jll, + :LibUnwind_jll, + :MbedTLS_jll, + :OpenLibm_jll, + :PCRE2_jll, + :Zlib_jll, + :dSFMT_jll, + :libLLVM_jll, + + # 2-depth packages + :LibSSH2_jll, + :MPFR_jll, + + # 3-depth packages + :LibGit2_jll, + + # 7-depth packages + :LLD_jll, + :SuiteSparse_jll, + + # 9-depth packages + :Statistics, + :SuiteSparse, +] + +depot = abspath(Sys.BINDIR, "..", "share", "julia") + +if haskey(ENV, "JULIA_CPU_TARGET") + target = ENV["JULIA_CPU_TARGET"] +else + target = "native" +end + +@info "Caching stdlibrary to" depot target +empty!(Base.DEPOT_PATH) +push!(Base.DEPOT_PATH, depot) + +for pkg in stdlibs + pkgid = Base.identify_package(string(pkg)) + Base.compilecache(pkgid) +end diff --git a/contrib/print_sorted_stdlibs.jl b/contrib/print_sorted_stdlibs.jl index 28d75f079b9dd..6bc2023c4f1cc 100644 --- a/contrib/print_sorted_stdlibs.jl +++ b/contrib/print_sorted_stdlibs.jl @@ -12,11 +12,12 @@ function check_flag(flag) end if check_flag("--help") || check_flag("-h") - println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls]") + println("Usage: julia print_sorted_stdlibs.jl [stdlib_dir] [--exclude-jlls] [--exclude-sysimage]") end # Allow users to ask for JLL or no JLLs exclude_jlls = check_flag("--exclude-jlls") +exclude_sysimage = check_flag("--exclude-sysimage") # Default to the `stdlib/vX.Y` directory STDLIB_DIR = get(ARGS, 1, joinpath(@__DIR__, "..", "usr", "share", "julia", "stdlib")) @@ -80,12 +81,20 @@ if exclude_jlls filter!(p -> !endswith(p, "_jll"), sorted_projects) end +if exclude_sysimage + loaded_modules = Set(map(k->k.name, collect(keys(Base.loaded_modules)))) + filter!(p->!in(p, loaded_modules), sorted_projects) +end + # Print out sorted projects, ready to be pasted into `sysimg.jl` last_depth = 0 println(" # Stdlibs sorted in dependency, then alphabetical, order by contrib/print_sorted_stdlibs.jl") if exclude_jlls println(" # Run with the `--exclude-jlls` option to filter out all JLL packages") end +if exclude_sysimage + println(" # Run with the `--exclude-sysimage` option to filter out all packages included in the system image") +end println(" stdlibs = [") println(" # No dependencies") for p in sorted_projects diff --git a/doc/make.jl b/doc/make.jl index 75e3598ced6f7..04b8af595e58f 100644 --- a/doc/make.jl +++ b/doc/make.jl @@ -142,6 +142,7 @@ DevDocs = [ "devdocs/subarrays.md", "devdocs/isbitsunionarrays.md", "devdocs/sysimg.md", + "devdocs/pkgimg.md", "devdocs/llvm.md", "devdocs/stdio.md", "devdocs/boundscheck.md", diff --git a/doc/src/devdocs/pkgimg.md b/doc/src/devdocs/pkgimg.md new file mode 100644 index 0000000000000..dfd11648ef784 --- /dev/null +++ b/doc/src/devdocs/pkgimg.md @@ -0,0 +1,47 @@ +# Package Images + +Julia package images provide object (native code) caches for Julia packages. +They are similar to Julia's [system image](@) and support many of the same features. +In fact the underlying serialization format is the same, and the system image is the base image that the package images are build against. + +## High-level overview + +Package images are shared libraries that contain both code and data. Like `.ji` cache files they are generated per package. The data section contains both global data (global variables in the package) as well as the necessary metadata about what methods and types are defined by the package. The code section contains native objects that cache the final output of Julia's LLVM based compiler. + +The command line option `--pkgimage-native-code=no` can be used to turn-off object caching for this session. Note that this means that cache files have to be regenerated. + +!!! note + While the package images present themselves as native shared libraries they are only an approximation thereof. You will not be able to link against them from a native program and they must be loaded from Julia. + + +## Linking + +Since the package images partially contain native code, we must run a linker over them before we can use them. You can use the environment variable `JULIA_VERBOSE_LINKING` to make the pkgimage linking process verbose. + +Furthermore we cannot assume that the user has a working system linker installed. Julia now comes with LLD, the LLVM linker, to provide a working out of the box experience. In `base/linking.jl` we implement a limited interface to be able to link package images on all supported platforms. + +### Quirks +Despite LLD being a multi-platform linker it does not provide a consistent interface across platforms. Furthermore it is meant to be used from `clang` or +another compiler driver, we therefore reimplement some of the logic from `llvm-project/clang/lib/Driver/ToolChains`. Thankfully one can use `lld -flavor` to set lld to the right platform + +#### Windows +To avoid having to deal with `link.exe` we use `-flavor gnu`, effectively turning `lld` into a cross-linker from a mingw32 environment. Windows DLLs are required to contain a `_DllMainCRTStartup` function and to minimize our dependenc on mingw32 libraries we inject a stub definition ourselves. + +#### MacOS +Dynamic libraries on MacOS need to link against `-lSystem`. On recent MacOS versions `-lSystem` is only available for linking when XCode is available. +To that effect we link with `-undefined dynamic_lookup`. + +## Package images optimized for multiple microarchitectures +Similar to [multi-versioning](@ref sysimg-multi-versioning) for system images, package images support multi-versioning. If you are in a heterogenous environment, with a unified cache, +you can set the environment variable `JULIA_CPU_TARGET` to multi-version the object caches. + +## Flags that impact pkgimage + +These are the Julia command line flags that impact cache selection. Pkgimages +that were created with different flags will be rejected. + +- `--debug-level`: Exact match required since it changes codegeneration. +- `--checkbounds`: Exact match required since it changes codegeneration. +- `--pkgimage-native-code`: To allow running without object caching enabled. +- `--optimize`: Reject pkgimages generated for a lower optimization level, + but allow for higher optimization levels to be loaded. diff --git a/doc/src/devdocs/sysimg.md b/doc/src/devdocs/sysimg.md index 5c976875846d3..a21e3ba265f9b 100644 --- a/doc/src/devdocs/sysimg.md +++ b/doc/src/devdocs/sysimg.md @@ -19,7 +19,7 @@ This operation is useful for multiple reasons. A user may: The [`PackageCompiler.jl` package](https://github.com/JuliaLang/PackageCompiler.jl) contains convenient wrapper functions to automate this process. -## System image optimized for multiple microarchitectures +## [System image optimized for multiple microarchitectures](@id sysimg-multi-versioning) The system image can be compiled simultaneously for multiple CPU microarchitectures under the same instruction set architecture (ISA). Multiple versions of the same function diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index f9575b0159d8c..74208dd876bb4 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -388,17 +388,18 @@ When a package with extensions is added to an environment, the `weakdeps` and `e are stored in the manifest file in the section for that package. The dependency lookup rules for a package are the same as for its "parent" except that the listed extension dependencies are also considered as dependencies. -### Package/Environment Preferences + +### [Package/Environment Preferences](@id preferences) Preferences are dictionaries of metadata that influence package behavior within an environment. -The preferences system supports reading preferences at compile-time, which means that at code-loading time, we must ensure that a particular `.ji` file was built with the same preferences as the current environment before loading it. +The preferences system supports reading preferences at compile-time, which means that at code-loading time, we must ensure that the chosen precompilation files were built with the same preferences as the current environment before loading them. The public API for modifying Preferences is contained within the [Preferences.jl](https://github.com/JuliaPackaging/Preferences.jl) package. Preferences are stored as TOML dictionaries within a `(Julia)LocalPreferences.toml` file next to the currently-active project. If a preference is "exported", it is instead stored within the `(Julia)Project.toml` instead. The intention is to allow shared projects to contain shared preferences, while allowing for users themselves to override those preferences with their own settings in the LocalPreferences.toml file, which should be .gitignored as the name implies. -Preferences that are accessed during compilation are automatically marked as compile-time preferences, and any change recorded to these preferences will cause the Julia compiler to recompile any cached precompilation `.ji` files for that module. -This is done by serializing the hash of all compile-time preferences during compilation, then checking that hash against the current environment when searching for the proper `.ji` file to load. +Preferences that are accessed during compilation are automatically marked as compile-time preferences, and any change recorded to these preferences will cause the Julia compiler to recompile any cached precompilation file(s) (`.ji` and corresponding `.so`, `.dll`, or `.dylib` files) for that module. +This is done by serializing the hash of all compile-time preferences during compilation, then checking that hash against the current environment when searching for the proper file(s) to load. Preferences can be set with depot-wide defaults; if package Foo is installed within your global environment and it has preferences set, these preferences will apply as long as your global environment is part of your `LOAD_PATH`. Preferences in environments higher up in the environment stack get overridden by the more proximal entries in the load path, ending with the currently active project. diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index bc4a742365d69..26a7316d71da2 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -166,6 +166,10 @@ $(DEPOT_PATH[1])/logs/repl_history.jl Sets the maximum number of different instances of a single package that are to be stored in the precompile cache (default = 10). +### `JULIA_VERBOSE_LINKING` + +If set to true, linker commands will be displayed during precompilation. + ## Pkg.jl ### `JULIA_CI` diff --git a/doc/src/manual/methods.md b/doc/src/manual/methods.md index 6cbcc4fad6a65..376a848808e43 100644 --- a/doc/src/manual/methods.md +++ b/doc/src/manual/methods.md @@ -265,8 +265,40 @@ julia> methods(+) ``` Multiple dispatch together with the flexible parametric type system give Julia its ability to -abstractly express high-level algorithms decoupled from implementation details, yet generate efficient, -specialized code to handle each case at run time. +abstractly express high-level algorithms decoupled from implementation details. + +## [Method specializations](@id man-method-specializations) + +When you create multiple methods of the same function, this is sometimes called +"specialization." In this case, you're specializing the *function* by adding additional +methods to it: each new method is a new specialization of the function. +As shown above, these specializations are returned by `methods`. + +There's another kind of specialization that occurs without programmer intervention: +Julia's compiler can automatically specialize the *method* for the specific argument types used. +Such specializations are *not* listed by `methods`, as this doesn't create new `Method`s, but tools like [`@code_typed`](@ref) allow you to inspect such specializations. + +For example, if you create a method + +``` +mysum(x::Real, y::Real) = x + y +``` + +you've given the function `mysum` one new method (possibly its only method), and that method takes any pair of `Real` number inputs. But if you then execute + +```julia-repl +julia> mysum(1, 2) +3 + +julia> mysum(1.0, 2.0) +3.0 +``` + +Julia will compile `mysum` twice, once for `x::Int, y::Int` and again for `x::Float64, y::Float64`. +The point of compiling twice is performance: the methods that get called for `+` (which `mysum` uses) vary depending on the specific types of `x` and `y`, and by compiling different specializations Julia can do all the method lookup ahead of time. This allows the program to run much more quickly, since it doesn't have to bother with method lookup while it is running. +Julia's automatic specialization allows you to write "generic" algorithms and expect that the compiler will generate efficient, specialized code to handle each case when you need it. + +In cases where the number of potential specializations might be effectively unlimited, Julia may avoid this default specialization. See [Be aware of when Julia avoids specializing](@ref) for more information. ## [Method Ambiguities](@id man-ambiguities) diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index f0a9a5110ded4..a8bc3f68fb80d 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -9,7 +9,7 @@ Modules in Julia help organize code into coherent units. They are delimited synt 2. Modules have facilities for detailed namespace management: each defines a set of names it `export`s, and can import names from other modules with `using` and `import` (we explain these below). -3. Modules can be precompiled for faster loading, and contain code for runtime initialization. +3. Modules can be precompiled for faster loading, and may contain code for runtime initialization. Typically, in larger Julia packages you will see module code organized into files, eg @@ -429,11 +429,14 @@ Large modules can take several seconds to load because executing all of the stat often involves compiling a large amount of code. Julia creates precompiled caches of the module to reduce this time. -The incremental precompiled module file are created and used automatically when using `import` -or `using` to load a module. This will cause it to be automatically compiled the first time -it is imported. Alternatively, you can manually call [`Base.compilecache(Base.identify_package("modulename"))`](@ref). The resulting -cache files will be stored in `DEPOT_PATH[1]/compiled/`. Subsequently, the module is automatically -recompiled upon `using` or `import` whenever any of its dependencies change; dependencies are modules it +Precompiled module files (sometimes called "cache files") are created and used automatically when `import` or `using` loads a module. If the cache file(s) do not yet exist, the module will be compiled and saved for future reuse. You can also manually call [`Base.compilecache(Base.identify_package("modulename"))`](@ref) to create these files without loading the module. The resulting +cache files will be stored in `DEPOT_PATH[1]/compiled/`. If nothing about your system changes, +such cache files will be used when you load the module with `import` or `using`. + +Precompilation cache files store definitions of modules, types, methods, and constants. They may also store method specializations and the code generated for them, but this typically requires that the developer add explicit [`precompile`](@ref) directives or execute workloads that force compilation during the package build. + +However, if you update the modules' dependencies or change its source code, the module is automatically +recompiled upon `using` or `import`. Dependencies are modules it imports, the Julia build, files it includes, or explicit dependencies declared by [`include_dependency(path)`](@ref) in the module file(s). @@ -445,6 +448,7 @@ by the search logic in `require` matches the path that had created the precompil into account the set of dependencies already loaded into the current process and won't recompile those modules, even if their files change or disappear, in order to avoid creating incompatibilities between the running system and the precompile cache. +Finally, it takes account of changes in any [compile-time preferences](@ref preferences). If you know that a module is *not* safe to precompile (for example, for one of the reasons described below), you should @@ -589,6 +593,12 @@ A few other points to be aware of: It is sometimes helpful during module development to turn off incremental precompilation. The command line flag `--compiled-modules={yes|no}` enables you to toggle module precompilation on and off. When Julia is started with `--compiled-modules=no` the serialized modules in the compile cache -are ignored when loading modules and module dependencies. `Base.compilecache` can still be called +are ignored when loading modules and module dependencies. +More fine-grained control is available with `--pkgimage-native-code=no`, which suppresses only +native-code storage during precompilation. `Base.compilecache` can still be called manually. The state of this command line flag is passed to `Pkg.build` to disable automatic precompilation triggering when installing, updating, and explicitly building packages. + +You can also debug some precompilation failures with environment variables. Setting +`JULIA_VERBOSE_LINKING=true` may help resolve failures in linking shared libraries of [compiled +native code](@ref pkgimages). \ No newline at end of file diff --git a/doc/src/manual/performance-tips.md b/doc/src/manual/performance-tips.md index 6bfdce4fc411b..1f3b7bc06ae7f 100644 --- a/doc/src/manual/performance-tips.md +++ b/doc/src/manual/performance-tips.md @@ -525,7 +525,7 @@ at the time `k` is compiled. ### Be aware of when Julia avoids specializing -As a heuristic, Julia avoids automatically specializing on argument type parameters in three +As a heuristic, Julia avoids automatically [specializing](@ref man-method-specializations) on argument type parameters in three specific cases: `Type`, `Function`, and `Vararg`. Julia will always specialize when the argument is used within the method, but not if the argument is just passed through to another function. This usually has no performance impact at runtime and diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 26ba66fa96737..5e9fa1efab03b 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -61,7 +61,9 @@ using namespace llvm; #include "jitlayers.h" +#include "serialize.h" #include "julia_assert.h" +#include "codegen_shared.h" #define DEBUG_TYPE "julia_aotcompile" @@ -93,6 +95,7 @@ typedef struct { std::vector jl_sysimg_gvars; std::map> jl_fvar_map; std::vector jl_value_to_llvm; + std::vector jl_external_to_llvm; } jl_native_code_desc_t; extern "C" JL_DLLEXPORT @@ -118,6 +121,15 @@ void jl_get_llvm_gvs_impl(void *native_code, arraylist_t *gvs) memcpy(gvs->items, data->jl_value_to_llvm.data(), gvs->len * sizeof(void*)); } +extern "C" JL_DLLEXPORT +void jl_get_llvm_external_fns_impl(void *native_code, arraylist_t *external_fns) +{ + jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; + arraylist_grow(external_fns, data->jl_external_to_llvm.size()); + memcpy(external_fns->items, data->jl_external_to_llvm.data(), + external_fns->len * sizeof(jl_code_instance_t*)); +} + extern "C" JL_DLLEXPORT LLVMOrcThreadSafeModuleRef jl_get_llvm_module_impl(void *native_code) { @@ -248,13 +260,17 @@ static void jl_ci_cache_lookup(const jl_cgparams_t &cgparams, jl_method_instance *ci_out = codeinst; } +void replaceUsesWithLoad(Function &F, function_ref should_replace, MDNode *tbaa_const); + // takes the running content that has collected in the shadow module and dump it to disk // this builds the object file portion of the sysimage files for fast startup, and can // also be used be extern consumers like GPUCompiler.jl to obtain a module containing -// all reachable & inferrrable functions. The `policy` flag switches between the default -// mode `0`, the extern mode `1`. +// all reachable & inferrrable functions. +// The `policy` flag switches between the default mode `0` and the extern mode `1` used by GPUCompiler. +// `_imaging_mode` controls if raw pointers can be embedded (e.g. the code will be loaded into the same session). +// `_external_linkage` create linkages between pkgimages. extern "C" JL_DLLEXPORT -void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _policy, int _imaging_mode) +void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _policy, int _imaging_mode, int _external_linkage) { ++CreateNativeCalls; CreateNativeMax.updateMax(jl_array_len(methods)); @@ -286,6 +302,7 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm compiler_start_time = jl_hrtime(); params.imaging = imaging; + params.external_linkage = _external_linkage; // compile all methods for the current world and type-inference world size_t compile_for[] = { jl_typeinf_world, jl_atomic_load_acquire(&jl_world_counter) }; @@ -344,6 +361,39 @@ void *jl_create_native_impl(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvm } CreateNativeMethods += emitted.size(); + size_t offset = gvars.size(); + data->jl_external_to_llvm.resize(params.external_fns.size()); + + auto tbaa_const = tbaa_make_child_with_context(*ctxt.getContext(), "jtbaa_const", nullptr, true).first; + for (auto &extern_fn : params.external_fns) { + jl_code_instance_t *this_code = std::get<0>(extern_fn.first); + bool specsig = std::get<1>(extern_fn.first); + assert(specsig && "Error external_fns doesn't handle non-specsig yet"); + (void)specsig; + Function *F = extern_fn.second; + Module *M = F->getParent(); + + Type *T_funcp = F->getFunctionType()->getPointerTo(); + // Can't create a GC with type FunctionType. Alias also doesn't work + GlobalVariable *GV = new GlobalVariable(*M, T_funcp, false, + GlobalVariable::ExternalLinkage, + Constant::getNullValue(T_funcp), + F->getName()); + + + // Need to insert load instruction, thus we can't use replace all uses with + replaceUsesWithLoad(*F, [GV](Instruction &) { return GV; }, tbaa_const); + + assert(F->getNumUses() == 0); // declaration counts as use + GV->takeName(F); + F->eraseFromParent(); + + size_t idx = gvars.size() - offset; + assert(idx >= 0); + data->jl_external_to_llvm.at(idx) = this_code; + gvars.push_back(std::string(GV->getName())); + } + // clones the contents of the module `m` to the shadow_output collector // while examining and recording what kind of function pointer we have for (auto &def : emitted) { @@ -474,7 +524,7 @@ extern "C" JL_DLLEXPORT void jl_dump_native_impl(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, - const char *sysimg_data, size_t sysimg_len) + const char *sysimg_data, size_t sysimg_len, ios_t *s) { JL_TIMING(NATIVE_DUMP); jl_native_code_desc_t *data = (jl_native_code_desc_t*)native_code; @@ -586,7 +636,7 @@ void jl_dump_native_impl(void *native_code, } // do the actual work - auto add_output = [&] (Module &M, StringRef unopt_bc_Name, StringRef bc_Name, StringRef obj_Name, StringRef asm_Name) { + auto add_output = [&] (Module &M, StringRef unopt_bc_Name, StringRef bc_Name, StringRef obj_Name, StringRef asm_Name, bool inject_crt) { preopt.run(M, empty.MAM); if (bc_fname || obj_fname || asm_fname) { assert(!verifyModule(M, &errs())); @@ -594,22 +644,49 @@ void jl_dump_native_impl(void *native_code, assert(!verifyModule(M, &errs())); } - // We would like to emit an alias or an weakref alias to redirect these symbols - // but LLVM doesn't let us emit a GlobalAlias to a declaration... - // So for now we inject a definition of these functions that calls our runtime - // functions. We do so after optimization to avoid cloning these functions. - injectCRTAlias(M, "__gnu_h2f_ieee", "julia__gnu_h2f_ieee", - FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); - injectCRTAlias(M, "__extendhfsf2", "julia__gnu_h2f_ieee", - FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); - injectCRTAlias(M, "__gnu_f2h_ieee", "julia__gnu_f2h_ieee", - FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); - injectCRTAlias(M, "__truncsfhf2", "julia__gnu_f2h_ieee", - FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); - injectCRTAlias(M, "__truncdfhf2", "julia__truncdfhf2", - FunctionType::get(Type::getHalfTy(Context), { Type::getDoubleTy(Context) }, false)); + if (inject_crt) { + // We would like to emit an alias or an weakref alias to redirect these symbols + // but LLVM doesn't let us emit a GlobalAlias to a declaration... + // So for now we inject a definition of these functions that calls our runtime + // functions. We do so after optimization to avoid cloning these functions. + injectCRTAlias(M, "__gnu_h2f_ieee", "julia__gnu_h2f_ieee", + FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); + injectCRTAlias(M, "__extendhfsf2", "julia__gnu_h2f_ieee", + FunctionType::get(Type::getFloatTy(Context), { Type::getHalfTy(Context) }, false)); + injectCRTAlias(M, "__gnu_f2h_ieee", "julia__gnu_f2h_ieee", + FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); + injectCRTAlias(M, "__truncsfhf2", "julia__gnu_f2h_ieee", + FunctionType::get(Type::getHalfTy(Context), { Type::getFloatTy(Context) }, false)); + injectCRTAlias(M, "__truncdfhf2", "julia__truncdfhf2", + FunctionType::get(Type::getHalfTy(Context), { Type::getDoubleTy(Context) }, false)); + +#if defined(_OS_WINDOWS_) + // Windows expect that the function `_DllMainStartup` is present in an dll. + // Normal compilers use something like Zig's crtdll.c instead we provide a + // a stub implementation. + auto T_pvoid = Type::getInt8Ty(Context)->getPointerTo(); + auto T_int32 = Type::getInt32Ty(Context); + auto FT = FunctionType::get(T_int32, {T_pvoid, T_int32, T_pvoid}, false); + auto F = Function::Create(FT, Function::ExternalLinkage, "_DllMainCRTStartup", M); + F->setCallingConv(CallingConv::X86_StdCall); + + llvm::IRBuilder<> builder(BasicBlock::Create(M.getContext(), "top", F)); + builder.CreateRet(ConstantInt::get(T_int32, 1)); +#endif + } postopt.run(M, empty.MAM); + + // Get target by snooping on multiversioning + GlobalVariable *target_ids = M.getNamedGlobal("jl_dispatch_target_ids"); + if (s && target_ids) { + if(auto targets = dyn_cast(target_ids->getInitializer())) { + auto rawTargets = targets->getRawDataValues(); + write_int32(s, rawTargets.size()); + ios_write(s, rawTargets.data(), rawTargets.size()); + }; + } + emitter.run(M); if (unopt_bc_fname) @@ -622,7 +699,7 @@ void jl_dump_native_impl(void *native_code, emit_result(asm_Archive, asm_Buffer, asm_Name, outputs); }; - add_output(*dataM, "unopt.bc", "text.bc", "text.o", "text.s"); + add_output(*dataM, "unopt.bc", "text.bc", "text.o", "text.s", true); orc::ThreadSafeModule sysimage(std::make_unique("sysimage", Context), TSCtx); auto sysimageM = sysimage.getModuleUnlocked(); @@ -645,7 +722,7 @@ void jl_dump_native_impl(void *native_code, GlobalVariable::ExternalLinkage, len, "jl_system_image_size")); } - add_output(*sysimageM, "data.bc", "data.bc", "data.o", "data.s"); + add_output(*sysimageM, "data.bc", "data.bc", "data.o", "data.s", false); object::Archive::Kind Kind = getDefaultForHost(TheTriple); if (unopt_bc_fname) diff --git a/src/codegen-stubs.c b/src/codegen-stubs.c index 01324e349f08f..e7b7d1fb791a5 100644 --- a/src/codegen-stubs.c +++ b/src/codegen-stubs.c @@ -12,8 +12,9 @@ JL_DLLEXPORT void jl_dump_native_fallback(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, - const char *sysimg_data, size_t sysimg_len) UNAVAILABLE + const char *sysimg_data, size_t sysimg_len, ios_t *s) UNAVAILABLE JL_DLLEXPORT void jl_get_llvm_gvs_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE +JL_DLLEXPORT void jl_get_llvm_external_fns_fallback(void *native_code, arraylist_t *gvs) UNAVAILABLE JL_DLLEXPORT void jl_extern_c_fallback(jl_function_t *f, jl_value_t *rt, jl_value_t *argt, char *name) UNAVAILABLE JL_DLLEXPORT jl_value_t *jl_dump_method_asm_fallback(jl_method_instance_t *linfo, size_t world, @@ -31,10 +32,10 @@ JL_DLLEXPORT int jl_getFunctionInfo_fallback(jl_frame_t **frames, uintptr_t poin return 0; } -JL_DLLEXPORT void jl_register_fptrs_fallback(uint64_t sysimage_base, const struct _jl_sysimg_fptrs_t *fptrs, +JL_DLLEXPORT void jl_register_fptrs_fallback(uint64_t image_base, const struct _jl_image_fptrs_t *fptrs, jl_method_instance_t **linfos, size_t n) { - (void)sysimage_base; (void)fptrs; (void)linfos; (void)n; + (void)image_base; (void)fptrs; (void)linfos; (void)n; } JL_DLLEXPORT jl_code_instance_t *jl_generate_fptr_fallback(jl_method_instance_t *mi JL_PROPAGATES_ROOT, size_t world) @@ -66,7 +67,7 @@ JL_DLLEXPORT size_t jl_jit_total_bytes_fallback(void) return 0; } -JL_DLLEXPORT void *jl_create_native_fallback(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _policy, int _imaging_mode) UNAVAILABLE +JL_DLLEXPORT void *jl_create_native_fallback(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int _policy, int _imaging_mode, int _external_linkage) UNAVAILABLE JL_DLLEXPORT void jl_dump_compiles_fallback(void *s) { diff --git a/src/codegen.cpp b/src/codegen.cpp index 024f30ad576e9..32a21aa10a556 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -1419,6 +1419,7 @@ class jl_codectx_t { jl_codegen_params_t &emission_context; llvm::MapVector call_targets; std::map &global_targets; + std::map, Function*> &external_calls; Function *f = NULL; // local var info. globals are not in here. std::vector slots; @@ -1455,6 +1456,7 @@ class jl_codectx_t { bool debug_enabled = false; bool use_cache = false; + bool external_linkage = false; const jl_cgparams_t *params = NULL; std::vector llvmcall_modules; @@ -1464,8 +1466,10 @@ class jl_codectx_t { emission_context(params), call_targets(), global_targets(params.globals), + external_calls(params.external_fns), world(params.world), use_cache(params.cache), + external_linkage(params.external_linkage), params(params.params) { } jl_typecache_t &types() { @@ -4028,9 +4032,18 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const std::string name; StringRef protoname; bool need_to_emit = true; - // TODO: We should check if the code is available externally - // and then emit a trampoline. - if (ctx.use_cache) { + bool cache_valid = ctx.use_cache; + bool external = false; + if (ctx.external_linkage) { + if (jl_object_in_image((jl_value_t*)codeinst)) { + // Target is present in another pkgimage + jl_printf(JL_STDERR, "\n (emit_invoke:) Want to resolve method!\n"); + cache_valid = true; + external = true; + } + } + + if (cache_valid) { // optimization: emit the correct name immediately, if we know it // TODO: use `emitted` map here too to try to consolidate names? // WARNING: isspecsig is protected by the codegen-lock. If that lock is removed, then the isspecsig load needs to be properly atomically sequenced with this. @@ -4058,6 +4071,13 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const result = emit_call_specfun_other(ctx, mi, codeinst->rettype, protoname, argv, nargs, &cc, &return_roots, rt); else result = emit_call_specfun_boxed(ctx, codeinst->rettype, protoname, argv, nargs, rt); + if (external) { + assert(!need_to_emit); + auto calledF = jl_Module->getFunction(protoname); + assert(calledF); + // TODO: Check if already present? + ctx.external_calls[std::make_tuple(codeinst, specsig)] = calledF; + } handled = true; if (need_to_emit) { Function *trampoline_decl = cast(jl_Module->getNamedValue(protoname)); @@ -5375,7 +5395,17 @@ static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Module *M, jl_cod Function *theFunc; Value *theFarg; auto invoke = jl_atomic_load_relaxed(&codeinst->invoke); - if (params.cache && invoke != NULL) { + + bool cache_valid = params.cache; + if (params.external_linkage) { + if (jl_object_in_image((jl_value_t*)codeinst)) { + // Target is present in another pkgimage + jl_printf(JL_STDERR, "\n (emit_jlinvoke) Want to resolve method\n"); + cache_valid = true; + } + } + + if (cache_valid && invoke != NULL) { StringRef theFptrName = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)invoke, codeinst); theFunc = cast( M->getOrInsertFunction(theFptrName, jlinvoke_func->_type(ctx.builder.getContext())).getCallee()); @@ -8272,12 +8302,12 @@ void jl_compile_workqueue( StringRef preal_decl = ""; bool preal_specsig = false; auto invoke = jl_atomic_load_relaxed(&codeinst->invoke); - // TODO: available_extern - // We need to emit a trampoline that loads the target address in an extern_module from a GV - // Right now we will unecessarily emit a function we have already compiled in a native module - // again in a calling module. + bool cache_valid = params.cache; + if (params.external_linkage) { + cache_valid = jl_object_in_image((jl_value_t*)codeinst); + } // WARNING: isspecsig is protected by the codegen-lock. If that lock is removed, then the isspecsig load needs to be properly atomically sequenced with this. - if (params.cache && invoke != NULL) { + if (cache_valid && invoke != NULL) { auto fptr = jl_atomic_load_relaxed(&codeinst->specptr.fptr); if (invoke == jl_fptr_args_addr) { preal_decl = jl_ExecutionEngine->getFunctionAtAddress((uintptr_t)fptr, codeinst); diff --git a/src/coverage.cpp b/src/coverage.cpp index 46363a7e9ac01..0dfb903798bfa 100644 --- a/src/coverage.cpp +++ b/src/coverage.cpp @@ -17,7 +17,7 @@ using namespace llvm; static int codegen_imaging_mode(void) { - return jl_options.image_codegen || (jl_generating_output() && !jl_options.incremental); + return jl_options.image_codegen || jl_generating_output(); } // Logging for code coverage and memory allocation diff --git a/src/debug-registry.h b/src/debug-registry.h index 3780bbee33718..165f0efa479e3 100644 --- a/src/debug-registry.h +++ b/src/debug-registry.h @@ -81,11 +81,11 @@ class JITDebugInfoRegistry ~Locked() JL_NOTSAFEPOINT = default; }; - struct sysimg_info_t { - uint64_t jl_sysimage_base; - jl_sysimg_fptrs_t sysimg_fptrs; - jl_method_instance_t **sysimg_fvars_linfo; - size_t sysimg_fvars_n; + struct image_info_t { + uint64_t base; + jl_image_fptrs_t fptrs; + jl_method_instance_t **fvars_linfo; + size_t fvars_n; }; struct libc_frames_t { @@ -122,7 +122,7 @@ class JITDebugInfoRegistry // that it came from (providing name, type signature, file info, etc.) Locked> codeinst_in_flight{}; - Locked sysimg_info{}; + Locked> image_info{}; Locked objfilemap{}; @@ -141,7 +141,7 @@ class JITDebugInfoRegistry std::function getLoadAddress, std::function lookupWriteAddress) JL_NOTSAFEPOINT; objectmap_t& getObjectMap() JL_NOTSAFEPOINT; - void set_sysimg_info(sysimg_info_t info) JL_NOTSAFEPOINT; - Locked::ConstLockT get_sysimg_info() const JL_NOTSAFEPOINT; + void add_image_info(image_info_t info) JL_NOTSAFEPOINT; + bool get_image_info(uint64_t base, image_info_t *info) const JL_NOTSAFEPOINT; Locked::LockT get_objfile_map() JL_NOTSAFEPOINT; }; diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index fe5614100f9e3..a4d1187077fa5 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -109,13 +109,19 @@ JITDebugInfoRegistry::getObjectMap() JL_NOTSAFEPOINT return objectmap; } -void JITDebugInfoRegistry::set_sysimg_info(sysimg_info_t info) JL_NOTSAFEPOINT { - (**this->sysimg_info) = info; +void JITDebugInfoRegistry::add_image_info(image_info_t info) JL_NOTSAFEPOINT { + (**this->image_info)[info.base] = info; } -JITDebugInfoRegistry::Locked::ConstLockT -JITDebugInfoRegistry::get_sysimg_info() const JL_NOTSAFEPOINT { - return *this->sysimg_info; + +bool JITDebugInfoRegistry::get_image_info(uint64_t base, JITDebugInfoRegistry::image_info_t *info) const JL_NOTSAFEPOINT { + auto infos = *this->image_info; + auto it = infos->find(base); + if (it != infos->end()) { + *info = it->second; + return true; + } + return false; } JITDebugInfoRegistry::Locked::LockT @@ -680,10 +686,10 @@ openDebugInfo(StringRef debuginfopath, const debug_link_info &info) std::move(SplitFile.get())); } extern "C" JL_DLLEXPORT -void jl_register_fptrs_impl(uint64_t sysimage_base, const jl_sysimg_fptrs_t *fptrs, +void jl_register_fptrs_impl(uint64_t image_base, const jl_image_fptrs_t *fptrs, jl_method_instance_t **linfos, size_t n) { - getJITDebugRegistry().set_sysimg_info({(uintptr_t) sysimage_base, *fptrs, linfos, n}); + getJITDebugRegistry().add_image_info({(uintptr_t) image_base, *fptrs, linfos, n}); } template @@ -694,12 +700,9 @@ static inline void ignoreError(T &err) JL_NOTSAFEPOINT #endif } -static void get_function_name_and_base(llvm::object::SectionRef Section, size_t pointer, int64_t slide, bool insysimage, +static void get_function_name_and_base(llvm::object::SectionRef Section, size_t pointer, int64_t slide, bool inimage, void **saddr, char **name, bool untrusted_dladdr) JL_NOTSAFEPOINT { - // Assume we only need base address for sysimg for now - if (!insysimage || !getJITDebugRegistry().get_sysimg_info()->sysimg_fptrs.base) - saddr = nullptr; bool needs_saddr = saddr && (!*saddr || untrusted_dladdr); bool needs_name = name && (!*name || untrusted_dladdr); // Try platform specific methods first since they are usually faster @@ -780,7 +783,7 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t } #ifdef _OS_WINDOWS_ // For ntdll and msvcrt since we are currently only parsing DWARF debug info through LLVM - if (!insysimage && needs_name) { + if (!inimage && needs_name) { static char frame_info_func[ sizeof(SYMBOL_INFO) + MAX_SYM_NAME * sizeof(TCHAR)]; @@ -1012,7 +1015,7 @@ static object::SectionRef getModuleSectionForAddress(const object::ObjectFile *o bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t *slide, llvm::DIContext **context, - bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename) JL_NOTSAFEPOINT + bool onlyImage, bool *isImage, uint64_t *_fbase, void **saddr, char **name, char **filename) JL_NOTSAFEPOINT { *Section = object::SectionRef(); *context = NULL; @@ -1046,10 +1049,11 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * if (fname.empty()) // empirically, LoadedImageName might be missing fname = ModuleInfo.ImageName; DWORD64 fbase = ModuleInfo.BaseOfImage; - bool insysimage = (fbase == getJITDebugRegistry().get_sysimg_info()->jl_sysimage_base); - if (isSysImg) - *isSysImg = insysimage; - if (onlySysImg && !insysimage) + JITDebugInfoRegistry::image_info_t image_info; + bool inimage = getJITDebugRegistry().get_image_info(fbase, &image_info); + if (isImage) + *isImage = inimage; + if (onlyImage && !inimage) return false; // If we didn't find the filename before in the debug // info, use the dll name @@ -1057,6 +1061,8 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * jl_copy_str(filename, fname.data()); if (saddr) *saddr = NULL; + if (_fbase) + *_fbase = fbase; #else // ifdef _OS_WINDOWS_ Dl_info dlinfo; @@ -1095,16 +1101,19 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * fbase = (uintptr_t)dlinfo.dli_fbase; #endif StringRef fname; - bool insysimage = (fbase == getJITDebugRegistry().get_sysimg_info()->jl_sysimage_base); - if (saddr && !(insysimage && untrusted_dladdr)) + JITDebugInfoRegistry::image_info_t image_info; + bool inimage = getJITDebugRegistry().get_image_info(fbase, &image_info); + if (saddr && !(inimage && untrusted_dladdr)) *saddr = dlinfo.dli_saddr; - if (isSysImg) - *isSysImg = insysimage; - if (onlySysImg && !insysimage) + if (isImage) + *isImage = inimage; + if (onlyImage && !inimage) return false; + if (_fbase) + *_fbase = fbase; // In case we fail with the debug info lookup, we at least still // have the function name, even if we don't have line numbers - if (name && !(insysimage && untrusted_dladdr)) + if (name && !(inimage && untrusted_dladdr)) jl_copy_str(name, dlinfo.dli_sname); if (filename) jl_copy_str(filename, dlinfo.dli_fname); @@ -1115,7 +1124,12 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * *context = entry.ctx; if (entry.obj) *Section = getModuleSectionForAddress(entry.obj, pointer + entry.slide); - get_function_name_and_base(*Section, pointer, entry.slide, insysimage, saddr, name, untrusted_dladdr); + + + // Assume we only need base address for sysimg for now + if (!inimage || !image_info.fptrs.base) + saddr = nullptr; + get_function_name_and_base(*Section, pointer, entry.slide, inimage, saddr, name, untrusted_dladdr); return true; } @@ -1144,34 +1158,36 @@ static int jl_getDylibFunctionInfo(jl_frame_t **frames, size_t pointer, int skip object::SectionRef Section; llvm::DIContext *context = NULL; int64_t slide; - bool isSysImg; + bool isImage; void *saddr; - if (!jl_dylib_DI_for_fptr(pointer, &Section, &slide, &context, skipC, &isSysImg, &saddr, &frame0->func_name, &frame0->file_name)) { + uint64_t fbase; + if (!jl_dylib_DI_for_fptr(pointer, &Section, &slide, &context, skipC, &isImage, &fbase, &saddr, &frame0->func_name, &frame0->file_name)) { frame0->fromC = 1; return 1; } - frame0->fromC = !isSysImg; + frame0->fromC = !isImage; { - auto sysimg_locked = getJITDebugRegistry().get_sysimg_info(); - if (isSysImg && sysimg_locked->sysimg_fptrs.base && saddr) { - intptr_t diff = (uintptr_t)saddr - (uintptr_t)sysimg_locked->sysimg_fptrs.base; - for (size_t i = 0; i < sysimg_locked->sysimg_fptrs.nclones; i++) { - if (diff == sysimg_locked->sysimg_fptrs.clone_offsets[i]) { - uint32_t idx = sysimg_locked->sysimg_fptrs.clone_idxs[i] & jl_sysimg_val_mask; - if (idx < sysimg_locked->sysimg_fvars_n) // items after this were cloned but not referenced directly by a method (such as our ccall PLT thunks) - frame0->linfo = sysimg_locked->sysimg_fvars_linfo[idx]; + JITDebugInfoRegistry::image_info_t image; + bool inimage = getJITDebugRegistry().get_image_info(fbase, &image); + if (isImage && saddr && inimage) { + intptr_t diff = (uintptr_t)saddr - (uintptr_t)image.fptrs.base; + for (size_t i = 0; i < image.fptrs.nclones; i++) { + if (diff == image.fptrs.clone_offsets[i]) { + uint32_t idx = image.fptrs.clone_idxs[i] & jl_sysimg_val_mask; + if (idx < image.fvars_n) // items after this were cloned but not referenced directly by a method (such as our ccall PLT thunks) + frame0->linfo = image.fvars_linfo[idx]; break; } } - for (size_t i = 0; i < sysimg_locked->sysimg_fvars_n; i++) { - if (diff == sysimg_locked->sysimg_fptrs.offsets[i]) { - frame0->linfo = sysimg_locked->sysimg_fvars_linfo[i]; + for (size_t i = 0; i < image.fvars_n; i++) { + if (diff == image.fptrs.offsets[i]) { + frame0->linfo = image.fvars_linfo[i]; break; } } } } - return lookup_pointer(Section, context, frames, pointer, slide, isSysImg, noInline); + return lookup_pointer(Section, context, frames, pointer, slide, isImage, noInline); } int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, diff --git a/src/debuginfo.h b/src/debuginfo.h index 5ea34350ac1fb..5b5cdcb82d534 100644 --- a/src/debuginfo.h +++ b/src/debuginfo.h @@ -6,7 +6,7 @@ int jl_DI_for_fptr(uint64_t fptr, uint64_t *symsize, int64_t *slide, llvm::object::SectionRef *Section, llvm::DIContext **context) JL_NOTSAFEPOINT; bool jl_dylib_DI_for_fptr(size_t pointer, llvm::object::SectionRef *Section, int64_t *slide, llvm::DIContext **context, - bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename) JL_NOTSAFEPOINT; + bool onlyImage, bool *isImage, uint64_t* fbase, void **saddr, char **name, char **filename) JL_NOTSAFEPOINT; static object::SectionedAddress makeAddress( llvm::object::SectionRef Section, uint64_t address) JL_NOTSAFEPOINT diff --git a/src/disasm.cpp b/src/disasm.cpp index cfc030f649fd6..5b510a24b33da 100644 --- a/src/disasm.cpp +++ b/src/disasm.cpp @@ -592,7 +592,7 @@ jl_value_t *jl_dump_fptr_asm_impl(uint64_t fptr, char raw_mc, const char* asm_va llvm::DIContext *context = NULL; if (!jl_DI_for_fptr(fptr, &symsize, &slide, &Section, &context)) { if (!jl_dylib_DI_for_fptr(fptr, &Section, &slide, &context, - false, NULL, NULL, NULL, NULL)) { + false, NULL, NULL, NULL, NULL, NULL)) { jl_printf(JL_STDERR, "WARNING: Unable to find function pointer\n"); return jl_pchar_to_string("", 0); } diff --git a/src/jitlayers.h b/src/jitlayers.h index 77ac5d64bb46d..bad5a91cad6c5 100644 --- a/src/jitlayers.h +++ b/src/jitlayers.h @@ -73,7 +73,7 @@ GlobalVariable *jl_emit_RTLD_DEFAULT_var(Module *M); DataLayout jl_create_datalayout(TargetMachine &TM); static inline bool imaging_default() { - return jl_options.image_codegen || (jl_generating_output() && !jl_options.incremental); + return jl_options.image_codegen || jl_generating_output(); } struct OptimizationOptions { @@ -173,6 +173,7 @@ typedef struct _jl_codegen_params_t { // outputs std::vector> workqueue; std::map globals; + std::map, Function*> external_fns; std::map ditypes; std::map llvmtypes; DenseMap mergedConstants; @@ -200,6 +201,7 @@ typedef struct _jl_codegen_params_t { size_t world = 0; const jl_cgparams_t *params = &jl_default_cgparams; bool cache = false; + bool external_linkage = false; bool imaging; _jl_codegen_params_t(orc::ThreadSafeContext ctx) : tsctx(std::move(ctx)), tsctx_lock(tsctx.getLock()), imaging(imaging_default()) {} } jl_codegen_params_t; diff --git a/src/jl_exported_data.inc b/src/jl_exported_data.inc index 6f0671ef0d6f7..c81ee410c9cd7 100644 --- a/src/jl_exported_data.inc +++ b/src/jl_exported_data.inc @@ -127,6 +127,8 @@ XX(jl_voidpointer_type) \ XX(jl_void_type) \ XX(jl_weakref_type) \ + XX(jl_build_ids) \ + XX(jl_linkage_blobs) \ // Data symbols that are defined inside the public libjulia #define JL_EXPORTED_DATA_SYMBOLS(XX) \ diff --git a/src/jl_exported_funcs.inc b/src/jl_exported_funcs.inc index 2e5df94dc7200..8ea77bfe12be3 100644 --- a/src/jl_exported_funcs.inc +++ b/src/jl_exported_funcs.inc @@ -118,6 +118,7 @@ XX(jl_dlopen) \ XX(jl_dlsym) \ XX(jl_dump_host_cpu) \ + XX(jl_check_pkgimage_clones) \ XX(jl_egal) \ XX(jl_egal__bits) \ XX(jl_egal__special) \ @@ -394,6 +395,8 @@ XX(jl_queue_work) \ XX(jl_raise_debugger) \ XX(jl_readuntil) \ + XX(jl_cache_flags) \ + XX(jl_match_cache_flags) \ XX(jl_read_verify_header) \ XX(jl_realloc) \ XX(jl_register_newmeth_tracer) \ @@ -536,6 +539,7 @@ YY(jl_get_LLVM_VERSION) \ YY(jl_dump_native) \ YY(jl_get_llvm_gvs) \ + YY(jl_get_llvm_external_fns) \ YY(jl_dump_function_asm) \ YY(jl_LLVMCreateDisasm) \ YY(jl_LLVMDisasmInstruction) \ diff --git a/src/jloptions.c b/src/jloptions.c index da362e0054b3e..dcb3c7532539b 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -70,6 +70,7 @@ JL_DLLEXPORT void jl_init_options(void) NULL, // cookie JL_OPTIONS_HANDLE_SIGNALS_ON, JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES, + JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_YES, JL_OPTIONS_USE_COMPILED_MODULES_YES, NULL, // bind-to NULL, // output-bc @@ -106,6 +107,8 @@ static const char opts[] = " --handle-signals={yes*|no} Enable or disable Julia's default signal handlers\n" " --sysimage-native-code={yes*|no}\n" " Use native code from system image if available\n" + " --pkgimage-native-code={yes*|no}\n" + " Use native code from package images if available\n" " --compiled-modules={yes*|no}\n" " Enable or disable incremental precompilation of modules\n\n" @@ -238,6 +241,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_help_hidden, opt_banner, opt_sysimage_native_code, + opt_pkgimage_native_code, opt_compiled_modules, opt_machine_file, opt_project, @@ -266,6 +270,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "bug-report", required_argument, 0, opt_bug_report }, { "sysimage", required_argument, 0, 'J' }, { "sysimage-native-code", required_argument, 0, opt_sysimage_native_code }, + { "pkgimage-native-code", required_argument, 0, opt_pkgimage_native_code }, { "compiled-modules",required_argument, 0, opt_compiled_modules }, { "cpu-target", required_argument, 0, 'C' }, { "procs", required_argument, 0, 'p' }, @@ -317,6 +322,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) const char **cmds = NULL; int codecov = JL_LOG_NONE; int malloclog = JL_LOG_NONE; + int pkgimage_explicit = 0; int argc = *argcp; char **argv = *argvp; char *endptr; @@ -436,6 +442,15 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) else jl_errorf("julia: invalid argument to --sysimage-native-code={yes|no} (%s)", optarg); break; + case opt_pkgimage_native_code: + pkgimage_explicit = 1; + if (!strcmp(optarg,"yes")) + jl_options.use_pkgimage_native_code = JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_YES; + else if (!strcmp(optarg,"no")) + jl_options.use_pkgimage_native_code = JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_NO; + else + jl_errorf("julia: invalid argument to --pkgimage-native-code={yes|no} (%s)", optarg); + break; case opt_compiled_modules: if (!strcmp(optarg,"yes")) jl_options.use_compiled_modules = JL_OPTIONS_USE_COMPILED_MODULES_YES; @@ -803,6 +818,13 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) "This is a bug, please report it.", c); } } + if (codecov || malloclog) { + if (pkgimage_explicit && jl_options.use_pkgimage_native_code) { + jl_errorf("julia: Can't use --pkgimage-native-code=yes together " + "with --track-allocation or --code-coverage."); + } + jl_options.use_pkgimage_native_code = 0; + } jl_options.code_coverage = codecov; jl_options.malloc_log = malloclog; int proc_args = *argcp < optind ? *argcp : optind; diff --git a/src/jloptions.h b/src/jloptions.h index d7be95348f01f..88a256f02e91e 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -41,6 +41,7 @@ typedef struct { const char *cookie; int8_t handle_signals; int8_t use_sysimage_native_code; + int8_t use_pkgimage_native_code; int8_t use_compiled_modules; const char *bindto; const char *outputbc; diff --git a/src/julia.h b/src/julia.h index 2c9be8ae1aa2a..95c3c7e6bf768 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1760,7 +1760,7 @@ JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void); JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s); JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname); JL_DLLEXPORT void jl_set_sysimg_so(void *handle); -JL_DLLEXPORT ios_t *jl_create_system_image(void *, jl_array_t *worklist); +JL_DLLEXPORT void jl_create_system_image(void *, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos); JL_DLLEXPORT void jl_restore_system_image(const char *fname); JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int complete); @@ -2178,6 +2178,9 @@ JL_DLLEXPORT int jl_generating_output(void) JL_NOTSAFEPOINT; #define JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES 1 #define JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_NO 0 +#define JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_YES 1 +#define JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_NO 0 + #define JL_OPTIONS_USE_COMPILED_MODULES_YES 1 #define JL_OPTIONS_USE_COMPILED_MODULES_NO 0 diff --git a/src/julia_internal.h b/src/julia_internal.h index 1e59cf6f18b5a..142551d202f2c 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -960,7 +960,7 @@ STATIC_INLINE size_t n_linkage_blobs(void) JL_NOTSAFEPOINT } // TODO: Makes this a binary search -STATIC_INLINE size_t external_blob_index(jl_value_t *v) JL_NOTSAFEPOINT { +STATIC_INLINE size_t external_blob_index(jl_value_t* v) JL_NOTSAFEPOINT { size_t i, nblobs = n_linkage_blobs(); assert(jl_linkage_blobs.len == 2*nblobs); for (i = 0; i < nblobs; i++) { @@ -994,11 +994,12 @@ JL_DLLEXPORT jl_value_t *jl_dump_fptr_asm(uint64_t fptr, char raw_mc, const char JL_DLLEXPORT jl_value_t *jl_dump_function_ir(jl_llvmf_dump_t *dump, char strip_ir_metadata, char dump_module, const char *debuginfo); JL_DLLEXPORT jl_value_t *jl_dump_function_asm(jl_llvmf_dump_t *dump, char raw_mc, const char* asm_variant, const char *debuginfo, char binary); -void *jl_create_native(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int policy, int imaging_mode); +void *jl_create_native(jl_array_t *methods, LLVMOrcThreadSafeModuleRef llvmmod, const jl_cgparams_t *cgparams, int policy, int imaging_mode, int cache); void jl_dump_native(void *native_code, const char *bc_fname, const char *unopt_bc_fname, const char *obj_fname, const char *asm_fname, - const char *sysimg_data, size_t sysimg_len); + const char *sysimg_data, size_t sysimg_len, ios_t *s); void jl_get_llvm_gvs(void *native_code, arraylist_t *gvs); +void jl_get_llvm_external_fns(void *native_code, arraylist_t *gvs); JL_DLLEXPORT void jl_get_function_id(void *native_code, jl_code_instance_t *ncode, int32_t *func_idx, int32_t *specfunc_idx); @@ -1620,9 +1621,9 @@ extern JL_DLLEXPORT jl_sym_t *jl_sequentially_consistent_sym; JL_DLLEXPORT enum jl_memory_order jl_get_atomic_order(jl_sym_t *order, char loading, char storing); JL_DLLEXPORT enum jl_memory_order jl_get_atomic_order_checked(jl_sym_t *order, char loading, char storing); -struct _jl_sysimg_fptrs_t; +struct _jl_image_fptrs_t; -void jl_register_fptrs(uint64_t sysimage_base, const struct _jl_sysimg_fptrs_t *fptrs, +void jl_register_fptrs(uint64_t image_base, const struct _jl_image_fptrs_t *fptrs, jl_method_instance_t **linfos, size_t n); void jl_write_coverage_data(const char*); void jl_write_malloc_log(void); diff --git a/src/llvm-multiversioning.cpp b/src/llvm-multiversioning.cpp index 815ebfe7ed101..97867f1d9f471 100644 --- a/src/llvm-multiversioning.cpp +++ b/src/llvm-multiversioning.cpp @@ -47,6 +47,8 @@ extern Optional always_have_fma(Function&); extern Optional always_have_fp16(); +void replaceUsesWithLoad(Function &F, function_ref should_replace, MDNode *tbaa_const); + namespace { constexpr uint32_t clone_mask = JL_TARGET_CLONE_LOOP | JL_TARGET_CLONE_SIMD | JL_TARGET_CLONE_MATH | JL_TARGET_CLONE_CPU; @@ -266,8 +268,6 @@ struct CloneCtx { uint32_t get_func_id(Function *F); template Constant *rewrite_gv_init(const Stack& stack); - template - Value *rewrite_inst_use(const Stack& stack, Value *replace, Instruction *insert_before); std::pair get_reloc_slot(Function *F); Constant *get_ptrdiff32(Constant *ptr, Constant *base) const; template @@ -820,7 +820,7 @@ std::pair CloneCtx::get_reloc_slot(Function *F) } template -Value *CloneCtx::rewrite_inst_use(const Stack& stack, Value *replace, Instruction *insert_before) +static Value *rewrite_inst_use(const Stack& stack, Value *replace, Instruction *insert_before) { SmallVector args; uint32_t nlevel = stack.size(); @@ -879,40 +879,24 @@ void CloneCtx::fix_inst_uses() continue; auto orig_f = orig_funcs[i]; auto F = grp.base_func(orig_f); - bool changed; - do { - changed = false; - for (auto uses = ConstantUses(F, M); !uses.done(); uses.next()) { - auto info = uses.get_info(); - auto use_i = info.val; - auto use_f = use_i->getFunction(); - if (!use_f->getName().endswith(suffix)) + replaceUsesWithLoad(*F, [&](Instruction &I) -> GlobalVariable * { + uint32_t id; + GlobalVariable *slot; + auto use_f = I.getFunction(); + if (!use_f->getName().endswith(suffix)) + return nullptr; + std::tie(id, slot) = get_reloc_slot(orig_f); + + grp.relocs.insert(id); + for (auto &tgt: grp.clones) { + // The enclosing function of the use is cloned, + // no need to deal with this use on this target. + if (map_get(*tgt.vmap, use_f)) continue; - Instruction *insert_before = use_i; - if (auto phi = dyn_cast(use_i)) - insert_before = phi->getIncomingBlock(*info.use)->getTerminator(); - uint32_t id; - GlobalVariable *slot; - std::tie(id, slot) = get_reloc_slot(orig_f); - Instruction *ptr = new LoadInst(orig_f->getType(), slot, "", false, insert_before); - ptr->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_const); - ptr->setMetadata(llvm::LLVMContext::MD_invariant_load, MDNode::get(ptr->getContext(), None)); - use_i->setOperand(info.use->getOperandNo(), - rewrite_inst_use(uses.get_stack(), ptr, - insert_before)); - - grp.relocs.insert(id); - for (auto &tgt: grp.clones) { - // The enclosing function of the use is cloned, - // no need to deal with this use on this target. - if (map_get(*tgt.vmap, use_f)) - continue; - tgt.relocs.insert(id); - } - - changed = true; + tgt.relocs.insert(id); } - } while (changed); + return slot; + }, tbaa_const); } } } @@ -1202,6 +1186,30 @@ static RegisterPass X("JuliaMultiVersioning", "JuliaMulti } // anonymous namespace +void replaceUsesWithLoad(Function &F, function_ref should_replace, MDNode *tbaa_const) { + bool changed; + do { + changed = false; + for (auto uses = ConstantUses(&F, *F.getParent()); !uses.done(); uses.next()) { + auto info = uses.get_info(); + auto use_i = info.val; + GlobalVariable *slot = should_replace(*use_i); + if (!slot) + continue; + Instruction *insert_before = use_i; + if (auto phi = dyn_cast(use_i)) + insert_before = phi->getIncomingBlock(*info.use)->getTerminator(); + Instruction *ptr = new LoadInst(F.getType(), slot, "", false, insert_before); + ptr->setMetadata(llvm::LLVMContext::MD_tbaa, tbaa_const); + ptr->setMetadata(llvm::LLVMContext::MD_invariant_load, MDNode::get(ptr->getContext(), None)); + use_i->setOperand(info.use->getOperandNo(), + rewrite_inst_use(uses.get_stack(), ptr, + insert_before)); + changed = true; + } + } while (changed); +} + PreservedAnalyses MultiVersioning::run(Module &M, ModuleAnalysisManager &AM) { auto &FAM = AM.getResult(M).getManager(); diff --git a/src/precompile.c b/src/precompile.c index ebe7afae69f64..a70f7cd882bc0 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -10,6 +10,7 @@ #include "julia.h" #include "julia_internal.h" #include "julia_assert.h" +#include "serialize.h" #ifdef __cplusplus extern "C" { @@ -23,6 +24,54 @@ JL_DLLEXPORT int jl_generating_output(void) static void *jl_precompile(int all); static void *jl_precompile_worklist(jl_array_t *worklist); +void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { + // Write the source-text for the dependent files + if (udeps) { + // Go back and update the source-text position to point to the current position + int64_t posfile = ios_pos(f); + ios_seek(f, srctextpos); + write_uint64(f, posfile); + ios_seek_end(f); + // Each source-text file is written as + // int32: length of abspath + // char*: abspath + // uint64: length of src text + // char*: src text + // At the end we write int32(0) as a terminal sentinel. + size_t len = jl_array_len(udeps); + ios_t srctext; + for (size_t i = 0; i < len; i++) { + jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); + jl_value_t *depmod = jl_fieldref(deptuple, 0); // module + // Dependencies declared with `include_dependency` are excluded + // because these may not be Julia code (and could be huge) + if (depmod != (jl_value_t*)jl_main_module) { + jl_value_t *dep = jl_fieldref(deptuple, 1); // file abspath + const char *depstr = jl_string_data(dep); + if (!depstr[0]) + continue; + ios_t *srctp = ios_file(&srctext, depstr, 1, 0, 0, 0); + if (!srctp) { + jl_printf(JL_STDERR, "WARNING: could not cache source text for \"%s\".\n", + jl_string_data(dep)); + continue; + } + size_t slen = jl_string_len(dep); + write_int32(f, slen); + ios_write(f, depstr, slen); + posfile = ios_pos(f); + write_uint64(f, 0); // placeholder for length of this file in bytes + uint64_t filelen = (uint64_t) ios_copyall(f, &srctext); + ios_close(&srctext); + ios_seek(f, posfile); + write_uint64(f, filelen); + ios_seek_end(f); + } + } + } + write_int32(f, 0); // mark the end of the source text +} + JL_DLLEXPORT void jl_write_compiler_output(void) { if (!jl_generating_output()) { @@ -35,7 +84,8 @@ JL_DLLEXPORT void jl_write_compiler_output(void) } jl_array_t *worklist = jl_module_init_order; - JL_GC_PUSH1(&worklist); + jl_array_t *udeps = NULL; + JL_GC_PUSH2(&worklist, &udeps); jl_module_init_order = jl_alloc_vec_any(0); int i, l = jl_array_len(worklist); for (i = 0; i < l; i++) { @@ -67,41 +117,53 @@ JL_DLLEXPORT void jl_write_compiler_output(void) jl_precompile_toplevel_module = NULL; } - if (jl_options.incremental) { - if (jl_options.outputbc || jl_options.outputunoptbc) - jl_printf(JL_STDERR, "WARNING: incremental output to a .bc file is not implemented\n"); - if (jl_options.outputasm) - jl_printf(JL_STDERR, "WARNING: incremental output to a .s file is not implemented\n"); - if (jl_options.outputo) { - jl_printf(JL_STDERR, "WARNING: incremental output to a .o file is not implemented\n"); - } - } + bool_t emit_native = jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm; - ios_t *s = jl_create_system_image(native_code, jl_options.incremental ? worklist : NULL); + bool_t emit_split = jl_options.outputji && emit_native; - if (jl_options.outputji) { - ios_t f; - if (ios_file(&f, jl_options.outputji, 1, 1, 1, 1) == NULL) - jl_errorf("cannot open system image file \"%s\" for writing", jl_options.outputji); - ios_write(&f, (const char*)s->buf, (size_t)s->size); - ios_close(&f); - } + ios_t *s = NULL; + ios_t *z = NULL; + int64_t srctextpos = 0 ; + jl_create_system_image(native_code, jl_options.incremental ? worklist : NULL, emit_split, + &s, &z, &udeps, &srctextpos); + + if (!emit_split) + z = s; + // jl_dump_native writes the clone_targets into `s` + // We need to postpone the srctext writing after that. if (native_code) { jl_dump_native(native_code, jl_options.outputbc, jl_options.outputunoptbc, jl_options.outputo, jl_options.outputasm, - (const char*)s->buf, (size_t)s->size); + (const char*)z->buf, (size_t)z->size, s); jl_postoutput_hook(); } + if ((jl_options.outputji || emit_native) && jl_options.incremental) { + write_srctext(s, udeps, srctextpos); + } + + if (jl_options.outputji) { + ios_t f; + if (ios_file(&f, jl_options.outputji, 1, 1, 1, 1) == NULL) + jl_errorf("cannot open system image file \"%s\" for writing", jl_options.outputji); + ios_write(&f, (const char*)s->buf, (size_t)s->size); + ios_close(&f); + } + if (s) { ios_close(s); free(s); } + if (emit_split) { + ios_close(z); + free(z); + } + for (size_t i = 0; i < jl_current_modules.size; i += 2) { if (jl_current_modules.table[i + 1] != HT_NOTFOUND) { jl_printf(JL_STDERR, "\nWARNING: detected unclosed module: "); @@ -342,7 +404,7 @@ static int precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), precompile_enq_all_specializations__, env); } -static void *jl_precompile_(jl_array_t *m) +static void *jl_precompile_(jl_array_t *m, int external_linkage) { jl_array_t *m2 = NULL; jl_method_instance_t *mi = NULL; @@ -365,7 +427,7 @@ static void *jl_precompile_(jl_array_t *m) jl_array_ptr_1d_push(m2, item); } } - void *native_code = jl_create_native(m2, NULL, NULL, 0, 1); + void *native_code = jl_create_native(m2, NULL, NULL, 0, 1, external_linkage); JL_GC_POP(); return native_code; } @@ -378,7 +440,7 @@ static void *jl_precompile(int all) if (all) jl_compile_all_defs(m); jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); - void *native_code = jl_precompile_(m); + void *native_code = jl_precompile_(m, 0); JL_GC_POP(); return native_code; } @@ -397,7 +459,7 @@ static void *jl_precompile_worklist(jl_array_t *worklist) assert(jl_is_module(mod)); foreach_mtable_in_module(mod, precompile_enq_all_specializations_, m); } - void *native_code = jl_precompile_(m); + void *native_code = jl_precompile_(m, 1); JL_GC_POP(); return native_code; } diff --git a/src/processor.cpp b/src/processor.cpp index df114b4d80257..13b40ec4f7363 100644 --- a/src/processor.cpp +++ b/src/processor.cpp @@ -621,9 +621,9 @@ static inline std::vector> &get_cmdline_targets(F &&feature_cb) // Load sysimg, use the `callback` for dispatch and perform all relocations // for the selected target. template -static inline jl_sysimg_fptrs_t parse_sysimg(void *hdl, F &&callback) +static inline jl_image_fptrs_t parse_sysimg(void *hdl, F &&callback) { - jl_sysimg_fptrs_t res = {nullptr, 0, nullptr, 0, nullptr, nullptr}; + jl_image_fptrs_t res = {nullptr, 0, nullptr, 0, nullptr, nullptr}; // .data base char *data_base; diff --git a/src/processor.h b/src/processor.h index ac00f8874141b..d1b18cb72b084 100644 --- a/src/processor.h +++ b/src/processor.h @@ -135,7 +135,7 @@ JL_DLLEXPORT int jl_test_cpu_feature(jl_cpu_feature_t feature); static const uint32_t jl_sysimg_tag_mask = 0x80000000u; static const uint32_t jl_sysimg_val_mask = ~((uint32_t)0x80000000u); -typedef struct _jl_sysimg_fptrs_t { +typedef struct _jl_image_fptrs_t { // base function pointer const char *base; // number of functions @@ -153,7 +153,7 @@ typedef struct _jl_sysimg_fptrs_t { const int32_t *clone_offsets; // sorted indices of the cloned functions (including the tag bit) const uint32_t *clone_idxs; -} jl_sysimg_fptrs_t; +} jl_image_fptrs_t; /** * Initialize the processor dispatch system with sysimg `hdl` (also initialize the sysimg itself). @@ -165,14 +165,15 @@ typedef struct _jl_sysimg_fptrs_t { * * Return the data about the function pointers selected. */ -jl_sysimg_fptrs_t jl_init_processor_sysimg(void *hdl); -jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl); +jl_image_fptrs_t jl_init_processor_sysimg(void *hdl); +jl_image_fptrs_t jl_init_processor_pkgimg(void *hdl); // Return the name of the host CPU as a julia string. JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void); // Dump the name and feature set of the host CPU // For debugging only JL_DLLEXPORT void jl_dump_host_cpu(void); +JL_DLLEXPORT void jl_check_pkgimage_clones(char* data); JL_DLLEXPORT int32_t jl_set_zero_subnormals(int8_t isZero); JL_DLLEXPORT int32_t jl_get_zero_subnormals(void); diff --git a/src/processor_arm.cpp b/src/processor_arm.cpp index f7a112993e3e5..3e7b22caf00d4 100644 --- a/src/processor_arm.cpp +++ b/src/processor_arm.cpp @@ -1802,14 +1802,14 @@ JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void) return jl_cstr_to_string(host_cpu_name().c_str()); } -jl_sysimg_fptrs_t jl_init_processor_sysimg(void *hdl) +jl_image_fptrs_t jl_init_processor_sysimg(void *hdl) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); return parse_sysimg(hdl, sysimg_init_cb); } -jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl) +jl_image_fptrs_t jl_init_processor_pkgimg(void *hdl) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); @@ -1818,6 +1818,11 @@ jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl) return parse_sysimg(hdl, pkgimg_init_cb); } +JL_DLLEXPORT void jl_check_pkgimage_clones(char *data) +{ + pkgimg_init_cb(data); +} + std::pair> jl_get_llvm_target(bool imaging, uint32_t &flags) { ensure_jit_target(imaging); diff --git a/src/processor_fallback.cpp b/src/processor_fallback.cpp index 3160bd0ba6750..c1353e1bb43b0 100644 --- a/src/processor_fallback.cpp +++ b/src/processor_fallback.cpp @@ -112,14 +112,14 @@ get_llvm_target_str(const TargetData<1> &data) using namespace Fallback; -jl_sysimg_fptrs_t jl_init_processor_sysimg(void *hdl) +jl_image_fptrs_t jl_init_processor_sysimg(void *hdl) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); return parse_sysimg(hdl, sysimg_init_cb); } -jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl) +jl_image_fptrs_t jl_init_processor_pkgimg(void *hdl) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); @@ -170,6 +170,11 @@ JL_DLLEXPORT void jl_dump_host_cpu(void) jl_safe_printf("Features: %s\n", jl_get_cpu_features_llvm().c_str()); } +JL_DLLEXPORT void jl_check_pkgimage_clones(char *data) +{ + pkgimg_init_cb(data); +} + extern "C" int jl_test_cpu_feature(jl_cpu_feature_t) { return 0; diff --git a/src/processor_x86.cpp b/src/processor_x86.cpp index b73838a55777e..6f064ddd47d19 100644 --- a/src/processor_x86.cpp +++ b/src/processor_x86.cpp @@ -1019,19 +1019,24 @@ JL_DLLEXPORT void jl_dump_host_cpu(void) cpus, ncpu_names); } +JL_DLLEXPORT void jl_check_pkgimage_clones(char *data) +{ + pkgimg_init_cb(data); +} + JL_DLLEXPORT jl_value_t *jl_get_cpu_name(void) { return jl_cstr_to_string(host_cpu_name().c_str()); } -jl_sysimg_fptrs_t jl_init_processor_sysimg(void *hdl) +jl_image_fptrs_t jl_init_processor_sysimg(void *hdl) { if (!jit_targets.empty()) jl_error("JIT targets already initialized"); return parse_sysimg(hdl, sysimg_init_cb); } -jl_sysimg_fptrs_t jl_init_processor_pkgimg(void *hdl) +jl_image_fptrs_t jl_init_processor_pkgimg(void *hdl) { if (jit_targets.empty()) jl_error("JIT targets not initialized"); diff --git a/src/staticdata.c b/src/staticdata.c index 2098596b9b612..9789803ccf582 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -321,7 +321,7 @@ typedef struct { uint64_t base; uintptr_t *gvars_base; int32_t *gvars_offsets; - jl_sysimg_fptrs_t fptrs; + jl_image_fptrs_t fptrs; } jl_image_t; // array of definitions for the predefined function pointers @@ -364,6 +364,7 @@ typedef struct { jl_array_t *link_ids_relocs; jl_array_t *link_ids_gctags; jl_array_t *link_ids_gvars; + jl_array_t *link_ids_external_fnvars; jl_ptls_t ptls; htable_t callers_with_edges; jl_image_t *image; @@ -856,7 +857,6 @@ static void write_padding(ios_t *s, size_t nb) JL_NOTSAFEPOINT ios_write(s, zeros, nb); } - static void write_pointer(ios_t *s) JL_NOTSAFEPOINT { assert((ios_pos(s) & (sizeof(void*) - 1)) == 0 && "stream misaligned for writing a word-sized value"); @@ -1082,6 +1082,24 @@ static void record_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAF } } +static void record_external_fns(jl_serializer_state *s, arraylist_t *external_fns) JL_NOTSAFEPOINT +{ + if (!s->incremental) { + assert(external_fns->len == 0); + (void) external_fns; + return; + } + + // We could call jl_queue_for_serialization here, but that should + // always be a no-op. +#ifndef JL_NDEBUG + for (size_t i = 0; i < external_fns->len; i++) { + jl_code_instance_t *ci = (jl_code_instance_t*)external_fns->items[i]; + assert(jl_object_in_image((jl_value_t*)ci)); + } +#endif +} + jl_value_t *jl_find_ptr = NULL; // The main function for serializing all the items queued in `serialization_order` // (They are also stored in `serialization_queue` which is order-preserving, unlike the hash table used @@ -1571,7 +1589,7 @@ static uintptr_t get_reloc_for_item(uintptr_t reloc_item, size_t reloc_offset) } // Compute target location at deserialization -static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, size_t size, uintptr_t reloc_id, jl_array_t *link_ids, int *link_index) +static inline uintptr_t get_item_for_reloc(jl_serializer_state *s, uintptr_t base, size_t size, uintptr_t reloc_id, jl_array_t *link_ids, int *link_index) JL_NOTSAFEPOINT { enum RefTags tag = (enum RefTags)(reloc_id >> RELOC_TAG_OFFSET); size_t offset = (reloc_id & (((uintptr_t)1 << RELOC_TAG_OFFSET) - 1)); @@ -1835,20 +1853,20 @@ static jl_value_t *jl_delayed_reloc(jl_serializer_state *s, uintptr_t offset) JL static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) { - jl_sysimg_fptrs_t fvars = image->fptrs; + jl_image_fptrs_t fvars = image->fptrs; // make these NULL now so we skip trying to restore GlobalVariable pointers later image->gvars_base = NULL; image->fptrs.base = NULL; if (fvars.base == NULL) return; - int sysimg_fvars_max = s->fptr_record->size / sizeof(void*); + int img_fvars_max = s->fptr_record->size / sizeof(void*); size_t i; uintptr_t base = (uintptr_t)&s->s->buf[0]; // These will become MethodInstance references, but they start out as a list of // offsets into `s` for CodeInstances jl_method_instance_t **linfos = (jl_method_instance_t**)&s->fptr_record->buf[0]; uint32_t clone_idx = 0; - for (i = 0; i < sysimg_fvars_max; i++) { + for (i = 0; i < img_fvars_max; i++) { reloc_t offset = *(reloc_t*)&linfos[i]; linfos[i] = NULL; if (offset != 0) { @@ -1883,12 +1901,13 @@ static void jl_update_all_fptrs(jl_serializer_state *s, jl_image_t *image) } } // Tell LLVM about the native code - jl_register_fptrs(image->base, &fvars, linfos, sysimg_fvars_max); + jl_register_fptrs(image->base, &fvars, linfos, img_fvars_max); } -static void write_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAFEPOINT +static uint32_t write_gvars(jl_serializer_state *s, arraylist_t *globals, arraylist_t *external_fns) JL_NOTSAFEPOINT { - ios_ensureroom(s->gvar_record, globals->len * sizeof(reloc_t)); + size_t len = globals->len + external_fns->len; + ios_ensureroom(s->gvar_record, len * sizeof(reloc_t)); for (size_t i = 0; i < globals->len; i++) { void *g = globals->items[i]; if (jl_is_binding((uintptr_t)g)) { @@ -1909,10 +1928,17 @@ static void write_gvars(jl_serializer_state *s, arraylist_t *globals) JL_NOTSAFE write_reloc_t(s->gvar_record, reloc); record_uniquing(s, (jl_value_t*)g, ((i << 2) | 2)); // mark as gvar && !tag } + for (size_t i = 0; i < external_fns->len; i++) { + jl_code_instance_t *ci = (jl_code_instance_t*)external_fns->items[i]; + uintptr_t item = backref_id(s, (void*)ci, s->link_ids_external_fnvars); + uintptr_t reloc = get_reloc_for_item(item, 0); + write_reloc_t(s->gvar_record, reloc); + } + return globals->len + 1; } // Pointer relocation for native-code referenced global variables -static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image) +static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image, uint32_t external_fns_begin) { if (image->gvars_base == NULL) return; @@ -1921,17 +1947,24 @@ static void jl_update_all_gvars(jl_serializer_state *s, jl_image_t *image) uintptr_t base = (uintptr_t)&s->s->buf[0]; size_t size = s->s->size; reloc_t *gvars = (reloc_t*)&s->gvar_record->buf[0]; - int link_index = 0; + int gvar_link_index = 0; + int external_fns_link_index = 0; for (i = 0; i < l; i++) { uintptr_t offset = gvars[i]; - uintptr_t v = get_item_for_reloc(s, base, size, offset, s->link_ids_gvars, &link_index); + uintptr_t v = 0; + if (i < external_fns_begin) { + v = get_item_for_reloc(s, base, size, offset, s->link_ids_gvars, &gvar_link_index); + } else { + v = get_item_for_reloc(s, base, size, offset, s->link_ids_external_fnvars, &external_fns_link_index); + } uintptr_t *gv = sysimg_gvars(image->gvars_base, image->gvars_offsets, i); *gv = v; } - assert(!s->link_ids_gvars || link_index == jl_array_len(s->link_ids_gvars)); + assert(!s->link_ids_gvars || gvar_link_index == jl_array_len(s->link_ids_gvars)); + assert(!s->link_ids_external_fnvars || external_fns_link_index == jl_array_len(s->link_ids_external_fnvars)); } -static void jl_root_new_gvars(jl_serializer_state *s, jl_image_t *image) +static void jl_root_new_gvars(jl_serializer_state *s, jl_image_t *image, uint32_t external_fns_begin) { if (image->gvars_base == NULL) return; @@ -1940,8 +1973,14 @@ static void jl_root_new_gvars(jl_serializer_state *s, jl_image_t *image) for (i = 0; i < l; i++) { uintptr_t *gv = sysimg_gvars(image->gvars_base, image->gvars_offsets, i); uintptr_t v = *gv; - if (!jl_is_binding(v)) - v = (uintptr_t)jl_as_global_root((jl_value_t*)v); + if (i < external_fns_begin) { + if (!jl_is_binding(v)) + v = (uintptr_t)jl_as_global_root((jl_value_t*)v); + } else { + jl_code_instance_t *codeinst = (jl_code_instance_t*) v; + assert(codeinst && codeinst->isspecsig); + v = (uintptr_t)codeinst->specptr.fptr; + } *gv = v; } } @@ -2293,13 +2332,18 @@ static void jl_save_system_image_to_stream(ios_t *f, s.link_ids_relocs = jl_alloc_array_1d(jl_array_uint64_type, 0); s.link_ids_gctags = jl_alloc_array_1d(jl_array_uint64_type, 0); s.link_ids_gvars = jl_alloc_array_1d(jl_array_uint64_type, 0); + s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_uint64_type, 0); htable_new(&s.callers_with_edges, 0); jl_value_t **const*const tags = get_tags(); // worklist == NULL ? get_tags() : NULL; arraylist_t gvars; + arraylist_t external_fns; arraylist_new(&gvars, 0); - if (native_functions) + arraylist_new(&external_fns, 0); + if (native_functions) { jl_get_llvm_gvs(native_functions, &gvars); + jl_get_llvm_external_fns(native_functions, &external_fns); + } if (worklist == NULL) { // empty!(Core.ARGS) @@ -2365,6 +2409,7 @@ static void jl_save_system_image_to_stream(ios_t *f, jl_serialize_reachable(&s); // step 1.2: now that we have marked all bindings (badly), ensure all gvars are part of the sysimage record_gvars(&s, &gvars); + record_external_fns(&s, &external_fns); jl_serialize_reachable(&s); // step 1.3: prune (garbage collect) some special weak references from // built-in type caches @@ -2378,10 +2423,11 @@ static void jl_save_system_image_to_stream(ios_t *f, } } + uint32_t external_fns_begin = 0; { // step 2: build all the sysimg sections write_padding(&sysimg, sizeof(uintptr_t)); jl_write_values(&s); - write_gvars(&s, &gvars); + external_fns_begin = write_gvars(&s, &gvars, &external_fns); jl_write_relocations(&s); } @@ -2481,6 +2527,9 @@ static void jl_save_system_image_to_stream(ios_t *f, ios_write(f, (char*)jl_array_data(s.link_ids_relocs), jl_array_len(s.link_ids_relocs)*sizeof(uint64_t)); write_uint32(f, jl_array_len(s.link_ids_gvars)); ios_write(f, (char*)jl_array_data(s.link_ids_gvars), jl_array_len(s.link_ids_gvars)*sizeof(uint64_t)); + write_uint32(f, jl_array_len(s.link_ids_external_fnvars)); + ios_write(f, (char*)jl_array_data(s.link_ids_external_fnvars), jl_array_len(s.link_ids_external_fnvars)*sizeof(uint64_t)); + write_uint32(f, external_fns_begin); jl_write_arraylist(s.s, &s.ccallable_list); } // Write the build_id key @@ -2498,6 +2547,7 @@ static void jl_save_system_image_to_stream(ios_t *f, arraylist_free(&s.relocs_list); arraylist_free(&s.gctags_list); arraylist_free(&gvars); + arraylist_free(&external_fns); htable_free(&field_replace); if (worklist) htable_free(&external_objects); @@ -2518,9 +2568,8 @@ static void jl_write_header_for_incremental(ios_t *f, jl_array_t *worklist, jl_a assert(jl_precompile_toplevel_module == NULL); jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); - write_header(f); - // last word of the header is the checksumpos - *checksumpos = ios_pos(f) - sizeof(uint64_t); + *checksumpos = write_header(f, 0); + write_uint8(f, jl_cache_flags()); // write description of contents (name, uuid, buildid) write_worklist_for_header(f, worklist); // Determine unique (module, abspath, mtime) dependencies for the files defining modules in the worklist @@ -2534,88 +2583,87 @@ static void jl_write_header_for_incremental(ios_t *f, jl_array_t *worklist, jl_a write_mod_list(f, *mod_array); } - -JL_DLLEXPORT ios_t *jl_create_system_image(void *_native_data, jl_array_t *worklist) +JL_DLLEXPORT void jl_create_system_image(void *_native_data, jl_array_t *worklist, bool_t emit_split, + ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos) { jl_gc_collect(JL_GC_FULL); jl_gc_collect(JL_GC_INCREMENTAL); // sweep finalizers JL_TIMING(SYSIMG_DUMP); + // iff emit_split + // write header and src_text to one file f/s + // write systemimg to a second file ff/z jl_task_t *ct = jl_current_task; ios_t *f = (ios_t*)malloc_s(sizeof(ios_t)); ios_mem(f, 0); - jl_array_t *mod_array = NULL, *udeps = NULL, *extext_methods = NULL, *new_specializations = NULL; + + ios_t *ff = NULL; + if (emit_split) { + ff = (ios_t*)malloc_s(sizeof(ios_t)); + ios_mem(ff, 0); + } else { + ff = f; + } + + jl_array_t *mod_array = NULL, *extext_methods = NULL, *new_specializations = NULL; jl_array_t *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL; - JL_GC_PUSH7(&mod_array, &udeps, &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges); - int64_t srctextpos = 0; int64_t checksumpos = 0; + int64_t checksumpos_ff = 0; int64_t datastartpos = 0; + JL_GC_PUSH6(&mod_array, &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges); if (worklist) { - jl_write_header_for_incremental(f, worklist, &mod_array, &udeps, &srctextpos, &checksumpos); + jl_write_header_for_incremental(f, worklist, &mod_array, udeps, srctextpos, &checksumpos); + if (emit_split) { + checksumpos_ff = write_header(ff, 1); + write_uint8(ff, jl_cache_flags()); + write_mod_list(ff, mod_array); + } else { + checksumpos_ff = checksumpos; + } jl_gc_enable_finalizers(ct, 0); // make sure we don't run any Julia code concurrently after this point jl_prepare_serialization_data(mod_array, newly_inferred, jl_worklist_key(worklist), &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges); - write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f)); - datastartpos = ios_pos(f); + + if (!emit_split) { + write_int32(f, 0); // No clone_targets + write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f)); + } else { + write_padding(ff, LLT_ALIGN(ios_pos(ff), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(ff)); + } + datastartpos = ios_pos(ff); } native_functions = _native_data; - jl_save_system_image_to_stream(f, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges); + jl_save_system_image_to_stream(ff, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges); native_functions = NULL; if (worklist) { jl_gc_enable_finalizers(ct, 1); // make sure we don't run any Julia code concurrently before this point + jl_precompile_toplevel_module = NULL; + } + + if (worklist) { // Go back and update the checksum in the header - int64_t dataendpos = ios_pos(f); - uint32_t checksum = jl_crc32c(0, &f->buf[datastartpos], dataendpos - datastartpos); - ios_seek(f, checksumpos); - write_uint64(f, checksum | ((uint64_t)0xfafbfcfd << 32)); - ios_seek(f, srctextpos); - write_uint64(f, dataendpos); - // Write the source-text for the dependent files - // Go back and update the source-text position to point to the current position - if (udeps) { - ios_seek_end(f); - // Each source-text file is written as - // int32: length of abspath - // char*: abspath - // uint64: length of src text - // char*: src text - // At the end we write int32(0) as a terminal sentinel. - size_t len = jl_array_len(udeps); - ios_t srctext; - for (size_t i = 0; i < len; i++) { - jl_value_t *deptuple = jl_array_ptr_ref(udeps, i); - jl_value_t *depmod = jl_fieldref(deptuple, 0); // module - // Dependencies declared with `include_dependency` are excluded - // because these may not be Julia code (and could be huge) - if (depmod != (jl_value_t*)jl_main_module) { - jl_value_t *dep = jl_fieldref(deptuple, 1); // file abspath - const char *depstr = jl_string_data(dep); - if (!depstr[0]) - continue; - ios_t *srctp = ios_file(&srctext, depstr, 1, 0, 0, 0); - if (!srctp) { - jl_printf(JL_STDERR, "WARNING: could not cache source text for \"%s\".\n", - jl_string_data(dep)); - continue; - } - size_t slen = jl_string_len(dep); - write_int32(f, slen); - ios_write(f, depstr, slen); - int64_t posfile = ios_pos(f); - write_uint64(f, 0); // placeholder for length of this file in bytes - uint64_t filelen = (uint64_t) ios_copyall(f, &srctext); - ios_close(&srctext); - ios_seek(f, posfile); - write_uint64(f, filelen); - ios_seek_end(f); - } - } + int64_t dataendpos = ios_pos(ff); + uint32_t checksum = jl_crc32c(0, &ff->buf[datastartpos], dataendpos - datastartpos); + ios_seek(ff, checksumpos_ff); + write_uint64(ff, checksum | ((uint64_t)0xfafbfcfd << 32)); + write_uint64(ff, datastartpos); + write_uint64(ff, dataendpos); + ios_seek(ff, dataendpos); + + // Write the checksum to the split header if necessary + if (emit_split) { + int64_t cur = ios_pos(f); + ios_seek(f, checksumpos); + write_uint64(f, checksum | ((uint64_t)0xfafbfcfd << 32)); + ios_seek(f, cur); + // Next we will write the clone_targets and afterwards the srctext } - write_int32(f, 0); // mark the end of the source text - jl_precompile_toplevel_module = NULL; } JL_GC_POP(); - return f; + *s = f; + if (emit_split) + *z = ff; + return; } JL_DLLEXPORT size_t ios_write_direct(ios_t *dest, ios_t *src); @@ -2678,7 +2726,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl s.ptls = jl_current_task->ptls; arraylist_new(&s.relocs_list, 0); arraylist_new(&s.gctags_list, 0); - s.link_ids_relocs = s.link_ids_gctags = s.link_ids_gvars = NULL; + s.link_ids_relocs = s.link_ids_gctags = s.link_ids_gvars = s.link_ids_external_fnvars = NULL; jl_value_t **const*const tags = get_tags(); htable_t new_dt_objs; htable_new(&new_dt_objs, 0); @@ -2770,6 +2818,12 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl s.link_ids_gvars = jl_alloc_array_1d(jl_array_uint64_type, nlinks_gvars); ios_read(f, (char*)jl_array_data(s.link_ids_gvars), nlinks_gvars * sizeof(uint64_t)); } + size_t nlinks_external_fnvars = read_uint32(f); + if (nlinks_external_fnvars > 0) { + s.link_ids_external_fnvars = jl_alloc_array_1d(jl_array_uint64_type, nlinks_external_fnvars); + ios_read(f, (char*)jl_array_data(s.link_ids_external_fnvars), nlinks_external_fnvars * sizeof(uint64_t)); + } + uint32_t external_fns_begin = read_uint32(f); jl_read_arraylist(s.s, ccallable_list ? ccallable_list : &s.ccallable_list); if (s.incremental) { assert(restored && init_order && extext_methods && new_specializations && method_roots_list && ext_targets && edges); @@ -2783,6 +2837,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl } s.s = NULL; + // step 3: apply relocations assert(!ios_eof(f)); jl_read_symbols(&s); @@ -2799,7 +2854,8 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl (void)sizeof_tags; jl_read_reloclist(&s, s.link_ids_relocs, 0); // general relocs // s.link_ids_gvars will be processed in `jl_update_all_gvars` - jl_update_all_gvars(&s, image); // gvars relocs + // s.link_ids_external_fns will be processed in `jl_update_all_gvars` + jl_update_all_gvars(&s, image, external_fns_begin); // gvars relocs if (s.incremental) { jl_read_arraylist(s.relocs, &s.uniquing_types); jl_read_arraylist(s.relocs, &s.uniquing_objs); @@ -3108,7 +3164,7 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl arraylist_free(&s.fixup_objs); if (s.incremental) - jl_root_new_gvars(&s, image); + jl_root_new_gvars(&s, image, external_fns_begin); ios_close(&relocs); ios_close(&const_data); ios_close(&gvar_record); @@ -3178,21 +3234,26 @@ static void jl_restore_system_image_from_stream_(ios_t *f, jl_image_t *image, jl jl_gc_enable(en); } -static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_t *checksum, int64_t *dataendpos) +static jl_value_t *jl_validate_cache_file(ios_t *f, jl_array_t *depmods, uint64_t *checksum, int64_t *dataendpos, int64_t *datastartpos) { - if (ios_eof(f) || 0 == (*checksum = jl_read_verify_header(f)) || (*checksum >> 32 != 0xfafbfcfd)) { + uint8_t pkgimage = 0; + if (ios_eof(f) || 0 == (*checksum = jl_read_verify_header(f, &pkgimage, dataendpos, datastartpos)) || (*checksum >> 32 != 0xfafbfcfd)) { return jl_get_exceptionf(jl_errorexception_type, "Precompile file header verification checks failed."); } - { // skip past the mod list + uint8_t flags = read_uint8(f); + if (pkgimage && !jl_match_cache_flags(flags)) { + return jl_get_exceptionf(jl_errorexception_type, "Pkgimage flags mismatch"); + } + if (!pkgimage) { + // skip past the worklist size_t len; while ((len = read_int32(f))) ios_skip(f, len + 3 * sizeof(uint64_t)); - } - { // skip past the dependency list + // skip past the dependency list size_t deplen = read_uint64(f); ios_skip(f, deplen - sizeof(uint64_t)); - *dataendpos = read_uint64(f); + read_uint64(f); // where is this write coming from? } // verify that the system state is valid @@ -3204,10 +3265,14 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im { uint64_t checksum = 0; int64_t dataendpos = 0; - jl_value_t *verify_fail = jl_validate_cache_file(f, depmods, &checksum, &dataendpos); + int64_t datastartpos = 0; + jl_value_t *verify_fail = jl_validate_cache_file(f, depmods, &checksum, &dataendpos, &datastartpos); + if (verify_fail) return verify_fail; + assert(datastartpos > 0 && datastartpos < dataendpos); + jl_value_t *restored = NULL; jl_array_t *init_order = NULL, *extext_methods = NULL, *new_specializations = NULL, *method_roots_list = NULL, *ext_targets = NULL, *edges = NULL; jl_svec_t *cachesizes_sv = NULL; @@ -3218,11 +3283,9 @@ static jl_value_t *jl_restore_package_image_from_stream(ios_t *f, jl_image_t *im { // make a permanent in-memory copy of f (excluding the header) ios_bufmode(f, bm_none); JL_SIGATOMIC_BEGIN(); - size_t len_begin = LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT); - assert(len_begin > 0 && len_begin < dataendpos); - size_t len = dataendpos - len_begin; + size_t len = dataendpos - datastartpos; char *sysimg = (char*)jl_gc_perm_alloc(len, 0, 64, 0); - ios_seek(f, len_begin); + ios_seek(f, datastartpos); if (ios_readall(f, sysimg, len) != len || jl_crc32c(0, sysimg, len) != (uint32_t)checksum) { restored = jl_get_exceptionf(jl_errorexception_type, "Error reading system image file."); JL_SIGATOMIC_END(); @@ -3363,9 +3426,9 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j if (!jl_dlsym(pkgimg_handle, "jl_sysimg_gvars_base", (void **)&pkgimage.gvars_base, 0)) { pkgimage.gvars_base = NULL; } + jl_dlsym(pkgimg_handle, "jl_sysimg_gvars_offsets", (void **)&pkgimage.gvars_offsets, 1); pkgimage.gvars_offsets += 1; - jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, 0); void *pgcstack_func_slot; jl_dlsym(pkgimg_handle, "jl_pgcstack_func_slot", &pgcstack_func_slot, 0); @@ -3379,6 +3442,20 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j *tls_offset_idx = (uintptr_t)(jl_tls_offset == -1 ? 0 : jl_tls_offset); } + #ifdef _OS_WINDOWS_ + pkgimage.base = (intptr_t)pkgimg_handle; + #else + Dl_info dlinfo; + if (dladdr((void*)pkgimage.gvars_base, &dlinfo) != 0) { + pkgimage.base = (intptr_t)dlinfo.dli_fbase; + } + else { + pkgimage.base = 0; + } + #endif + + jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, 0); + return mod; } diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 3d02dddbd5a70..0ec61540de8bd 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -632,11 +632,36 @@ static void write_mod_list(ios_t *s, jl_array_t *a) write_int32(s, 0); } +JL_DLLEXPORT uint8_t jl_cache_flags(void) +{ + // ??OOPCDD + uint8_t flags = 0; + flags |= jl_options.debug_level & 3; + flags |= (jl_options.check_bounds & 1) << 2; + flags |= (jl_options.use_pkgimage_native_code & 1) << 3; + flags |= (jl_options.opt_level & 3) << 4; + // NOTES: + // In contrast to check-bounds, inline has no "observable effect" + return flags; +} + +JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t flags) +{ + uint8_t current_flags = jl_cache_flags(); + uint8_t mask = (1 << 4)-1; + if ((flags & mask) != (current_flags & mask)) + return 0; + // allow for higher optimization flags in cache + flags >>= 4; + current_flags >>= 4; + return flags >= current_flags; +} + // "magic" string and version header of .ji file static const int JI_FORMAT_VERSION = 12; static const char JI_MAGIC[] = "\373jli\r\n\032\n"; // based on PNG signature static const uint16_t BOM = 0xFEFF; // byte-order marker -static void write_header(ios_t *s) +static int64_t write_header(ios_t *s, uint8_t pkgimage) { ios_write(s, JI_MAGIC, strlen(JI_MAGIC)); write_uint16(s, JI_FORMAT_VERSION); @@ -648,7 +673,12 @@ static void write_header(ios_t *s) const char *branch = jl_git_branch(), *commit = jl_git_commit(); ios_write(s, branch, strlen(branch)+1); ios_write(s, commit, strlen(commit)+1); + write_uint8(s, pkgimage); + int64_t checksumpos = ios_pos(s); write_uint64(s, 0); // eventually will hold checksum for the content portion of this (build_id.hi) + write_uint64(s, 0); // eventually will hold dataendpos + write_uint64(s, 0); // eventually will hold datastartpos + return checksumpos; } // serialize information about the result of deserializing this file @@ -1262,9 +1292,10 @@ static int readstr_verify(ios_t *s, const char *str, int include_null) return 1; } -JL_DLLEXPORT uint64_t jl_read_verify_header(ios_t *s) +JL_DLLEXPORT uint64_t jl_read_verify_header(ios_t *s, uint8_t *pkgimage, int64_t *dataendpos, int64_t *datastartpos) { uint16_t bom; + uint64_t checksum = 0; if (readstr_verify(s, JI_MAGIC, 0) && read_uint16(s) == JI_FORMAT_VERSION && ios_read(s, (char *) &bom, 2) == 2 && bom == BOM && @@ -1274,6 +1305,11 @@ JL_DLLEXPORT uint64_t jl_read_verify_header(ios_t *s) readstr_verify(s, JULIA_VERSION_STRING, 1) && readstr_verify(s, jl_git_branch(), 1) && readstr_verify(s, jl_git_commit(), 1)) - return read_uint64(s); - return 0; + { + *pkgimage = read_uint8(s); + checksum = read_uint64(s); + *datastartpos = (int64_t)read_uint64(s); + *dataendpos = (int64_t)read_uint64(s); + } + return checksum; } diff --git a/test/compiler/contextual.jl b/test/compiler/contextual.jl index 75cbcf2de37fc..79285f62b0947 100644 --- a/test/compiler/contextual.jl +++ b/test/compiler/contextual.jl @@ -209,7 +209,11 @@ try @test length(Bar.mt) == 1 finally rm(load_path, recursive=true, force=true) - rm(depot_path, recursive=true, force=true) + try + rm(depot_path, force=true, recursive=true) + catch err + @show err + end filter!((≠)(load_path), LOAD_PATH) filter!((≠)(depot_path), DEPOT_PATH) end diff --git a/test/loading.jl b/test/loading.jl index 99f39ae237532..5a8fa3f64b298 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -746,7 +746,11 @@ for env in keys(envs) rm(env, force=true, recursive=true) end for depot in depots - rm(depot, force=true, recursive=true) + try + rm(depot, force=true, recursive=true) + catch err + @show err + end end append!(empty!(LOAD_PATH), saved_load_path) diff --git a/test/precompile.jl b/test/precompile.jl index 688286a113b55..952af1e869aef 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -28,8 +28,18 @@ function precompile_test_harness(@nospecialize(f), separate::Bool) pushfirst!(DEPOT_PATH, load_cache_path) f(load_path) finally - rm(load_path, recursive=true, force=true) - separate && rm(load_cache_path, recursive=true, force=true) + try + rm(load_path, force=true, recursive=true) + catch err + @show err + end + if separate + try + rm(load_cache_path, force=true, recursive=true) + catch err + @show err + end + end filter!((≠)(load_path), LOAD_PATH) separate && filter!((≠)(load_cache_path), DEPOT_PATH) end @@ -318,11 +328,16 @@ precompile_test_harness(false) do dir cachedir = joinpath(dir, "compiled", "v$(VERSION.major).$(VERSION.minor)") cachedir2 = joinpath(dir2, "compiled", "v$(VERSION.major).$(VERSION.minor)") cachefile = joinpath(cachedir, "$Foo_module.ji") + if Base.JLOptions().use_pkgimage_native_code == 1 + ocachefile = Base.ocachefile_from_cachefile(cachefile) + else + ocachefile = nothing + end # use _require_from_serialized to ensure that the test fails if # the module doesn't reload from the image: @test_warn "@ccallable was already defined for this method name" begin @test_logs (:warn, "Replacing module `$Foo_module`") begin - m = Base._require_from_serialized(Base.PkgId(Foo), cachefile) + m = Base._require_from_serialized(Base.PkgId(Foo), cachefile, ocachefile) @test isa(m, Module) end end @@ -343,7 +358,7 @@ precompile_test_harness(false) do dir @test string(Base.Docs.doc(Foo.Bar.bar)) == "bar function\n" @test string(Base.Docs.doc(Foo.Bar)) == "Bar module\n" - modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile) + modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) discard_module = mod_fl_mt -> (mod_fl_mt.filename, mod_fl_mt.mtime) @test modules == [ Base.PkgId(Foo) => Base.module_build_id(Foo) % UInt64 ] @test map(x -> x.filename, deps) == [ Foo_file, joinpath(dir, "foo.jl"), joinpath(dir, "bar.jl") ] @@ -378,7 +393,7 @@ precompile_test_harness(false) do dir ), ) @test discard_module.(deps) == deps1 - modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile; srcfiles_only=true) + modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile; srcfiles_only=true) @test map(x -> x.filename, deps) == [Foo_file] @test current_task()(0x01, 0x4000, 0x30031234) == 2 @@ -441,7 +456,7 @@ precompile_test_harness(false) do dir """) Nest = Base.require(Main, Nest_module) cachefile = joinpath(cachedir, "$Nest_module.ji") - modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile) + modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) @test last(deps).modpath == ["NestInner"] UsesB_module = :UsesB4b3a94a1a081a8cb @@ -463,7 +478,7 @@ precompile_test_harness(false) do dir """) UsesB = Base.require(Main, UsesB_module) cachefile = joinpath(cachedir, "$UsesB_module.ji") - modules, (deps, requires), required_modules = Base.parse_cache_header(cachefile) + modules, (deps, requires), required_modules, _... = Base.parse_cache_header(cachefile) id1, id2 = only(requires) @test Base.pkgorigins[id1].cachepath == cachefile @test Base.pkgorigins[id2].cachepath == joinpath(cachedir, "$B_module.ji") @@ -497,18 +512,18 @@ precompile_test_harness(false) do dir end """) - cachefile = Base.compilecache(Base.PkgId("FooBar")) + cachefile, _ = Base.compilecache(Base.PkgId("FooBar")) empty_prefs_hash = Base.get_preferences_hash(nothing, String[]) @test cachefile == Base.compilecache_path(Base.PkgId("FooBar"), empty_prefs_hash) @test isfile(joinpath(cachedir, "FooBar.ji")) - @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Vector + @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tuple{<:Vector, String} @test !isdefined(Main, :FooBar) @test !isdefined(Main, :FooBar1) relFooBar_file = joinpath(dir, "subfolder", "..", "FooBar.jl") - @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa (Sys.iswindows() ? Vector : Bool) # `..` is not a symlink on Windows + @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa (Sys.iswindows() ? Tuple{<:Vector, String} : Bool) # `..` is not a symlink on Windows mkdir(joinpath(dir, "subfolder")) - @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa Vector + @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tuple{<:Vector, String} @eval using FooBar fb_uuid = Base.module_build_id(FooBar) @@ -520,7 +535,7 @@ precompile_test_harness(false) do dir @test !isfile(joinpath(cachedir, "FooBar1.ji")) @test isfile(joinpath(cachedir2, "FooBar1.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) === true - @test Base.stale_cachefile(FooBar1_file, joinpath(cachedir2, "FooBar1.ji")) isa Vector + @test Base.stale_cachefile(FooBar1_file, joinpath(cachedir2, "FooBar1.ji")) isa Tuple{<:Vector, String} @test fb_uuid == Base.module_build_id(FooBar) fb_uuid1 = Base.module_build_id(FooBar1) @test fb_uuid != fb_uuid1 @@ -1260,7 +1275,11 @@ end end finally cd(save_cwd) - rm(temp_path, recursive=true) + try + rm(temp_path, recursive=true) + catch err + @show err + end pop!(test_workers) # remove myid rmprocs(test_workers) end @@ -1400,13 +1419,13 @@ precompile_test_harness("Issue #25971") do load_path sourcefile = joinpath(load_path, "Foo25971.jl") write(sourcefile, "module Foo25971 end") chmod(sourcefile, 0o666) - cachefile = Base.compilecache(Base.PkgId("Foo25971")) + cachefile, _ = Base.compilecache(Base.PkgId("Foo25971")) @test filemode(sourcefile) == filemode(cachefile) chmod(sourcefile, 0o600) - cachefile = Base.compilecache(Base.PkgId("Foo25971")) + cachefile, _ = Base.compilecache(Base.PkgId("Foo25971")) @test filemode(sourcefile) == filemode(cachefile) chmod(sourcefile, 0o444) - cachefile = Base.compilecache(Base.PkgId("Foo25971")) + cachefile, _ = Base.compilecache(Base.PkgId("Foo25971")) # Check writable @test touch(cachefile) == cachefile end From 0f135a350f14e62ae7d550937d322e2e4a562f1a Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 12 Dec 2022 12:28:45 -0600 Subject: [PATCH 02/25] Improve wording, fix whitespace --- doc/src/manual/methods.md | 2 +- doc/src/manual/modules.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/manual/methods.md b/doc/src/manual/methods.md index 376a848808e43..2e9629122e4af 100644 --- a/doc/src/manual/methods.md +++ b/doc/src/manual/methods.md @@ -296,7 +296,7 @@ julia> mysum(1.0, 2.0) Julia will compile `mysum` twice, once for `x::Int, y::Int` and again for `x::Float64, y::Float64`. The point of compiling twice is performance: the methods that get called for `+` (which `mysum` uses) vary depending on the specific types of `x` and `y`, and by compiling different specializations Julia can do all the method lookup ahead of time. This allows the program to run much more quickly, since it doesn't have to bother with method lookup while it is running. -Julia's automatic specialization allows you to write "generic" algorithms and expect that the compiler will generate efficient, specialized code to handle each case when you need it. +Julia's automatic specialization allows you to write generic algorithms and expect that the compiler will generate efficient, specialized code to handle each case you need. In cases where the number of potential specializations might be effectively unlimited, Julia may avoid this default specialization. See [Be aware of when Julia avoids specializing](@ref) for more information. diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index a8bc3f68fb80d..66796f0a87059 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -601,4 +601,4 @@ precompilation triggering when installing, updating, and explicitly building pac You can also debug some precompilation failures with environment variables. Setting `JULIA_VERBOSE_LINKING=true` may help resolve failures in linking shared libraries of [compiled -native code](@ref pkgimages). \ No newline at end of file +native code](@ref pkgimages). From 90db48c2a7f1e450f59886b54108f4bb64da1904 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 12 Dec 2022 15:46:51 -0600 Subject: [PATCH 03/25] Can't reference devdocs from manual --- doc/src/manual/modules.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index 66796f0a87059..a9d97b415eac5 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -600,5 +600,5 @@ manually. The state of this command line flag is passed to `Pkg.build` to disabl precompilation triggering when installing, updating, and explicitly building packages. You can also debug some precompilation failures with environment variables. Setting -`JULIA_VERBOSE_LINKING=true` may help resolve failures in linking shared libraries of [compiled -native code](@ref pkgimages). +`JULIA_VERBOSE_LINKING=true` may help resolve failures in linking shared libraries of compiled +native code. See the `devdocs` section of the manual for further details. From 5043270f00e3d1199bf2c18888cea2ec28fda887 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 12 Dec 2022 15:47:35 -0600 Subject: [PATCH 04/25] Capitalize Xcode correctly Co-authored-by: Max Horn --- doc/src/devdocs/pkgimg.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/devdocs/pkgimg.md b/doc/src/devdocs/pkgimg.md index dfd11648ef784..43475024011f2 100644 --- a/doc/src/devdocs/pkgimg.md +++ b/doc/src/devdocs/pkgimg.md @@ -28,7 +28,7 @@ another compiler driver, we therefore reimplement some of the logic from `llvm-p To avoid having to deal with `link.exe` we use `-flavor gnu`, effectively turning `lld` into a cross-linker from a mingw32 environment. Windows DLLs are required to contain a `_DllMainCRTStartup` function and to minimize our dependenc on mingw32 libraries we inject a stub definition ourselves. #### MacOS -Dynamic libraries on MacOS need to link against `-lSystem`. On recent MacOS versions `-lSystem` is only available for linking when XCode is available. +Dynamic libraries on MacOS need to link against `-lSystem`. On recent MacOS versions `-lSystem` is only available for linking when Xcode is available. To that effect we link with `-undefined dynamic_lookup`. ## Package images optimized for multiple microarchitectures From 4e97943db4409ff573bd46f05c2c0432d5767266 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 13 Dec 2022 09:41:07 -0600 Subject: [PATCH 05/25] Clarify documentation Co-authored-by: Michael Schlottke-Lakemper --- doc/src/devdocs/pkgimg.md | 26 +++++++++++++------------- doc/src/manual/code-loading.md | 2 +- doc/src/manual/methods.md | 2 +- doc/src/manual/modules.md | 4 ++-- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/doc/src/devdocs/pkgimg.md b/doc/src/devdocs/pkgimg.md index 43475024011f2..9e1cbc32670f8 100644 --- a/doc/src/devdocs/pkgimg.md +++ b/doc/src/devdocs/pkgimg.md @@ -1,47 +1,47 @@ # Package Images Julia package images provide object (native code) caches for Julia packages. -They are similar to Julia's [system image](@) and support many of the same features. +They are similar to Julia's [system image](@dev-sysimg) and support many of the same features. In fact the underlying serialization format is the same, and the system image is the base image that the package images are build against. ## High-level overview -Package images are shared libraries that contain both code and data. Like `.ji` cache files they are generated per package. The data section contains both global data (global variables in the package) as well as the necessary metadata about what methods and types are defined by the package. The code section contains native objects that cache the final output of Julia's LLVM based compiler. +Package images are shared libraries that contain both code and data. Like `.ji` cache files, they are generated per package. The data section contains both global data (global variables in the package) as well as the necessary metadata about what methods and types are defined by the package. The code section contains native objects that cache the final output of Julia's LLVM-based compiler. The command line option `--pkgimage-native-code=no` can be used to turn-off object caching for this session. Note that this means that cache files have to be regenerated. !!! note - While the package images present themselves as native shared libraries they are only an approximation thereof. You will not be able to link against them from a native program and they must be loaded from Julia. + While the package images present themselves as native shared libraries, they are only an approximation thereof. You will not be able to link against them from a native program and they must be loaded from Julia. ## Linking -Since the package images partially contain native code, we must run a linker over them before we can use them. You can use the environment variable `JULIA_VERBOSE_LINKING` to make the pkgimage linking process verbose. +Since the package images contain native code, we must run a linker over them before we can use them. You can set the environment variable `JULIA_VERBOSE_LINKING` to `true` to make the package image linking process verbose. -Furthermore we cannot assume that the user has a working system linker installed. Julia now comes with LLD, the LLVM linker, to provide a working out of the box experience. In `base/linking.jl` we implement a limited interface to be able to link package images on all supported platforms. +Furthermore, we cannot assume that the user has a working system linker installed. Therefore, Julia ships with LLD, the LLVM linker, to provide a working out of the box experience. In `base/linking.jl`, we implement a limited interface to be able to link package images on all supported platforms. ### Quirks -Despite LLD being a multi-platform linker it does not provide a consistent interface across platforms. Furthermore it is meant to be used from `clang` or +Despite LLD being a multi-platform linker, it does not provide a consistent interface across platforms. Furthermore, it is meant to be used from `clang` or another compiler driver, we therefore reimplement some of the logic from `llvm-project/clang/lib/Driver/ToolChains`. Thankfully one can use `lld -flavor` to set lld to the right platform #### Windows -To avoid having to deal with `link.exe` we use `-flavor gnu`, effectively turning `lld` into a cross-linker from a mingw32 environment. Windows DLLs are required to contain a `_DllMainCRTStartup` function and to minimize our dependenc on mingw32 libraries we inject a stub definition ourselves. +To avoid having to deal with `link.exe` we use `-flavor gnu`, effectively turning `lld` into a cross-linker from a mingw32 environment. Windows DLLs are required to contain a `_DllMainCRTStartup` function and to minimize our dependence on mingw32 libraries, we inject a stub definition ourselves. #### MacOS -Dynamic libraries on MacOS need to link against `-lSystem`. On recent MacOS versions `-lSystem` is only available for linking when Xcode is available. +Dynamic libraries on macOS need to link against `-lSystem`. On recent macOS versions, `-lSystem` is only available for linking when Xcode is available. To that effect we link with `-undefined dynamic_lookup`. ## Package images optimized for multiple microarchitectures Similar to [multi-versioning](@ref sysimg-multi-versioning) for system images, package images support multi-versioning. If you are in a heterogenous environment, with a unified cache, you can set the environment variable `JULIA_CPU_TARGET` to multi-version the object caches. -## Flags that impact pkgimage +## Flags that impact package image creation and selection -These are the Julia command line flags that impact cache selection. Pkgimages +These are the Julia command line flags that impact cache selection. Package images that were created with different flags will be rejected. -- `--debug-level`: Exact match required since it changes codegeneration. -- `--checkbounds`: Exact match required since it changes codegeneration. +- `-g`: Exact match required since it changes code generation. +- `--check-bounds`: Exact match required since it changes code generation. - `--pkgimage-native-code`: To allow running without object caching enabled. -- `--optimize`: Reject pkgimages generated for a lower optimization level, +- `-O`, `--optimize`: Reject package images generated for a lower optimization level, but allow for higher optimization levels to be loaded. diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 74208dd876bb4..f39c1f6ae4178 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -392,7 +392,7 @@ dependencies. ### [Package/Environment Preferences](@id preferences) Preferences are dictionaries of metadata that influence package behavior within an environment. -The preferences system supports reading preferences at compile-time, which means that at code-loading time, we must ensure that the chosen precompilation files were built with the same preferences as the current environment before loading them. +The preferences system supports reading preferences at compile-time, which means that at code-loading time, we must ensure that the precompilation files selected by Julia were built with the same preferences as the current environment before loading them. The public API for modifying Preferences is contained within the [Preferences.jl](https://github.com/JuliaPackaging/Preferences.jl) package. Preferences are stored as TOML dictionaries within a `(Julia)LocalPreferences.toml` file next to the currently-active project. If a preference is "exported", it is instead stored within the `(Julia)Project.toml` instead. diff --git a/doc/src/manual/methods.md b/doc/src/manual/methods.md index 2e9629122e4af..a504f8e3511b2 100644 --- a/doc/src/manual/methods.md +++ b/doc/src/manual/methods.md @@ -295,7 +295,7 @@ julia> mysum(1.0, 2.0) ``` Julia will compile `mysum` twice, once for `x::Int, y::Int` and again for `x::Float64, y::Float64`. -The point of compiling twice is performance: the methods that get called for `+` (which `mysum` uses) vary depending on the specific types of `x` and `y`, and by compiling different specializations Julia can do all the method lookup ahead of time. This allows the program to run much more quickly, since it doesn't have to bother with method lookup while it is running. +The point of compiling twice is performance: the methods that get called for `+` (which `mysum` uses) vary depending on the specific types of `x` and `y`, and by compiling different specializations Julia can do all the method lookup ahead of time. This allows the program to run much more quickly, since it does not have to bother with method lookup while it is running. Julia's automatic specialization allows you to write generic algorithms and expect that the compiler will generate efficient, specialized code to handle each case you need. In cases where the number of potential specializations might be effectively unlimited, Julia may avoid this default specialization. See [Be aware of when Julia avoids specializing](@ref) for more information. diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index a9d97b415eac5..3cb14a03ad01b 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -430,12 +430,12 @@ often involves compiling a large amount of code. Julia creates precompiled caches of the module to reduce this time. Precompiled module files (sometimes called "cache files") are created and used automatically when `import` or `using` loads a module. If the cache file(s) do not yet exist, the module will be compiled and saved for future reuse. You can also manually call [`Base.compilecache(Base.identify_package("modulename"))`](@ref) to create these files without loading the module. The resulting -cache files will be stored in `DEPOT_PATH[1]/compiled/`. If nothing about your system changes, +cache files will be stored in the `compiled` subfolder of `DEPOT_PATH[1]`. If nothing about your system changes, such cache files will be used when you load the module with `import` or `using`. Precompilation cache files store definitions of modules, types, methods, and constants. They may also store method specializations and the code generated for them, but this typically requires that the developer add explicit [`precompile`](@ref) directives or execute workloads that force compilation during the package build. -However, if you update the modules' dependencies or change its source code, the module is automatically +However, if you update the module's dependencies or change its source code, the module is automatically recompiled upon `using` or `import`. Dependencies are modules it imports, the Julia build, files it includes, or explicit dependencies declared by [`include_dependency(path)`](@ref) in the module file(s). From 22233f6fbba0a12cab541a16c9223088a7eb0765 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 13 Dec 2022 10:39:05 -0600 Subject: [PATCH 06/25] Docs: command-line options --- doc/src/devdocs/pkgimg.md | 6 +++--- doc/src/manual/command-line-interface.md | 9 +++++---- src/jloptions.c | 12 ++++++------ 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/doc/src/devdocs/pkgimg.md b/doc/src/devdocs/pkgimg.md index 9e1cbc32670f8..915bc51348fe0 100644 --- a/doc/src/devdocs/pkgimg.md +++ b/doc/src/devdocs/pkgimg.md @@ -1,7 +1,7 @@ # Package Images Julia package images provide object (native code) caches for Julia packages. -They are similar to Julia's [system image](@dev-sysimg) and support many of the same features. +They are similar to Julia's [system image](@ref dev-sysimg) and support many of the same features. In fact the underlying serialization format is the same, and the system image is the base image that the package images are build against. ## High-level overview @@ -33,14 +33,14 @@ To that effect we link with `-undefined dynamic_lookup`. ## Package images optimized for multiple microarchitectures Similar to [multi-versioning](@ref sysimg-multi-versioning) for system images, package images support multi-versioning. If you are in a heterogenous environment, with a unified cache, -you can set the environment variable `JULIA_CPU_TARGET` to multi-version the object caches. +you can set the environment variable `JULIA_CPU_TARGET=generic` to multi-version the object caches. ## Flags that impact package image creation and selection These are the Julia command line flags that impact cache selection. Package images that were created with different flags will be rejected. -- `-g`: Exact match required since it changes code generation. +- `-g`, `--debug-info`: Exact match required since it changes code generation. - `--check-bounds`: Exact match required since it changes code generation. - `--pkgimage-native-code`: To allow running without object caching enabled. - `-O`, `--optimize`: Reject package images generated for a lower optimization level, diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 4af3c05d51eb6..7a50a3e903e91 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -88,7 +88,7 @@ There are various ways to run Julia code and provide options, similar to those a julia [switches] -- [programfile] [args...] ``` -The following is a complete list of command-line switches available when launching julia (a '*' marks the default value, if applicable): +The following is a complete list of command-line switches available when launching julia (a '*' marks the default value, if applicable; settings marked '($)' may trigger package precompilation): |Switch |Description| |:--- |:---| @@ -101,6 +101,7 @@ The following is a complete list of command-line switches available when launchi |`--startup-file={yes*\|no}` |Load `JULIA_DEPOT_PATH/config/startup.jl`; if `JULIA_DEPOT_PATH` environment variable is unset, load `~/.julia/config/startup.jl`| |`--handle-signals={yes*\|no}` |Enable or disable Julia's default signal handlers| |`--sysimage-native-code={yes*\|no}` |Use native code from system image if available| +|`--pkgimage-native-code={yes*|no}` |Use native code from package images if available ($)| |`--compiled-modules={yes*\|no}` |Enable or disable incremental precompilation of modules| |`-e`, `--eval ` |Evaluate ``| |`-E`, `--print ` |Evaluate `` and display the result| @@ -117,11 +118,11 @@ The following is a complete list of command-line switches available when launchi |`--warn-overwrite={yes\|no*}` |Enable or disable method overwrite warnings| |`--warn-scope={yes*\|no}` |Enable or disable warning for ambiguous top-level scope| |`-C`, `--cpu-target ` |Limit usage of CPU features up to ``; set to `help` to see the available options| -|`-O`, `--optimize={0,1,2*,3}` |Set the optimization level (level is 3 if `-O` is used without a level)| +|`-O`, `--optimize={0,1,2*,3}` |Set the optimization level (level is 3 if `-O` is used without a level) ($)| |`--min-optlevel={0*,1,2,3}` |Set the lower bound on per-module optimization| -|`-g {0,1*,2}` |Set the level of debug info generation (level is 2 if `-g` is used without a level)| +|`-g`, `--debug-info={0,1*,2}` |Set the level of debug info generation (level is 2 if `-g` is used without a level) ($)| |`--inline={yes\|no}` |Control whether inlining is permitted, including overriding `@inline` declarations| -|`--check-bounds={yes\|no\|auto*}` |Emit bounds checks always, never, or respect `@inbounds` declarations| +|`--check-bounds={yes\|no\|auto*}` |Emit bounds checks always, never, or respect `@inbounds` declarations ($)| |`--math-mode={ieee,fast}` |Disallow or enable unsafe floating point optimizations (overrides `@fastmath` declaration)| |`--code-coverage[={none*\|user\|all}]` |Count executions of source lines (omitting setting is equivalent to `user`)| |`--code-coverage=tracefile.info` |Append coverage information to the LCOV tracefile (filename supports format tokens).| diff --git a/src/jloptions.c b/src/jloptions.c index dcb3c7532539b..eb875711296fc 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -93,7 +93,7 @@ JL_DLLEXPORT void jl_init_options(void) static const char usage[] = "\n julia [switches] -- [programfile] [args...]\n\n"; static const char opts[] = - "Switches (a '*' marks the default value, if applicable):\n\n" + "Switches (a '*' marks the default value, if applicable; settings marked '($)' may trigger package precompilation):\n\n" " -v, --version Display version information\n" " -h, --help Print this message (--help-hidden for more)\n" " --help-hidden Uncommon options not shown by `-h`\n\n" @@ -108,7 +108,7 @@ static const char opts[] = " --sysimage-native-code={yes*|no}\n" " Use native code from system image if available\n" " --pkgimage-native-code={yes*|no}\n" - " Use native code from package images if available\n" + " Use native code from package images if available ($)\n" " --compiled-modules={yes*|no}\n" " Enable or disable incremental precompilation of modules\n\n" @@ -146,16 +146,16 @@ static const char opts[] = // code generation options " -C, --cpu-target Limit usage of CPU features up to ; set to `help` to see the available options\n" - " -O, --optimize={0,1,2*,3} Set the optimization level (level 3 if `-O` is used without a level)\n" + " -O, --optimize={0,1,2*,3} Set the optimization level (level 3 if `-O` is used without a level) ($)\n" " --min-optlevel={0*,1,2,3} Set a lower bound on the optimization level\n" #ifdef JL_DEBUG_BUILD - " -g, --debug-info=[{0,1,2*}] Set the level of debug info generation in the julia-debug build\n" + " -g, --debug-info=[{0,1,2*}] Set the level of debug info generation in the julia-debug build ($)\n" #else - " -g, --debug-info=[{0,1*,2}] Set the level of debug info generation (level 2 if `-g` is used without a level)\n" + " -g, --debug-info=[{0,1*,2}] Set the level of debug info generation (level 2 if `-g` is used without a level) ($)\n" #endif " --inline={yes*|no} Control whether inlining is permitted, including overriding @inline declarations\n" " --check-bounds={yes|no|auto*}\n" - " Emit bounds checks always, never, or respect @inbounds declarations\n" + " Emit bounds checks always, never, or respect @inbounds declarations ($)\n" #ifdef USE_POLLY " --polly={yes*|no} Enable or disable the polyhedral optimizer Polly (overrides @polly declaration)\n" #endif From b8872d1fbbe115077dd2edfdaaa9ba2b6f5a9c7f Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 13 Dec 2022 10:40:23 -0600 Subject: [PATCH 07/25] Expand "link" to devdocs Co-authored-by: Michael Schlottke-Lakemper --- doc/src/manual/modules.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index 3cb14a03ad01b..197c70e4c80fb 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -601,4 +601,4 @@ precompilation triggering when installing, updating, and explicitly building pac You can also debug some precompilation failures with environment variables. Setting `JULIA_VERBOSE_LINKING=true` may help resolve failures in linking shared libraries of compiled -native code. See the `devdocs` section of the manual for further details. +native code. See the **Developer Documentation** part of the Julia manual, where you will find further details in the section documenting Julia's internals under "Package Images". From 8b308f1bc186fa6de9ac35154dd50b0e056e454d Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Tue, 13 Dec 2022 11:41:16 -0600 Subject: [PATCH 08/25] Update doc/src/manual/command-line-interface.md Co-authored-by: Alex Ames --- doc/src/manual/command-line-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 7a50a3e903e91..1576309f9d898 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -101,7 +101,7 @@ The following is a complete list of command-line switches available when launchi |`--startup-file={yes*\|no}` |Load `JULIA_DEPOT_PATH/config/startup.jl`; if `JULIA_DEPOT_PATH` environment variable is unset, load `~/.julia/config/startup.jl`| |`--handle-signals={yes*\|no}` |Enable or disable Julia's default signal handlers| |`--sysimage-native-code={yes*\|no}` |Use native code from system image if available| -|`--pkgimage-native-code={yes*|no}` |Use native code from package images if available ($)| +|`--pkgimage-native-code={yes*\|no}` |Use native code from package images if available ($)| |`--compiled-modules={yes*\|no}` |Enable or disable incremental precompilation of modules| |`-e`, `--eval ` |Evaluate ``| |`-E`, `--print ` |Evaluate `` and display the result| From 58b4ac12ef996498c2a58ede5288e90b281a60c4 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 13 Dec 2022 21:23:53 -0500 Subject: [PATCH 09/25] Mixin cpu_target --- base/loading.jl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/base/loading.jl b/base/loading.jl index 587353ff3b820..b470c51d33b9d 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1966,6 +1966,12 @@ function compilecache_path(pkg::PkgId, prefs_hash::UInt64)::String crc = _crc32c(unsafe_string(JLOptions().julia_bin), crc) crc = _crc32c(ccall(:jl_cache_flags, UInt8, ()), crc) + cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) + if cpu_target === nothing + cpu_target = unsafe_string(JLOptions().cpu_target) + end + crc = _crc32c(cpu_target, crc) + crc = _crc32c(prefs_hash, crc) project_precompile_slug = slug(crc, 5) abspath(cachepath, string(entryfile, "_", project_precompile_slug, ".ji")) From 1785d517abec912afe13e8c5f7d02041bd211d5d Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 13 Dec 2022 21:31:15 -0500 Subject: [PATCH 10/25] Don't overcount --- base/loading.jl | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index b470c51d33b9d..e320ed351770f 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2077,10 +2077,16 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in # prune the directory with cache files if pkg.uuid !== nothing entrypath, entryfile = cache_file_entry(pkg) - cachefiles = filter!(x -> startswith(x, entryfile * "_"), readdir(cachepath)) + cachefiles = filter!(x -> startswith(x, entryfile * "_") && endswith(x, ".ji"), readdir(cachepath)) + if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES[] idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2] - rm(joinpath(cachepath, cachefiles[idx]); force=true) + cachefile = joinpath(cachepath, cachefiles[idx]) + rm(cachefile; force=true) + try + rm(ocachefile_from_cachefile(cachefile); force=true) + catch + end end end From 5c114ba4fa616f4581a88c1d06d73299c240abad Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 13 Dec 2022 21:36:36 -0500 Subject: [PATCH 11/25] Update the documentation a bit --- doc/src/devdocs/pkgimg.md | 3 ++- doc/src/manual/environment-variables.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/src/devdocs/pkgimg.md b/doc/src/devdocs/pkgimg.md index 915bc51348fe0..44054736e57b1 100644 --- a/doc/src/devdocs/pkgimg.md +++ b/doc/src/devdocs/pkgimg.md @@ -8,7 +8,8 @@ In fact the underlying serialization format is the same, and the system image is Package images are shared libraries that contain both code and data. Like `.ji` cache files, they are generated per package. The data section contains both global data (global variables in the package) as well as the necessary metadata about what methods and types are defined by the package. The code section contains native objects that cache the final output of Julia's LLVM-based compiler. -The command line option `--pkgimage-native-code=no` can be used to turn-off object caching for this session. Note that this means that cache files have to be regenerated. +The command line option `--pkgimage-native-code=no` can be used to turn off object caching for this session. Note that this means that cache files have to likely be regenerated. +See [`JULIA_MAX_NUM_PRECOMPILE_FILES`](@ref env-max-num-precompile-files) for the upper limit of variants Julia caches per default. !!! note While the package images present themselves as native shared libraries, they are only an approximation thereof. You will not be able to link against them from a native program and they must be loaded from Julia. diff --git a/doc/src/manual/environment-variables.md b/doc/src/manual/environment-variables.md index 26a7316d71da2..f29e5b7aaf8f7 100644 --- a/doc/src/manual/environment-variables.md +++ b/doc/src/manual/environment-variables.md @@ -162,7 +162,7 @@ The absolute path `REPL.find_hist_file()` of the REPL's history file. If $(DEPOT_PATH[1])/logs/repl_history.jl ``` -### `JULIA_MAX_NUM_PRECOMPILE_FILES` +### [`JULIA_MAX_NUM_PRECOMPILE_FILES`](@id env-max-num-precompile-files) Sets the maximum number of different instances of a single package that are to be stored in the precompile cache (default = 10). From 71622a1461052ee2119cbe573acb455fa6d4bbfc Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 13 Dec 2022 21:54:34 -0500 Subject: [PATCH 12/25] Reject .ji on pkgimage --- base/loading.jl | 8 ++++---- src/staticdata_utils.c | 18 ++++++++++++++---- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index e320ed351770f..151bbac40c89e 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2508,12 +2508,12 @@ end if isempty(modules) return true # ignore empty file end + if ccall(:jl_match_cache_flags, UInt8, (UInt8,), flags) == 0 + @debug "Rejecting cache file $cachefile for $modkey since the flags are mismatched" cachefile_flags=flags current_flags=ccall(:jl_cache_flags, UInt8, ()) + return true + end pkgimage = !isempty(clone_targets) if pkgimage - if ccall(:jl_match_cache_flags, UInt8, (UInt8,), flags) == 0 - @debug "Rejecting cache file $cachefile for $modkey since the flags are mismatched" cachefile_flags=flags current_flags=ccall(:jl_cache_flags, UInt8, ()) - return true - end ocachefile = ocachefile_from_cachefile(cachefile) if JLOptions().use_pkgimage_native_code == 0 # presence of clone_targets means native code cache diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 2c26c11ba8f81..7e6fdc8479d24 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -613,11 +613,11 @@ static void write_mod_list(ios_t *s, jl_array_t *a) JL_DLLEXPORT uint8_t jl_cache_flags(void) { - // ??OOPCDD + // ??OOCDDP uint8_t flags = 0; - flags |= jl_options.debug_level & 3; + flags |= (jl_options.use_pkgimage_native_code & 1); + flags |= (jl_options.debug_level & 3) << 1; flags |= (jl_options.check_bounds & 1) << 2; - flags |= (jl_options.use_pkgimage_native_code & 1) << 3; flags |= (jl_options.opt_level & 3) << 4; // NOTES: // In contrast to check-bounds, inline has no "observable effect" @@ -626,11 +626,21 @@ JL_DLLEXPORT uint8_t jl_cache_flags(void) JL_DLLEXPORT uint8_t jl_match_cache_flags(uint8_t flags) { + // 1. Check which flags are relevant uint8_t current_flags = jl_cache_flags(); + uint8_t supports_pkgimage = (current_flags & 1); + uint8_t is_pkgimage = (flags & 1); + + // For .ji packages ignore other flags + if (!supports_pkgimage && !is_pkgimage) { + return 1; + } + + // 2. Check all flags that must be exact uint8_t mask = (1 << 4)-1; if ((flags & mask) != (current_flags & mask)) return 0; - // allow for higher optimization flags in cache + // 3. allow for higher optimization flags in cache flags >>= 4; current_flags >>= 4; return flags >= current_flags; From 6ff32847af1a7fe1067628f64f690d95cd70daf2 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Tue, 13 Dec 2022 22:11:12 -0500 Subject: [PATCH 13/25] fixup! Don't overcount --- base/loading.jl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 151bbac40c89e..69dc04eb57751 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2081,10 +2081,11 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in if length(cachefiles) >= MAX_NUM_PRECOMPILE_FILES[] idx = findmin(mtime.(joinpath.(cachepath, cachefiles)))[2] - cachefile = joinpath(cachepath, cachefiles[idx]) - rm(cachefile; force=true) + evicted_cachefile = joinpath(cachepath, cachefiles[idx]) + @debug "Evicting file from cache" evicted_cachefile + rm(evicted_cachefile; force=true) try - rm(ocachefile_from_cachefile(cachefile); force=true) + rm(ocachefile_from_cachefile(evicted_cachefile); force=true) catch end end From c405552c7120f9209355fccb77eb36d8091db529 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 15 Dec 2022 12:40:21 -0600 Subject: [PATCH 14/25] Fix & test PkgCacheInspector --- base/loading.jl | 2 +- src/staticdata.c | 4 ++-- test/precompile.jl | 55 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index b1fca5b178379..8a45b53cdd672 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -946,7 +946,7 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No if ocachepath !== nothing @debug "Loading object cache file $ocachepath for $pkg" - sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any), ocachepath, depmods) + sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint), ocachepath, depmods, false) else @debug "Loading cache file $path for $pkg" sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint), path, depmods, false) diff --git a/src/staticdata.c b/src/staticdata.c index 50bdeaed50c3d..6d306f927dce5 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -3396,7 +3396,7 @@ JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len) JL_SIGATOMIC_END(); } -JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods) +JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, jl_array_t *depmods, int complete) { void *pkgimg_handle = jl_dlopen(fname, JL_RTLD_LAZY); if (!pkgimg_handle) { @@ -3448,7 +3448,7 @@ JL_DLLEXPORT jl_value_t *jl_restore_package_image_from_file(const char *fname, j } #endif - jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, 0); + jl_value_t* mod = jl_restore_incremental_from_buf(pkgimg_data, &pkgimage, *plen, depmods, complete); return mod; } diff --git a/test/precompile.jl b/test/precompile.jl index 952af1e869aef..7014c041fa5ba 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1616,6 +1616,61 @@ precompile_test_harness("Module tparams") do load_path @test ModuleTparams.the_struct === Base.invokelatest(ModuleTparams.ParamStruct{ModuleTparams.TheTParam}) end +precompile_test_harness("PkgCacheInspector") do load_path + # Test functionality needed by PkgCacheInspector.jl + write(joinpath(load_path, "PCI.jl"), + """ + module PCI + Base.repl_cmd() = 55 # external method + f() = Base.repl_cmd(7, "hello") # external specialization (should never exist otherwise) + try + f() + catch + end + end + """) + cachefile, ocachefile = Base.compilecache(Base.PkgId("PCI")) + + # Get the depmods + local depmods + @lock Base.require_lock begin + local depmodnames + io = open(cachefile, "r") + try + # isvalid_cache_header returns checksum id or zero + Base.isvalid_cache_header(io) == 0 && throw(ArgumentError("Invalid header in cache file $cachefile.")) + depmodnames = Base.parse_cache_header(io)[3] + Base.isvalid_file_crc(io) || throw(ArgumentError("Invalid checksum in cache file $cachefile.")) + finally + close(io) + end + ndeps = length(depmodnames) + depmods = Vector{Any}(undef, ndeps) + for i in 1:ndeps + modkey, build_id = depmodnames[i] + dep = Base._tryrequire_from_serialized(modkey, build_id) + if !isa(dep, Module) + return dep + end + depmods[i] = dep + end + end + + if ocachefile !== nothing + sv = ccall(:jl_restore_package_image_from_file, Any, (Cstring, Any, Cint), ocachefile, depmods, true) + else + sv = ccall(:jl_restore_incremental, Any, (Cstring, Any, Cint), cachefile, depmods, true) + end + + modules, init_order, external_methods, new_specializations, new_method_roots, external_targets, edges = sv + m = only(external_methods) + @test m.name == :repl_cmd && m.nargs < 2 + @test any(new_specializations) do ci + mi = ci.def + mi.specTypes == Tuple{typeof(Base.repl_cmd), Int64, String} + end +end + empty!(Base.DEPOT_PATH) append!(Base.DEPOT_PATH, original_depot_path) empty!(Base.LOAD_PATH) From 6373b890fc82ac71420454ca613c06548219b91c Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 15 Dec 2022 13:25:20 -0600 Subject: [PATCH 15/25] Split post-load work into separate function The motiviation here is to allow PkgCacheInspector to both load the package as well as return the contents of the cache file. That makes it possible to collect & compare multiple cache files within the same session. --- base/loading.jl | 54 +++++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 24 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 8a45b53cdd672..5c387bb310fae 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -954,31 +954,8 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No if isa(sv, Exception) return sv end - sv = sv::SimpleVector - restored = sv[1]::Vector{Any} - for M in restored - M = M::Module - if isdefined(M, Base.Docs.META) && getfield(M, Base.Docs.META) !== nothing - push!(Base.Docs.modules, M) - end - if parentmodule(M) === M - register_root_module(M) - end - end - # Register this cache path now - If Requires.jl is loaded, Revise may end - # up looking at the cache path during the init callback. - get!(PkgOrigin, pkgorigins, pkg).cachepath = path - - inits = sv[2]::Vector{Any} - if !isempty(inits) - unlock(require_lock) # temporarily _unlock_ during these callbacks - try - ccall(:jl_init_restored_modules, Cvoid, (Any,), inits) - finally - lock(require_lock) - end - end + restored = register_restored_modules(sv, pkg, path) for M in restored M = M::Module @@ -1007,6 +984,35 @@ function _include_from_serialized(pkg::PkgId, path::String, ocachepath::Union{No end end +function register_restored_modules(sv::SimpleVector, pkg::PkgId, path::String) + # This function is also used by PkgCacheInspector.jl + restored = sv[1]::Vector{Any} + for M in restored + M = M::Module + if isdefined(M, Base.Docs.META) && getfield(M, Base.Docs.META) !== nothing + push!(Base.Docs.modules, M) + end + if parentmodule(M) === M + register_root_module(M) + end + end + + # Register this cache path now - If Requires.jl is loaded, Revise may end + # up looking at the cache path during the init callback. + get!(PkgOrigin, pkgorigins, pkg).cachepath = path + + inits = sv[2]::Vector{Any} + if !isempty(inits) + unlock(require_lock) # temporarily _unlock_ during these callbacks + try + ccall(:jl_init_restored_modules, Cvoid, (Any,), inits) + finally + lock(require_lock) + end + end + return restored +end + function run_package_callbacks(modkey::PkgId) assert_havelock(require_lock) unlock(require_lock) From 9b6bc2b333cb88ba2eda942018632dad9ed9923b Mon Sep 17 00:00:00 2001 From: Ian Date: Fri, 16 Dec 2022 23:04:24 -0500 Subject: [PATCH 16/25] fix 32bit test failure --- test/precompile.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/precompile.jl b/test/precompile.jl index 7014c041fa5ba..6b0489429bc4b 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1667,7 +1667,7 @@ precompile_test_harness("PkgCacheInspector") do load_path @test m.name == :repl_cmd && m.nargs < 2 @test any(new_specializations) do ci mi = ci.def - mi.specTypes == Tuple{typeof(Base.repl_cmd), Int64, String} + mi.specTypes == Tuple{typeof(Base.repl_cmd), Int, String} end end From bb10c8c3073dbf7ff40b72803417eeae61104ff6 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sun, 18 Dec 2022 10:12:18 -0500 Subject: [PATCH 17/25] Rename command line flag --- base/loading.jl | 4 +-- base/options.jl | 2 +- base/util.jl | 4 +-- doc/src/devdocs/pkgimg.md | 4 +-- doc/src/manual/command-line-interface.md | 2 +- doc/src/manual/modules.md | 2 +- src/jloptions.c | 36 ++++++++++++------------ src/jloptions.h | 2 +- src/julia.h | 6 ++-- src/staticdata_utils.c | 2 +- test/precompile.jl | 2 +- 11 files changed, 33 insertions(+), 33 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index 5c387bb310fae..34db1091dff74 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2025,7 +2025,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in # create a temporary file in `cachepath` directory, write the cache in it, # write the checksum, _and then_ atomically move the file to `cachefile`. mkpath(cachepath) - cache_objects = JLOptions().use_pkgimage_native_code != 0 + cache_objects = JLOptions().use_pkgimages != 0 tmppath, tmpio = mktemp(cachepath) if cache_objects @@ -2523,7 +2523,7 @@ end pkgimage = !isempty(clone_targets) if pkgimage ocachefile = ocachefile_from_cachefile(cachefile) - if JLOptions().use_pkgimage_native_code == 0 + if JLOptions().use_pkgimages == 0 # presence of clone_targets means native code cache @debug "Rejecting cache file $cachefile for $modkey since it would require usage of pkgimage" return true diff --git a/base/options.jl b/base/options.jl index 8b580f6c900b0..dda0e8b377076 100644 --- a/base/options.jl +++ b/base/options.jl @@ -37,8 +37,8 @@ struct JLOptions cookie::Ptr{UInt8} handle_signals::Int8 use_sysimage_native_code::Int8 - use_pkgimage_native_code::Int8 use_compiled_modules::Int8 + use_pkgimages::Int8 bindto::Ptr{UInt8} outputbc::Ptr{UInt8} outputunoptbc::Ptr{UInt8} diff --git a/base/util.jl b/base/util.jl index e1fead9484dac..8d1a4a9fa02ef 100644 --- a/base/util.jl +++ b/base/util.jl @@ -225,8 +225,8 @@ function julia_cmd(julia=joinpath(Sys.BINDIR, julia_exename()); cpu_target::Unio if opts.use_sysimage_native_code == 0 push!(addflags, "--sysimage-native-code=no") end - if opts.use_pkgimage_native_code == 0 - push!(addflags, "--pkgimage-native-code=no") + if opts.use_pkgimages == 0 + push!(addflags, "--pkgimages=no") else # If pkgimage is set, malloc_log and code_coverage should not @assert opts.malloc_log == 0 && opts.code_coverage == 0 diff --git a/doc/src/devdocs/pkgimg.md b/doc/src/devdocs/pkgimg.md index 44054736e57b1..8230c4b91b338 100644 --- a/doc/src/devdocs/pkgimg.md +++ b/doc/src/devdocs/pkgimg.md @@ -8,7 +8,7 @@ In fact the underlying serialization format is the same, and the system image is Package images are shared libraries that contain both code and data. Like `.ji` cache files, they are generated per package. The data section contains both global data (global variables in the package) as well as the necessary metadata about what methods and types are defined by the package. The code section contains native objects that cache the final output of Julia's LLVM-based compiler. -The command line option `--pkgimage-native-code=no` can be used to turn off object caching for this session. Note that this means that cache files have to likely be regenerated. +The command line option `--pkgimages=no` can be used to turn off object caching for this session. Note that this means that cache files have to likely be regenerated. See [`JULIA_MAX_NUM_PRECOMPILE_FILES`](@ref env-max-num-precompile-files) for the upper limit of variants Julia caches per default. !!! note @@ -43,6 +43,6 @@ that were created with different flags will be rejected. - `-g`, `--debug-info`: Exact match required since it changes code generation. - `--check-bounds`: Exact match required since it changes code generation. -- `--pkgimage-native-code`: To allow running without object caching enabled. +- `--pkgimages`: To allow running without object caching enabled. - `-O`, `--optimize`: Reject package images generated for a lower optimization level, but allow for higher optimization levels to be loaded. diff --git a/doc/src/manual/command-line-interface.md b/doc/src/manual/command-line-interface.md index 1576309f9d898..e35cbf5e313e7 100644 --- a/doc/src/manual/command-line-interface.md +++ b/doc/src/manual/command-line-interface.md @@ -101,8 +101,8 @@ The following is a complete list of command-line switches available when launchi |`--startup-file={yes*\|no}` |Load `JULIA_DEPOT_PATH/config/startup.jl`; if `JULIA_DEPOT_PATH` environment variable is unset, load `~/.julia/config/startup.jl`| |`--handle-signals={yes*\|no}` |Enable or disable Julia's default signal handlers| |`--sysimage-native-code={yes*\|no}` |Use native code from system image if available| -|`--pkgimage-native-code={yes*\|no}` |Use native code from package images if available ($)| |`--compiled-modules={yes*\|no}` |Enable or disable incremental precompilation of modules| +|`--pkgimages={yes*\|no}` |Enable or disable usage of native code caching in the form of pkgimages| |`-e`, `--eval ` |Evaluate ``| |`-E`, `--print ` |Evaluate `` and display the result| |`-L`, `--load ` |Load `` immediately on all processors| diff --git a/doc/src/manual/modules.md b/doc/src/manual/modules.md index 197c70e4c80fb..90680828d2bc2 100644 --- a/doc/src/manual/modules.md +++ b/doc/src/manual/modules.md @@ -594,7 +594,7 @@ It is sometimes helpful during module development to turn off incremental precom command line flag `--compiled-modules={yes|no}` enables you to toggle module precompilation on and off. When Julia is started with `--compiled-modules=no` the serialized modules in the compile cache are ignored when loading modules and module dependencies. -More fine-grained control is available with `--pkgimage-native-code=no`, which suppresses only +More fine-grained control is available with `--pkgimages=no`, which suppresses only native-code storage during precompilation. `Base.compilecache` can still be called manually. The state of this command line flag is passed to `Pkg.build` to disable automatic precompilation triggering when installing, updating, and explicitly building packages. diff --git a/src/jloptions.c b/src/jloptions.c index eb875711296fc..45f6c0d441629 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -70,8 +70,8 @@ JL_DLLEXPORT void jl_init_options(void) NULL, // cookie JL_OPTIONS_HANDLE_SIGNALS_ON, JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES, - JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_YES, JL_OPTIONS_USE_COMPILED_MODULES_YES, + JL_OPTIONS_USE_PKGIMAGES_YES, NULL, // bind-to NULL, // output-bc NULL, // output-unopt-bc @@ -107,10 +107,10 @@ static const char opts[] = " --handle-signals={yes*|no} Enable or disable Julia's default signal handlers\n" " --sysimage-native-code={yes*|no}\n" " Use native code from system image if available\n" - " --pkgimage-native-code={yes*|no}\n" - " Use native code from package images if available ($)\n" " --compiled-modules={yes*|no}\n" - " Enable or disable incremental precompilation of modules\n\n" + " Enable or disable incremental precompilation of modules\n" + " --pkgimages={yes*|no}\n" + " Enable or disable usage of native code caching in the form of pkgimages\n\n" // actions " -e, --eval Evaluate \n" @@ -241,8 +241,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_help_hidden, opt_banner, opt_sysimage_native_code, - opt_pkgimage_native_code, opt_compiled_modules, + opt_pkgimages, opt_machine_file, opt_project, opt_bug_report, @@ -270,8 +270,8 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "bug-report", required_argument, 0, opt_bug_report }, { "sysimage", required_argument, 0, 'J' }, { "sysimage-native-code", required_argument, 0, opt_sysimage_native_code }, - { "pkgimage-native-code", required_argument, 0, opt_pkgimage_native_code }, { "compiled-modules",required_argument, 0, opt_compiled_modules }, + { "pkgimages", required_argument, 0, opt_pkgimages }, { "cpu-target", required_argument, 0, 'C' }, { "procs", required_argument, 0, 'p' }, { "threads", required_argument, 0, 't' }, @@ -442,15 +442,6 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) else jl_errorf("julia: invalid argument to --sysimage-native-code={yes|no} (%s)", optarg); break; - case opt_pkgimage_native_code: - pkgimage_explicit = 1; - if (!strcmp(optarg,"yes")) - jl_options.use_pkgimage_native_code = JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_YES; - else if (!strcmp(optarg,"no")) - jl_options.use_pkgimage_native_code = JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_NO; - else - jl_errorf("julia: invalid argument to --pkgimage-native-code={yes|no} (%s)", optarg); - break; case opt_compiled_modules: if (!strcmp(optarg,"yes")) jl_options.use_compiled_modules = JL_OPTIONS_USE_COMPILED_MODULES_YES; @@ -459,6 +450,15 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) else jl_errorf("julia: invalid argument to --compiled-modules={yes|no} (%s)", optarg); break; + case opt_pkgimages: + pkgimage_explicit = 1; + if (!strcmp(optarg,"yes")) + jl_options.use_pkgimages = JL_OPTIONS_USE_PKGIMAGES_YES; + else if (!strcmp(optarg,"no")) + jl_options.use_pkgimages = JL_OPTIONS_USE_PKGIMAGES_NO; + else + jl_errorf("julia: invalid argument to --pkgimage={yes|no} (%s)", optarg); + break; case 'C': // cpu-target jl_options.cpu_target = strdup(optarg); if (!jl_options.cpu_target) @@ -819,11 +819,11 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) } } if (codecov || malloclog) { - if (pkgimage_explicit && jl_options.use_pkgimage_native_code) { - jl_errorf("julia: Can't use --pkgimage-native-code=yes together " + if (pkgimage_explicit && jl_options.use_pkgimages) { + jl_errorf("julia: Can't use --pkgimages=yes together " "with --track-allocation or --code-coverage."); } - jl_options.use_pkgimage_native_code = 0; + jl_options.use_pkgimages = 0; } jl_options.code_coverage = codecov; jl_options.malloc_log = malloclog; diff --git a/src/jloptions.h b/src/jloptions.h index 88a256f02e91e..d0aba777027e7 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -41,8 +41,8 @@ typedef struct { const char *cookie; int8_t handle_signals; int8_t use_sysimage_native_code; - int8_t use_pkgimage_native_code; int8_t use_compiled_modules; + int8_t use_pkgimages; const char *bindto; const char *outputbc; const char *outputunoptbc; diff --git a/src/julia.h b/src/julia.h index d837e8d256184..739c4d856c18c 100644 --- a/src/julia.h +++ b/src/julia.h @@ -2180,12 +2180,12 @@ JL_DLLEXPORT int jl_generating_output(void) JL_NOTSAFEPOINT; #define JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_YES 1 #define JL_OPTIONS_USE_SYSIMAGE_NATIVE_CODE_NO 0 -#define JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_YES 1 -#define JL_OPTIONS_USE_PKGIMAGE_NATIVE_CODE_NO 0 - #define JL_OPTIONS_USE_COMPILED_MODULES_YES 1 #define JL_OPTIONS_USE_COMPILED_MODULES_NO 0 +#define JL_OPTIONS_USE_PKGIMAGES_YES 1 +#define JL_OPTIONS_USE_PKGIMAGES_NO 0 + // Version information #include // Generated file diff --git a/src/staticdata_utils.c b/src/staticdata_utils.c index 7e6fdc8479d24..6ff4a35df6be0 100644 --- a/src/staticdata_utils.c +++ b/src/staticdata_utils.c @@ -615,7 +615,7 @@ JL_DLLEXPORT uint8_t jl_cache_flags(void) { // ??OOCDDP uint8_t flags = 0; - flags |= (jl_options.use_pkgimage_native_code & 1); + flags |= (jl_options.use_pkgimages & 1); flags |= (jl_options.debug_level & 3) << 1; flags |= (jl_options.check_bounds & 1) << 2; flags |= (jl_options.opt_level & 3) << 4; diff --git a/test/precompile.jl b/test/precompile.jl index 6b0489429bc4b..e2db9087fe804 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -328,7 +328,7 @@ precompile_test_harness(false) do dir cachedir = joinpath(dir, "compiled", "v$(VERSION.major).$(VERSION.minor)") cachedir2 = joinpath(dir2, "compiled", "v$(VERSION.major).$(VERSION.minor)") cachefile = joinpath(cachedir, "$Foo_module.ji") - if Base.JLOptions().use_pkgimage_native_code == 1 + if Base.JLOptions().use_pkgimages == 1 ocachefile = Base.ocachefile_from_cachefile(cachefile) else ocachefile = nothing From 313bb9a4bcec78fa3cfb587860bfbc1dc858897d Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Sun, 18 Dec 2022 12:44:52 -0500 Subject: [PATCH 18/25] Apply suggestions from code review --- src/codegen.cpp | 2 -- src/debuginfo.cpp | 2 -- src/jloptions.c | 2 +- src/julia_internal.h | 2 +- 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/codegen.cpp b/src/codegen.cpp index 5b7de40a6408e..52ab03bd2b040 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -4037,7 +4037,6 @@ static jl_cgval_t emit_invoke(jl_codectx_t &ctx, const jl_cgval_t &lival, const if (ctx.external_linkage) { if (jl_object_in_image((jl_value_t*)codeinst)) { // Target is present in another pkgimage - jl_printf(JL_STDERR, "\n (emit_invoke:) Want to resolve method!\n"); cache_valid = true; external = true; } @@ -5400,7 +5399,6 @@ static Function *emit_tojlinvoke(jl_code_instance_t *codeinst, Module *M, jl_cod if (params.external_linkage) { if (jl_object_in_image((jl_value_t*)codeinst)) { // Target is present in another pkgimage - jl_printf(JL_STDERR, "\n (emit_jlinvoke) Want to resolve method\n"); cache_valid = true; } } diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index a4d1187077fa5..997c04aff6445 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -1124,8 +1124,6 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * *context = entry.ctx; if (entry.obj) *Section = getModuleSectionForAddress(entry.obj, pointer + entry.slide); - - // Assume we only need base address for sysimg for now if (!inimage || !image_info.fptrs.base) saddr = nullptr; diff --git a/src/jloptions.c b/src/jloptions.c index 45f6c0d441629..90bb39955ee42 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -110,7 +110,7 @@ static const char opts[] = " --compiled-modules={yes*|no}\n" " Enable or disable incremental precompilation of modules\n" " --pkgimages={yes*|no}\n" - " Enable or disable usage of native code caching in the form of pkgimages\n\n" + " Enable or disable usage of native code caching in the form of pkgimages ($)\n\n" // actions " -e, --eval Evaluate \n" diff --git a/src/julia_internal.h b/src/julia_internal.h index a2706159d7c90..1897c25cf906c 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -957,7 +957,7 @@ STATIC_INLINE size_t n_linkage_blobs(void) JL_NOTSAFEPOINT } // TODO: Makes this a binary search -STATIC_INLINE size_t external_blob_index(jl_value_t* v) JL_NOTSAFEPOINT { +STATIC_INLINE size_t external_blob_index(jl_value_t *v) JL_NOTSAFEPOINT { size_t i, nblobs = n_linkage_blobs(); assert(jl_linkage_blobs.len == 2*nblobs); for (i = 0; i < nblobs; i++) { From 0d469bd0f822ab9079c5bcd8d64780700ef3e987 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mos=C3=A8=20Giordano?= Date: Sun, 18 Dec 2022 20:11:12 +0000 Subject: [PATCH 19/25] [CompilerSupportLibraries_jll] Upgrade to v1.0.2 and install `libsspdll.a` on Windows (#47864) The new build provides the import library for `libssp` on Windows, instead of the static library. --- Makefile | 2 +- deps/checksums/compilersupportlibraries | 184 +++++++++--------- .../CompilerSupportLibraries_jll/Project.toml | 2 +- 3 files changed, 94 insertions(+), 94 deletions(-) diff --git a/Makefile b/Makefile index f085d005402ba..6670bf1206fd3 100644 --- a/Makefile +++ b/Makefile @@ -269,7 +269,7 @@ endif # We have a single exception; we want 7z.dll to live in libexec, not bin, so that 7z.exe can find it. -mv $(DESTDIR)$(bindir)/7z.dll $(DESTDIR)$(libexecdir)/ -$(INSTALL_M) $(build_bindir)/libopenlibm.dll.a $(DESTDIR)$(libdir)/ - -$(INSTALL_M) $(build_libdir)/libssp.a $(DESTDIR)$(libdir)/ # TODO use implib + -$(INSTALL_M) $(build_libdir)/libssp.dll.a $(DESTDIR)$(libdir)/ # The rest are compiler dependencies, as an example memcpy is exported by msvcrt # These are files from mingw32 and required for creating shared libraries like our caches. -$(INSTALL_M) $(build_libdir)/libgcc_s.a $(DESTDIR)$(libdir)/ diff --git a/deps/checksums/compilersupportlibraries b/deps/checksums/compilersupportlibraries index 721ad2e8a8759..098c181ca5c87 100644 --- a/deps/checksums/compilersupportlibraries +++ b/deps/checksums/compilersupportlibraries @@ -1,92 +1,92 @@ -CompilerSupportLibraries.v1.0.1+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/20ebaad57850393b6ac9fa924e511fe4 -CompilerSupportLibraries.v1.0.1+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/020de4d8b0ff6bedbadaa305ff8445e6849f12053762ea4aa68412d1ec763dbd86f479587a2fbb862487f1feb04d976c38099ddf3887817a3d32b3f029cf85b1 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran3.tar.gz/md5/3908fa1a2f739b330e787468c9bfb5c8 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/1741e3403ac7aa99e7cfd9a01222c4153ed300f47cc1b347e1af1a6cd07a82caaa54b9cfbebae8751440420551621cc6524504413446d104f9493dff2c081853 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran4.tar.gz/md5/2444dbb7637b32cf543675cc12330878 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/8537f0b243df8544350c884021b21c585fd302e8dd462a30a6ee84c7a36a049133262e5d1bc362f972066b8e8d6a091c32c3b746bab1feb9fccf2e7cca65756c -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/d79c1434594c0c5e7d6be798bf52c99e -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/7e71accc401a45b51b298702fb4c79a2fc856c7b28f0935f6ad3a0db5381c55fe5432daff371842930d718024b7c6c1d80e2bd09d397145203673bebbe3496ae -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran3.tar.gz/md5/f212059053d99558a9b0bf54b20180e1 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran3.tar.gz/sha512/5c104b1282cec8a944e5d008f44a4d60f4394fd5d797fec7d1f487d13e7328cd9c88ec4916dabf18596d87160756bda914e4f8c5a356b5577f9349d0d9e976d6 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran4.tar.gz/md5/3e3b3795ee93ef317223050e803a9875 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran4.tar.gz/sha512/85d3c955e15f66bfe8bfec2f28c9160bc03d4d531ea4ffe6bc6b51e0d69ccea3ab67a16ca752dabc870861c407381c4519d75c6be3832e8dccd6122ec8c6ed75 -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/cf2d1315f6a348af2e6c065e2a286e7a -CompilerSupportLibraries.v1.0.1+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/58420377bc77aa7678034ee5f708eb6be7db359faef2c2638869765453633da9bf455512bd88e95b38ae0428ecc4053561517b176b2371129bdaef9d8d5dadfd -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/f5c09ed7e0eeb8d345d328f950582f26 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/9c657f55c8fcdeb404be168a3a63a5e84304730fe34f25673d92cdae4b0a1fcc6a877ee1433f060e1be854c7811d66632e32510a2ed591d88330f1340b9c20de -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/c685518aca4721cd8621d510e2039683 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b760468c6377dcd2b8dd50200daaabe604006afc070984d78152b2becd0680b59036c9a6e91dea490121bd85b58d285bfc1e1cf696d29af236528400101de36c -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/8faf5c8ad62ab10f71dd2ec9683053e2 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/921239f241a5c89710cf07272d7f6c3f10201a7533068ed1e9643f9fb2f439e1bb765a4966d913829866ee0ce4f1589d30d06e4b5c1361e3c016a9473f087177 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/b38fcb70691ac2621379d298eef8c79e -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/06c7f64257ce721f5941f6e50a0d2717cdc9394fc532ded19ce3eaacd5e92a416969534227562e4fee04d2b6340c650d8bc9779e14519b90038bc41e8d1f5ce3 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/cdfab2c7bc41765caf4441c3caeed761 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/7109d4a7b32c00309c42685f54a86fc2cc63c0c00f65584ad296b6e44ad3320eed1aaf49684a8831841cdffa5555d72f89272fb722a780596e27ef020528026b -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/441980ebd23d72772cbe603f1c275336 -CompilerSupportLibraries.v1.0.1+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/e273d9f1af259a3080df8f173e1808a1ade976a943aba97216bf59a96178e7c052e7a048b0ceee53ab486ed577a2ecb92579857be2f7b29e76322ee1f13c9d76 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/f5c09ed7e0eeb8d345d328f950582f26 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/9c657f55c8fcdeb404be168a3a63a5e84304730fe34f25673d92cdae4b0a1fcc6a877ee1433f060e1be854c7811d66632e32510a2ed591d88330f1340b9c20de -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/c685518aca4721cd8621d510e2039683 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b760468c6377dcd2b8dd50200daaabe604006afc070984d78152b2becd0680b59036c9a6e91dea490121bd85b58d285bfc1e1cf696d29af236528400101de36c -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/8faf5c8ad62ab10f71dd2ec9683053e2 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/921239f241a5c89710cf07272d7f6c3f10201a7533068ed1e9643f9fb2f439e1bb765a4966d913829866ee0ce4f1589d30d06e4b5c1361e3c016a9473f087177 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/b38fcb70691ac2621379d298eef8c79e -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/06c7f64257ce721f5941f6e50a0d2717cdc9394fc532ded19ce3eaacd5e92a416969534227562e4fee04d2b6340c650d8bc9779e14519b90038bc41e8d1f5ce3 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/cdfab2c7bc41765caf4441c3caeed761 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/7109d4a7b32c00309c42685f54a86fc2cc63c0c00f65584ad296b6e44ad3320eed1aaf49684a8831841cdffa5555d72f89272fb722a780596e27ef020528026b -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/441980ebd23d72772cbe603f1c275336 -CompilerSupportLibraries.v1.0.1+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/e273d9f1af259a3080df8f173e1808a1ade976a943aba97216bf59a96178e7c052e7a048b0ceee53ab486ed577a2ecb92579857be2f7b29e76322ee1f13c9d76 -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran3.tar.gz/md5/6decf8fd5afb50451771c761e63a8917 -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran3.tar.gz/sha512/4984724bcc847724b1bc005b6f760a18b68147f7d5402d0faf4e28fc0d14fa10975368a951f9caf2a8856500046dec8343043274557d58269e77492b929a9e4b -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran4.tar.gz/md5/39d1e8a3baa144c018d3eaf7f3806482 -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran4.tar.gz/sha512/fc4d429279c5a93b6c28b6e911b1e7cfd1c1cfe46f11f2e901b3832ce90d45f49d3d29f0ef18518a94af6cc8651f67c4ed81672680f9281ada390440b172a2af -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran5.tar.gz/md5/37dabd9cd224c9fed9633dedccb6c565 -CompilerSupportLibraries.v1.0.1+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/b253149e72eef9486888fbaace66e9b6945f4477f6b818f64f3047331165b0e2bc17aa6e3fc8c88686a72e478eb62c8f53883415d5419db448d8016fa3a1da5e -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran3.tar.gz/md5/afdd32bfadd465848e6be458817a44ae -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran3.tar.gz/sha512/eebd679c499143014514c7c9d1875dedbbab9e3af51526c4dd445a9e3dbade95d24522da8bbad0a50ab400755e47b018828b324c4ad7705e212ccd990e34439a -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran4.tar.gz/md5/bc4a0f0b7cea328f7e8850583774496b -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran4.tar.gz/sha512/82285b67946212b49cddf6259f2c60ff5469f8c5263ccefe44f1d93ace98ab68e2c152e1b54434b2f075fd8d192c06d5451bc8cca26d951ad15f3453102f02b5 -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran5.tar.gz/md5/177f0232abce8d523882530ed7a93092 -CompilerSupportLibraries.v1.0.1+0.i686-linux-musl-libgfortran5.tar.gz/sha512/db80acf0f2434f28ee7680e1beb34f564940071815d1ad89fb5913cbd9ac24da528e826d0d54be6265a7340ebd661b6d308ed79d96b67fa5d8c98dc3f1bee8d6 -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran3.tar.gz/md5/c723e7d3c3038f59b9bf0cc3a65826bc -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran3.tar.gz/sha512/0545561ccd7e525b6cd86935366a2724a5e013411a1c01564db21b66da5fef959cf06b0839b96f1dc2c970eb6c8fb19c012e6cd2c17bc381b55420c72fe1b9f6 -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran4.tar.gz/md5/763bd82645d2f3c72b6244d68bebb40f -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran4.tar.gz/sha512/700e719eeab486915a9fb0954125cb9a3e9a813d7a069eca05be3a16621f4875668918a5ed5f645e734ac62b0c2ddbaa6234adc9109e98fb88b8ca1197481ed8 -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/18e90d15dc6dd0a836e9aa076b342105 -CompilerSupportLibraries.v1.0.1+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/9ff61e8da2b431a8cb09818bde5daab2d7b8cf7a934f184f14ea50eccf5796ae91558e06a22137eb021c4055c54faf4a524a54dbbd718e8ea0abb5dcec844fdb -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/md5/4e5e4b23dc87450738da33926a07511d -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/sha512/fc09879d94b750e75775d8b64a41ab9924d675fb53c5700467604412928fe7f5cb21911da0f64898d2463fa77ffbaf4c96c397b9060f4746eec152747930cddc -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/md5/9a92138ed69aa317a932a615c6e62d69 -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/sha512/0b7785379936a2a209b074177b1424dd7e00b29b5165f564e799b0aa4e06a582e9d616525d97274ba2507cb88192028f1ac485d3f99bdc7ee53fc63c1a7e85de -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/8ffee3d6de5197c7a1f354d72c8238fa -CompilerSupportLibraries.v1.0.1+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/deadc4d7224c84f9b82dc956b69e815c44ae036802838365d870ab9f58c8bcf8ce0645f2f387c8ff344ac2108fc8e7e1ee907fa55e93c91aa5d9fd921bf3fdcb -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran3.tar.gz/md5/87449e72e3f33dbb69b7053cdc2649d4 -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/5ce02ad10c6f4686a476eb2a5de2988cd8b482f5e693db2880c84ad1c82f468ef03fe01b9d0feefe5d4ee741d1d16643d36b144e6261ed32311b3b6f312fac2f -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran4.tar.gz/md5/0407cde92cfa42fa89ac83217ca0ec16 -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/032c831f1166a336551138939ac40eb2c68a048ce786c0c1403b879a20c1b706caac16d22560b2c7f2b3d6373986c347188675674116005ca251336ee048d09f -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/23418763b808371ee94772a90d501f4d -CompilerSupportLibraries.v1.0.1+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/7867b843551457b11bda7821dd384c1c1cf23b80a308b2058a693de7b7da099f0b37eb0a6de2b84c04b625a68c60eea55138e200d5d6ec6f6af09bd7ce406a96 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran3.tar.gz/md5/e3d33ae03c18affea74699bdc1fabb68 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/42013f4921de5a69ad857195ce5c19ad1bca3c920d79699e5501f1f4534ab132fabd422362b2b5056f5d182215d6c069db5df460bafa700903faf962cc00f77b -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran4.tar.gz/md5/d40c1e8c0393213c6057c53a12f44175 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/fe7baa4de7490065ab7b953cc12f41462a24bcb49d0a4a64b23249e98e7569b19bb1cb455af2f76090e34066a7d3cdd7a48cae6515ce6c7a5c8486b0cacc5106 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/48541b90f715c4c86ee4da0570275947 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/7f2683fb98e80f12629f4ed3bea9fd59d32b7e7a9ed1699e782d8e238ff0915ecc61bf00adaf4597cfe41caf82cdca0f9be250f595f5f0bea6d8f77dba99eaf4 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran3.tar.gz/md5/4547059eb905995667be48bf85d49911 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran3.tar.gz/sha512/7400fdabc924434ab4a4949248c3603887ac06ffd2f205ae33e14495d86cd4f816bbd1999eeafa0257f518df1e7f7c522f596e847a71dbfbfccff4859f50acc7 -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran4.tar.gz/md5/46267543cad6584d7b7b9fcc8f18f21d -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran4.tar.gz/sha512/0353d7d724be48d4185d3c181692970b7996f53f6a01723072aa5c94b53a8c5055faeed30df51659c252a46f4b941dec0cb24569323e3c85c166f14c5b7c8e9e -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/14dba2897a6e9d370fa9091c045375fc -CompilerSupportLibraries.v1.0.1+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/10b79f9c059839f5b57fa8d2a381a034c4067262c4088bd354d14ea56bec097878069383aa9cfadaa09d73bd20fc348fb61662d863a8d62cb25d7af6b8e29858 -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/eed836d1addeb10d0901f836724aff1e -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/e33eca424d1529a1fb23ba9cf7fac345ed1cfc8073c975b6b31ca44d2e8c3f5083af65433df009b22483dceb2e43149f3c1e8433681fec5fb812e1d5b4243ce4 -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/d5ae9f9519341fdaabf62267c89461d2 -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/6421aa5d1bd6f08ad43f59ed4dc1bef8b9b598ebbbd3e48149730f3bec3471f8e2c02ffb338427326924290b8f52ef9e626e3313448bc931a61d866c5dc544ae -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/fc1df521395362a5aaa2e2aeef707207 -CompilerSupportLibraries.v1.0.1+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/f2e5a08e3cae171242ae6a20d2d4838c1529ce042745dc466148b7bbc06896d94476fd05c7787e6e8641bea752dfc0e6b09e95b160bede600d20d2ad68e7705f -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/80c337837a9032e4c9614f0d3218993b -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/cf07e459ca55cb9ee3d38e6858320530c1d1ab2ffd35bfa2a33b2505d3189f13b9743a0e279d70f85d227cee8a8974448f1371a122dcbea03fb1e414f8df8337 -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/792cae36932dd53af20b7f61c80f623b -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/805f2b64fe9d2b94fc6c966945e10458d8d1c47a8d95fcda057c03a13999d7d0f136c754e4b1e152faaf23e4949861c2ad42b4437dba19f59b3db745d7a76108 -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/063c07fcbba4b9c3bd23ab0d987f1dbb -CompilerSupportLibraries.v1.0.1+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/1d0344b30b5fb34a63f6844be0501c0ad08f1116b0c7b00e13d47860cc6bbdd39734416ad3b492414a28ba1744240bd05aca0d1560873f687d3f61747058626b +CompilerSupportLibraries.v1.0.2+0.aarch64-apple-darwin-libgfortran5.tar.gz/md5/20ebaad57850393b6ac9fa924e511fe4 +CompilerSupportLibraries.v1.0.2+0.aarch64-apple-darwin-libgfortran5.tar.gz/sha512/020de4d8b0ff6bedbadaa305ff8445e6849f12053762ea4aa68412d1ec763dbd86f479587a2fbb862487f1feb04d976c38099ddf3887817a3d32b3f029cf85b1 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran3.tar.gz/md5/3908fa1a2f739b330e787468c9bfb5c8 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran3.tar.gz/sha512/1741e3403ac7aa99e7cfd9a01222c4153ed300f47cc1b347e1af1a6cd07a82caaa54b9cfbebae8751440420551621cc6524504413446d104f9493dff2c081853 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran4.tar.gz/md5/2444dbb7637b32cf543675cc12330878 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran4.tar.gz/sha512/8537f0b243df8544350c884021b21c585fd302e8dd462a30a6ee84c7a36a049133262e5d1bc362f972066b8e8d6a091c32c3b746bab1feb9fccf2e7cca65756c +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran5.tar.gz/md5/d79c1434594c0c5e7d6be798bf52c99e +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-gnu-libgfortran5.tar.gz/sha512/7e71accc401a45b51b298702fb4c79a2fc856c7b28f0935f6ad3a0db5381c55fe5432daff371842930d718024b7c6c1d80e2bd09d397145203673bebbe3496ae +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran3.tar.gz/md5/f212059053d99558a9b0bf54b20180e1 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran3.tar.gz/sha512/5c104b1282cec8a944e5d008f44a4d60f4394fd5d797fec7d1f487d13e7328cd9c88ec4916dabf18596d87160756bda914e4f8c5a356b5577f9349d0d9e976d6 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran4.tar.gz/md5/3e3b3795ee93ef317223050e803a9875 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran4.tar.gz/sha512/85d3c955e15f66bfe8bfec2f28c9160bc03d4d531ea4ffe6bc6b51e0d69ccea3ab67a16ca752dabc870861c407381c4519d75c6be3832e8dccd6122ec8c6ed75 +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran5.tar.gz/md5/cf2d1315f6a348af2e6c065e2a286e7a +CompilerSupportLibraries.v1.0.2+0.aarch64-linux-musl-libgfortran5.tar.gz/sha512/58420377bc77aa7678034ee5f708eb6be7db359faef2c2638869765453633da9bf455512bd88e95b38ae0428ecc4053561517b176b2371129bdaef9d8d5dadfd +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/md5/f5c09ed7e0eeb8d345d328f950582f26 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran3.tar.gz/sha512/9c657f55c8fcdeb404be168a3a63a5e84304730fe34f25673d92cdae4b0a1fcc6a877ee1433f060e1be854c7811d66632e32510a2ed591d88330f1340b9c20de +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/md5/c685518aca4721cd8621d510e2039683 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b760468c6377dcd2b8dd50200daaabe604006afc070984d78152b2becd0680b59036c9a6e91dea490121bd85b58d285bfc1e1cf696d29af236528400101de36c +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/md5/8faf5c8ad62ab10f71dd2ec9683053e2 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-gnueabihf-libgfortran5.tar.gz/sha512/921239f241a5c89710cf07272d7f6c3f10201a7533068ed1e9643f9fb2f439e1bb765a4966d913829866ee0ce4f1589d30d06e4b5c1361e3c016a9473f087177 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/md5/b38fcb70691ac2621379d298eef8c79e +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran3.tar.gz/sha512/06c7f64257ce721f5941f6e50a0d2717cdc9394fc532ded19ce3eaacd5e92a416969534227562e4fee04d2b6340c650d8bc9779e14519b90038bc41e8d1f5ce3 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/md5/cdfab2c7bc41765caf4441c3caeed761 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran4.tar.gz/sha512/7109d4a7b32c00309c42685f54a86fc2cc63c0c00f65584ad296b6e44ad3320eed1aaf49684a8831841cdffa5555d72f89272fb722a780596e27ef020528026b +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/md5/441980ebd23d72772cbe603f1c275336 +CompilerSupportLibraries.v1.0.2+0.armv6l-linux-musleabihf-libgfortran5.tar.gz/sha512/e273d9f1af259a3080df8f173e1808a1ade976a943aba97216bf59a96178e7c052e7a048b0ceee53ab486ed577a2ecb92579857be2f7b29e76322ee1f13c9d76 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/md5/f5c09ed7e0eeb8d345d328f950582f26 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran3.tar.gz/sha512/9c657f55c8fcdeb404be168a3a63a5e84304730fe34f25673d92cdae4b0a1fcc6a877ee1433f060e1be854c7811d66632e32510a2ed591d88330f1340b9c20de +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/md5/c685518aca4721cd8621d510e2039683 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran4.tar.gz/sha512/b760468c6377dcd2b8dd50200daaabe604006afc070984d78152b2becd0680b59036c9a6e91dea490121bd85b58d285bfc1e1cf696d29af236528400101de36c +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/md5/8faf5c8ad62ab10f71dd2ec9683053e2 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-gnueabihf-libgfortran5.tar.gz/sha512/921239f241a5c89710cf07272d7f6c3f10201a7533068ed1e9643f9fb2f439e1bb765a4966d913829866ee0ce4f1589d30d06e4b5c1361e3c016a9473f087177 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/md5/b38fcb70691ac2621379d298eef8c79e +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran3.tar.gz/sha512/06c7f64257ce721f5941f6e50a0d2717cdc9394fc532ded19ce3eaacd5e92a416969534227562e4fee04d2b6340c650d8bc9779e14519b90038bc41e8d1f5ce3 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/md5/cdfab2c7bc41765caf4441c3caeed761 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran4.tar.gz/sha512/7109d4a7b32c00309c42685f54a86fc2cc63c0c00f65584ad296b6e44ad3320eed1aaf49684a8831841cdffa5555d72f89272fb722a780596e27ef020528026b +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/md5/441980ebd23d72772cbe603f1c275336 +CompilerSupportLibraries.v1.0.2+0.armv7l-linux-musleabihf-libgfortran5.tar.gz/sha512/e273d9f1af259a3080df8f173e1808a1ade976a943aba97216bf59a96178e7c052e7a048b0ceee53ab486ed577a2ecb92579857be2f7b29e76322ee1f13c9d76 +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran3.tar.gz/md5/6decf8fd5afb50451771c761e63a8917 +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran3.tar.gz/sha512/4984724bcc847724b1bc005b6f760a18b68147f7d5402d0faf4e28fc0d14fa10975368a951f9caf2a8856500046dec8343043274557d58269e77492b929a9e4b +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran4.tar.gz/md5/39d1e8a3baa144c018d3eaf7f3806482 +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran4.tar.gz/sha512/fc4d429279c5a93b6c28b6e911b1e7cfd1c1cfe46f11f2e901b3832ce90d45f49d3d29f0ef18518a94af6cc8651f67c4ed81672680f9281ada390440b172a2af +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran5.tar.gz/md5/37dabd9cd224c9fed9633dedccb6c565 +CompilerSupportLibraries.v1.0.2+0.i686-linux-gnu-libgfortran5.tar.gz/sha512/b253149e72eef9486888fbaace66e9b6945f4477f6b818f64f3047331165b0e2bc17aa6e3fc8c88686a72e478eb62c8f53883415d5419db448d8016fa3a1da5e +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran3.tar.gz/md5/afdd32bfadd465848e6be458817a44ae +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran3.tar.gz/sha512/eebd679c499143014514c7c9d1875dedbbab9e3af51526c4dd445a9e3dbade95d24522da8bbad0a50ab400755e47b018828b324c4ad7705e212ccd990e34439a +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran4.tar.gz/md5/bc4a0f0b7cea328f7e8850583774496b +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran4.tar.gz/sha512/82285b67946212b49cddf6259f2c60ff5469f8c5263ccefe44f1d93ace98ab68e2c152e1b54434b2f075fd8d192c06d5451bc8cca26d951ad15f3453102f02b5 +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran5.tar.gz/md5/177f0232abce8d523882530ed7a93092 +CompilerSupportLibraries.v1.0.2+0.i686-linux-musl-libgfortran5.tar.gz/sha512/db80acf0f2434f28ee7680e1beb34f564940071815d1ad89fb5913cbd9ac24da528e826d0d54be6265a7340ebd661b6d308ed79d96b67fa5d8c98dc3f1bee8d6 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran3.tar.gz/md5/756718e5eaa4547b874a71a8e3545492 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran3.tar.gz/sha512/c21c1be10ca8810f56e435b3629e2ab0678926ea9c4f4c3dd003f9e292c075493b83df04401d3bcf7738f1a44098f674f9b01bba9db4b9a9e45ad7af3497444e +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran4.tar.gz/md5/65ce0024bf8fe3276addbf185ed03e48 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran4.tar.gz/sha512/5e8105a12ab04e2949e41eda50a060dea04ccd98660c7528cfc86e120fe61cca8bab878fd2c92a3858f02ac3f3c55d0e48789907e5fbd2392a8e84b183ed4636 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran5.tar.gz/md5/b7727324d550f637209db795238c46a4 +CompilerSupportLibraries.v1.0.2+0.i686-w64-mingw32-libgfortran5.tar.gz/sha512/864b1db2642e68665b9d3322563c7ce964835d0e720325ea00b193e2cbf6791760e0014710e2a79876165ab0daffa6d53d61b87a5034f956ba6e255b0144652c +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/md5/4e5e4b23dc87450738da33926a07511d +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran3.tar.gz/sha512/fc09879d94b750e75775d8b64a41ab9924d675fb53c5700467604412928fe7f5cb21911da0f64898d2463fa77ffbaf4c96c397b9060f4746eec152747930cddc +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/md5/9a92138ed69aa317a932a615c6e62d69 +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran4.tar.gz/sha512/0b7785379936a2a209b074177b1424dd7e00b29b5165f564e799b0aa4e06a582e9d616525d97274ba2507cb88192028f1ac485d3f99bdc7ee53fc63c1a7e85de +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/md5/8ffee3d6de5197c7a1f354d72c8238fa +CompilerSupportLibraries.v1.0.2+0.powerpc64le-linux-gnu-libgfortran5.tar.gz/sha512/deadc4d7224c84f9b82dc956b69e815c44ae036802838365d870ab9f58c8bcf8ce0645f2f387c8ff344ac2108fc8e7e1ee907fa55e93c91aa5d9fd921bf3fdcb +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran3.tar.gz/md5/87449e72e3f33dbb69b7053cdc2649d4 +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran3.tar.gz/sha512/5ce02ad10c6f4686a476eb2a5de2988cd8b482f5e693db2880c84ad1c82f468ef03fe01b9d0feefe5d4ee741d1d16643d36b144e6261ed32311b3b6f312fac2f +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran4.tar.gz/md5/0407cde92cfa42fa89ac83217ca0ec16 +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran4.tar.gz/sha512/032c831f1166a336551138939ac40eb2c68a048ce786c0c1403b879a20c1b706caac16d22560b2c7f2b3d6373986c347188675674116005ca251336ee048d09f +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran5.tar.gz/md5/23418763b808371ee94772a90d501f4d +CompilerSupportLibraries.v1.0.2+0.x86_64-apple-darwin-libgfortran5.tar.gz/sha512/7867b843551457b11bda7821dd384c1c1cf23b80a308b2058a693de7b7da099f0b37eb0a6de2b84c04b625a68c60eea55138e200d5d6ec6f6af09bd7ce406a96 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran3.tar.gz/md5/e3d33ae03c18affea74699bdc1fabb68 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran3.tar.gz/sha512/42013f4921de5a69ad857195ce5c19ad1bca3c920d79699e5501f1f4534ab132fabd422362b2b5056f5d182215d6c069db5df460bafa700903faf962cc00f77b +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran4.tar.gz/md5/d40c1e8c0393213c6057c53a12f44175 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran4.tar.gz/sha512/fe7baa4de7490065ab7b953cc12f41462a24bcb49d0a4a64b23249e98e7569b19bb1cb455af2f76090e34066a7d3cdd7a48cae6515ce6c7a5c8486b0cacc5106 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran5.tar.gz/md5/48541b90f715c4c86ee4da0570275947 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-gnu-libgfortran5.tar.gz/sha512/7f2683fb98e80f12629f4ed3bea9fd59d32b7e7a9ed1699e782d8e238ff0915ecc61bf00adaf4597cfe41caf82cdca0f9be250f595f5f0bea6d8f77dba99eaf4 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran3.tar.gz/md5/4547059eb905995667be48bf85d49911 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran3.tar.gz/sha512/7400fdabc924434ab4a4949248c3603887ac06ffd2f205ae33e14495d86cd4f816bbd1999eeafa0257f518df1e7f7c522f596e847a71dbfbfccff4859f50acc7 +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran4.tar.gz/md5/46267543cad6584d7b7b9fcc8f18f21d +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran4.tar.gz/sha512/0353d7d724be48d4185d3c181692970b7996f53f6a01723072aa5c94b53a8c5055faeed30df51659c252a46f4b941dec0cb24569323e3c85c166f14c5b7c8e9e +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran5.tar.gz/md5/14dba2897a6e9d370fa9091c045375fc +CompilerSupportLibraries.v1.0.2+0.x86_64-linux-musl-libgfortran5.tar.gz/sha512/10b79f9c059839f5b57fa8d2a381a034c4067262c4088bd354d14ea56bec097878069383aa9cfadaa09d73bd20fc348fb61662d863a8d62cb25d7af6b8e29858 +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/md5/eed836d1addeb10d0901f836724aff1e +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran3.tar.gz/sha512/e33eca424d1529a1fb23ba9cf7fac345ed1cfc8073c975b6b31ca44d2e8c3f5083af65433df009b22483dceb2e43149f3c1e8433681fec5fb812e1d5b4243ce4 +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/md5/d5ae9f9519341fdaabf62267c89461d2 +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran4.tar.gz/sha512/6421aa5d1bd6f08ad43f59ed4dc1bef8b9b598ebbbd3e48149730f3bec3471f8e2c02ffb338427326924290b8f52ef9e626e3313448bc931a61d866c5dc544ae +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/md5/fc1df521395362a5aaa2e2aeef707207 +CompilerSupportLibraries.v1.0.2+0.x86_64-unknown-freebsd-libgfortran5.tar.gz/sha512/f2e5a08e3cae171242ae6a20d2d4838c1529ce042745dc466148b7bbc06896d94476fd05c7787e6e8641bea752dfc0e6b09e95b160bede600d20d2ad68e7705f +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran3.tar.gz/md5/2338f8aa2696935f7460454e708ce308 +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran3.tar.gz/sha512/5a4b0e97928c26eee16bbec4c3e69e55fa9c768101257c3e2f161118809c778aa0feaf21307198822c3172a58ed12ca0a49285b2941ed0b8f2b367e64ca1c51a +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran4.tar.gz/md5/b393d2bf0d181d218130ac572c17d369 +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran4.tar.gz/sha512/76e0f7caa24bb734c6f7542be9f834d5b912f082cb3c4c3c52a63e37d4b8c33dd94e576c43f4bee6c04bfb44af2f2b67ba70773fa52ad0de6c8c0059b3e51b83 +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran5.tar.gz/md5/23db836e6e4142f621862971017fe61e +CompilerSupportLibraries.v1.0.2+0.x86_64-w64-mingw32-libgfortran5.tar.gz/sha512/c0b04f7fe5aabfe6af509c77a1f68e0bcfd14714758042fe502b968c4cc272156fc84c8b4c1ee574754bb2fddaa810f6a4215cbd164ddc11b697b3adaef09a81 diff --git a/stdlib/CompilerSupportLibraries_jll/Project.toml b/stdlib/CompilerSupportLibraries_jll/Project.toml index b072831326627..fc5883cc79802 100644 --- a/stdlib/CompilerSupportLibraries_jll/Project.toml +++ b/stdlib/CompilerSupportLibraries_jll/Project.toml @@ -4,7 +4,7 @@ uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" # NOTE: When updating this, also make sure to update the value # `CSL_NEXT_GLIBCXX_VERSION` in `deps/csl.mk`, to properly disable # automatic usage of BB-built CSLs on extremely up-to-date systems! -version = "1.0.1+0" +version = "1.0.2+0" [deps] Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb" From 17ff877a14e844dc1fbcea2d158e1500dd13ab79 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Thu, 22 Dec 2022 16:23:46 -0600 Subject: [PATCH 20/25] Find native code in non-worklist modules Formerly we only looked for native code in `worklist` modules. This adds the new "external method extensions" (new methods owned by the `worklist` but for functions owned by non-`worklist` modules) and new specialization of external methods. --- src/julia.h | 2 +- src/precompile.c | 302 +--------------------------------------- src/precompile_utils.c | 306 +++++++++++++++++++++++++++++++++++++++++ src/staticdata.c | 14 +- 4 files changed, 320 insertions(+), 304 deletions(-) create mode 100644 src/precompile_utils.c diff --git a/src/julia.h b/src/julia.h index 3983f283b72e5..0344e1defeeda 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1762,7 +1762,7 @@ JL_DLLEXPORT jl_gcframe_t **jl_adopt_thread(void); JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s); JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname); JL_DLLEXPORT void jl_set_sysimg_so(void *handle); -JL_DLLEXPORT void jl_create_system_image(void *, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos); +JL_DLLEXPORT void jl_create_system_image(void **, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos); JL_DLLEXPORT void jl_restore_system_image(const char *fname); JL_DLLEXPORT void jl_restore_system_image_data(const char *buf, size_t len); JL_DLLEXPORT jl_value_t *jl_restore_incremental(const char *fname, jl_array_t *depmods, int complete); diff --git a/src/precompile.c b/src/precompile.c index a70f7cd882bc0..bfc123cf3fda8 100644 --- a/src/precompile.c +++ b/src/precompile.c @@ -21,9 +21,6 @@ JL_DLLEXPORT int jl_generating_output(void) return jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputji || jl_options.outputasm; } -static void *jl_precompile(int all); -static void *jl_precompile_worklist(jl_array_t *worklist); - void write_srctext(ios_t *f, jl_array_t *udeps, int64_t srctextpos) { // Write the source-text for the dependent files if (udeps) { @@ -109,13 +106,6 @@ JL_DLLEXPORT void jl_write_compiler_output(void) assert(jl_precompile_toplevel_module == NULL); void *native_code = NULL; - if (jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm) { - if (jl_options.incremental) - jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); - native_code = jl_options.incremental ? jl_precompile_worklist(worklist) : jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); - if (jl_options.incremental) - jl_precompile_toplevel_module = NULL; - } bool_t emit_native = jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm; @@ -124,7 +114,7 @@ JL_DLLEXPORT void jl_write_compiler_output(void) ios_t *s = NULL; ios_t *z = NULL; int64_t srctextpos = 0 ; - jl_create_system_image(native_code, jl_options.incremental ? worklist : NULL, emit_split, + jl_create_system_image(&native_code, jl_options.incremental ? worklist : NULL, emit_split, &s, &z, &udeps, &srctextpos); if (!emit_split) @@ -174,296 +164,6 @@ JL_DLLEXPORT void jl_write_compiler_output(void) JL_GC_POP(); } -// f{<:Union{...}}(...) is a common pattern -// and expanding the Union may give a leaf function -static void _compile_all_tvar_union(jl_value_t *methsig) -{ - int tvarslen = jl_subtype_env_size(methsig); - jl_value_t *sigbody = methsig; - jl_value_t **roots; - JL_GC_PUSHARGS(roots, 1 + 2 * tvarslen); - jl_value_t **env = roots + 1; - int *idx = (int*)alloca(sizeof(int) * tvarslen); - int i; - for (i = 0; i < tvarslen; i++) { - assert(jl_is_unionall(sigbody)); - idx[i] = 0; - env[2 * i] = (jl_value_t*)((jl_unionall_t*)sigbody)->var; - env[2 * i + 1] = jl_bottom_type; // initialize the list with Union{}, since T<:Union{} is always a valid option - sigbody = ((jl_unionall_t*)sigbody)->body; - } - - for (i = 0; i < tvarslen; /* incremented by inner loop */) { - jl_value_t **sig = &roots[0]; - JL_TRY { - // TODO: wrap in UnionAll for each tvar in env[2*i + 1] ? - // currently doesn't matter much, since jl_compile_hint doesn't work on abstract types - *sig = (jl_value_t*)jl_instantiate_type_with(sigbody, env, tvarslen); - } - JL_CATCH { - goto getnext; // sigh, we found an invalid type signature. should we warn the user? - } - if (!jl_has_concrete_subtype(*sig)) - goto getnext; // signature wouldn't be callable / is invalid -- skip it - if (jl_is_concrete_type(*sig)) { - if (jl_compile_hint((jl_tupletype_t *)*sig)) - goto getnext; // success - } - - getnext: - for (i = 0; i < tvarslen; i++) { - jl_tvar_t *tv = (jl_tvar_t*)env[2 * i]; - if (jl_is_uniontype(tv->ub)) { - size_t l = jl_count_union_components(tv->ub); - size_t j = idx[i]; - if (j == l) { - env[2 * i + 1] = jl_bottom_type; - idx[i] = 0; - } - else { - jl_value_t *ty = jl_nth_union_component(tv->ub, j); - if (!jl_is_concrete_type(ty)) - ty = (jl_value_t*)jl_new_typevar(tv->name, tv->lb, ty); - env[2 * i + 1] = ty; - idx[i] = j + 1; - break; - } - } - else { - env[2 * i + 1] = (jl_value_t*)tv; - } - } - } - JL_GC_POP(); -} - -// f(::Union{...}, ...) is a common pattern -// and expanding the Union may give a leaf function -static void _compile_all_union(jl_value_t *sig) -{ - jl_tupletype_t *sigbody = (jl_tupletype_t*)jl_unwrap_unionall(sig); - size_t count_unions = 0; - size_t i, l = jl_svec_len(sigbody->parameters); - jl_svec_t *p = NULL; - jl_value_t *methsig = NULL; - - for (i = 0; i < l; i++) { - jl_value_t *ty = jl_svecref(sigbody->parameters, i); - if (jl_is_uniontype(ty)) - ++count_unions; - else if (ty == jl_bottom_type) - return; // why does this method exist? - else if (jl_is_datatype(ty) && !jl_has_free_typevars(ty) && - ((!jl_is_kind(ty) && ((jl_datatype_t*)ty)->isconcretetype) || - ((jl_datatype_t*)ty)->name == jl_type_typename)) - return; // no amount of union splitting will make this a leaftype signature - } - - if (count_unions == 0 || count_unions >= 6) { - _compile_all_tvar_union(sig); - return; - } - - int *idx = (int*)alloca(sizeof(int) * count_unions); - for (i = 0; i < count_unions; i++) { - idx[i] = 0; - } - - JL_GC_PUSH2(&p, &methsig); - int idx_ctr = 0, incr = 0; - while (!incr) { - p = jl_alloc_svec_uninit(l); - for (i = 0, idx_ctr = 0, incr = 1; i < l; i++) { - jl_value_t *ty = jl_svecref(sigbody->parameters, i); - if (jl_is_uniontype(ty)) { - assert(idx_ctr < count_unions); - size_t l = jl_count_union_components(ty); - size_t j = idx[idx_ctr]; - jl_svecset(p, i, jl_nth_union_component(ty, j)); - ++j; - if (incr) { - if (j == l) { - idx[idx_ctr] = 0; - } - else { - idx[idx_ctr] = j; - incr = 0; - } - } - ++idx_ctr; - } - else { - jl_svecset(p, i, ty); - } - } - methsig = (jl_value_t*)jl_apply_tuple_type(p); - methsig = jl_rewrap_unionall(methsig, sig); - _compile_all_tvar_union(methsig); - } - - JL_GC_POP(); -} - -static int compile_all_collect__(jl_typemap_entry_t *ml, void *env) -{ - jl_array_t *allmeths = (jl_array_t*)env; - jl_method_t *m = ml->func.method; - if (m->source) { - // method has a non-generated definition; can be compiled generically - jl_array_ptr_1d_push(allmeths, (jl_value_t*)m); - } - return 1; -} - -static int compile_all_collect_(jl_methtable_t *mt, void *env) -{ - jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), compile_all_collect__, env); - return 1; -} - -static void jl_compile_all_defs(jl_array_t *mis) -{ - jl_array_t *allmeths = jl_alloc_vec_any(0); - JL_GC_PUSH1(&allmeths); - - jl_foreach_reachable_mtable(compile_all_collect_, allmeths); - - size_t i, l = jl_array_len(allmeths); - for (i = 0; i < l; i++) { - jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(allmeths, i); - if (jl_is_datatype(m->sig) && jl_isa_compileable_sig((jl_tupletype_t*)m->sig, jl_emptysvec, m)) { - // method has a single compilable specialization, e.g. its definition - // signature is concrete. in this case we can just hint it. - jl_compile_hint((jl_tupletype_t*)m->sig); - } - else { - // first try to create leaf signatures from the signature declaration and compile those - _compile_all_union(m->sig); - - // finally, compile a fully generic fallback that can work for all arguments - jl_method_instance_t *unspec = jl_get_unspecialized(m); - if (unspec) - jl_array_ptr_1d_push(mis, (jl_value_t*)unspec); - } - } - - JL_GC_POP(); -} - -static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closure) -{ - assert(jl_is_method_instance(mi)); - jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); - while (codeinst) { - int do_compile = 0; - if (jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return) { - jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); - if (inferred && - inferred != jl_nothing && - jl_ir_flag_inferred((jl_array_t*)inferred) && - (jl_ir_inlining_cost((jl_array_t*)inferred) == UINT16_MAX)) { - do_compile = 1; - } - else if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL || jl_atomic_load_relaxed(&codeinst->precompile)) { - do_compile = 1; - } - } - if (do_compile) { - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); - return 1; - } - codeinst = jl_atomic_load_relaxed(&codeinst->next); - } - return 1; -} - -static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *closure) -{ - jl_method_t *m = def->func.method; - if ((m->name == jl_symbol("__init__") || m->ccallable) && jl_is_dispatch_tupletype(m->sig)) { - // ensure `__init__()` and @ccallables get strongly-hinted, specialized, and compiled - jl_method_instance_t *mi = jl_specializations_get_linfo(m, m->sig, jl_emptysvec); - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); - } - else { - jl_svec_t *specializations = jl_atomic_load_relaxed(&def->func.method->specializations); - size_t i, l = jl_svec_len(specializations); - for (i = 0; i < l; i++) { - jl_value_t *mi = jl_svecref(specializations, i); - if (mi != jl_nothing) - precompile_enq_specialization_((jl_method_instance_t*)mi, closure); - } - } - if (m->ccallable) - jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)m->ccallable); - return 1; -} - -static int precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) -{ - return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), precompile_enq_all_specializations__, env); -} - -static void *jl_precompile_(jl_array_t *m, int external_linkage) -{ - jl_array_t *m2 = NULL; - jl_method_instance_t *mi = NULL; - JL_GC_PUSH2(&m2, &mi); - m2 = jl_alloc_vec_any(0); - for (size_t i = 0; i < jl_array_len(m); i++) { - jl_value_t *item = jl_array_ptr_ref(m, i); - if (jl_is_method_instance(item)) { - mi = (jl_method_instance_t*)item; - size_t min_world = 0; - size_t max_world = ~(size_t)0; - if (mi != jl_atomic_load_relaxed(&mi->def.method->unspecialized) && !jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->sparam_vals, mi->def.method)) - mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), &min_world, &max_world, 0); - if (mi) - jl_array_ptr_1d_push(m2, (jl_value_t*)mi); - } - else { - assert(jl_is_simplevector(item)); - assert(jl_svec_len(item) == 2); - jl_array_ptr_1d_push(m2, item); - } - } - void *native_code = jl_create_native(m2, NULL, NULL, 0, 1, external_linkage); - JL_GC_POP(); - return native_code; -} - -static void *jl_precompile(int all) -{ - // array of MethodInstances and ccallable aliases to include in the output - jl_array_t *m = jl_alloc_vec_any(0); - JL_GC_PUSH1(&m); - if (all) - jl_compile_all_defs(m); - jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); - void *native_code = jl_precompile_(m, 0); - JL_GC_POP(); - return native_code; -} - -static void *jl_precompile_worklist(jl_array_t *worklist) -{ - if (!worklist) - return NULL; - // this "found" array will contain function - // type signatures that were inferred but haven't been compiled - jl_array_t *m = jl_alloc_vec_any(0); - JL_GC_PUSH1(&m); - size_t i, nw = jl_array_len(worklist); - for (i = 0; i < nw; i++) { - jl_module_t *mod = (jl_module_t*)jl_array_ptr_ref(worklist, i); - assert(jl_is_module(mod)); - foreach_mtable_in_module(mod, precompile_enq_all_specializations_, m); - } - void *native_code = jl_precompile_(m, 1); - JL_GC_POP(); - return native_code; -} - #ifdef __cplusplus } #endif diff --git a/src/precompile_utils.c b/src/precompile_utils.c new file mode 100644 index 0000000000000..f251d00f76cfd --- /dev/null +++ b/src/precompile_utils.c @@ -0,0 +1,306 @@ +// f{<:Union{...}}(...) is a common pattern +// and expanding the Union may give a leaf function +static void _compile_all_tvar_union(jl_value_t *methsig) +{ + int tvarslen = jl_subtype_env_size(methsig); + jl_value_t *sigbody = methsig; + jl_value_t **roots; + JL_GC_PUSHARGS(roots, 1 + 2 * tvarslen); + jl_value_t **env = roots + 1; + int *idx = (int*)alloca(sizeof(int) * tvarslen); + int i; + for (i = 0; i < tvarslen; i++) { + assert(jl_is_unionall(sigbody)); + idx[i] = 0; + env[2 * i] = (jl_value_t*)((jl_unionall_t*)sigbody)->var; + env[2 * i + 1] = jl_bottom_type; // initialize the list with Union{}, since T<:Union{} is always a valid option + sigbody = ((jl_unionall_t*)sigbody)->body; + } + + for (i = 0; i < tvarslen; /* incremented by inner loop */) { + jl_value_t **sig = &roots[0]; + JL_TRY { + // TODO: wrap in UnionAll for each tvar in env[2*i + 1] ? + // currently doesn't matter much, since jl_compile_hint doesn't work on abstract types + *sig = (jl_value_t*)jl_instantiate_type_with(sigbody, env, tvarslen); + } + JL_CATCH { + goto getnext; // sigh, we found an invalid type signature. should we warn the user? + } + if (!jl_has_concrete_subtype(*sig)) + goto getnext; // signature wouldn't be callable / is invalid -- skip it + if (jl_is_concrete_type(*sig)) { + if (jl_compile_hint((jl_tupletype_t *)*sig)) + goto getnext; // success + } + + getnext: + for (i = 0; i < tvarslen; i++) { + jl_tvar_t *tv = (jl_tvar_t*)env[2 * i]; + if (jl_is_uniontype(tv->ub)) { + size_t l = jl_count_union_components(tv->ub); + size_t j = idx[i]; + if (j == l) { + env[2 * i + 1] = jl_bottom_type; + idx[i] = 0; + } + else { + jl_value_t *ty = jl_nth_union_component(tv->ub, j); + if (!jl_is_concrete_type(ty)) + ty = (jl_value_t*)jl_new_typevar(tv->name, tv->lb, ty); + env[2 * i + 1] = ty; + idx[i] = j + 1; + break; + } + } + else { + env[2 * i + 1] = (jl_value_t*)tv; + } + } + } + JL_GC_POP(); +} + +// f(::Union{...}, ...) is a common pattern +// and expanding the Union may give a leaf function +static void _compile_all_union(jl_value_t *sig) +{ + jl_tupletype_t *sigbody = (jl_tupletype_t*)jl_unwrap_unionall(sig); + size_t count_unions = 0; + size_t i, l = jl_svec_len(sigbody->parameters); + jl_svec_t *p = NULL; + jl_value_t *methsig = NULL; + + for (i = 0; i < l; i++) { + jl_value_t *ty = jl_svecref(sigbody->parameters, i); + if (jl_is_uniontype(ty)) + ++count_unions; + else if (ty == jl_bottom_type) + return; // why does this method exist? + else if (jl_is_datatype(ty) && !jl_has_free_typevars(ty) && + ((!jl_is_kind(ty) && ((jl_datatype_t*)ty)->isconcretetype) || + ((jl_datatype_t*)ty)->name == jl_type_typename)) + return; // no amount of union splitting will make this a leaftype signature + } + + if (count_unions == 0 || count_unions >= 6) { + _compile_all_tvar_union(sig); + return; + } + + int *idx = (int*)alloca(sizeof(int) * count_unions); + for (i = 0; i < count_unions; i++) { + idx[i] = 0; + } + + JL_GC_PUSH2(&p, &methsig); + int idx_ctr = 0, incr = 0; + while (!incr) { + p = jl_alloc_svec_uninit(l); + for (i = 0, idx_ctr = 0, incr = 1; i < l; i++) { + jl_value_t *ty = jl_svecref(sigbody->parameters, i); + if (jl_is_uniontype(ty)) { + assert(idx_ctr < count_unions); + size_t l = jl_count_union_components(ty); + size_t j = idx[idx_ctr]; + jl_svecset(p, i, jl_nth_union_component(ty, j)); + ++j; + if (incr) { + if (j == l) { + idx[idx_ctr] = 0; + } + else { + idx[idx_ctr] = j; + incr = 0; + } + } + ++idx_ctr; + } + else { + jl_svecset(p, i, ty); + } + } + methsig = (jl_value_t*)jl_apply_tuple_type(p); + methsig = jl_rewrap_unionall(methsig, sig); + _compile_all_tvar_union(methsig); + } + + JL_GC_POP(); +} + +static int compile_all_collect__(jl_typemap_entry_t *ml, void *env) +{ + jl_array_t *allmeths = (jl_array_t*)env; + jl_method_t *m = ml->func.method; + if (m->source) { + // method has a non-generated definition; can be compiled generically + jl_array_ptr_1d_push(allmeths, (jl_value_t*)m); + } + return 1; +} + +static int compile_all_collect_(jl_methtable_t *mt, void *env) +{ + jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), compile_all_collect__, env); + return 1; +} + +static void jl_compile_all_defs(jl_array_t *mis) +{ + jl_array_t *allmeths = jl_alloc_vec_any(0); + JL_GC_PUSH1(&allmeths); + + jl_foreach_reachable_mtable(compile_all_collect_, allmeths); + + size_t i, l = jl_array_len(allmeths); + for (i = 0; i < l; i++) { + jl_method_t *m = (jl_method_t*)jl_array_ptr_ref(allmeths, i); + if (jl_is_datatype(m->sig) && jl_isa_compileable_sig((jl_tupletype_t*)m->sig, jl_emptysvec, m)) { + // method has a single compilable specialization, e.g. its definition + // signature is concrete. in this case we can just hint it. + jl_compile_hint((jl_tupletype_t*)m->sig); + } + else { + // first try to create leaf signatures from the signature declaration and compile those + _compile_all_union(m->sig); + + // finally, compile a fully generic fallback that can work for all arguments + jl_method_instance_t *unspec = jl_get_unspecialized(m); + if (unspec) + jl_array_ptr_1d_push(mis, (jl_value_t*)unspec); + } + } + + JL_GC_POP(); +} + +static int precompile_enq_specialization_(jl_method_instance_t *mi, void *closure) +{ + assert(jl_is_method_instance(mi)); + jl_code_instance_t *codeinst = jl_atomic_load_relaxed(&mi->cache); + while (codeinst) { + int do_compile = 0; + if (jl_atomic_load_relaxed(&codeinst->invoke) != jl_fptr_const_return) { + jl_value_t *inferred = jl_atomic_load_relaxed(&codeinst->inferred); + if (inferred && + inferred != jl_nothing && + jl_ir_flag_inferred((jl_array_t*)inferred) && + (jl_ir_inlining_cost((jl_array_t*)inferred) == UINT16_MAX)) { + do_compile = 1; + } + else if (jl_atomic_load_relaxed(&codeinst->invoke) != NULL || jl_atomic_load_relaxed(&codeinst->precompile)) { + do_compile = 1; + } + } + if (do_compile) { + jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); + return 1; + } + codeinst = jl_atomic_load_relaxed(&codeinst->next); + } + return 1; +} + +static int precompile_enq_all_specializations__(jl_typemap_entry_t *def, void *closure) +{ + jl_method_t *m = def->func.method; + if ((m->name == jl_symbol("__init__") || m->ccallable) && jl_is_dispatch_tupletype(m->sig)) { + // ensure `__init__()` and @ccallables get strongly-hinted, specialized, and compiled + jl_method_instance_t *mi = jl_specializations_get_linfo(m, m->sig, jl_emptysvec); + jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)mi); + } + else { + jl_svec_t *specializations = jl_atomic_load_relaxed(&def->func.method->specializations); + size_t i, l = jl_svec_len(specializations); + for (i = 0; i < l; i++) { + jl_value_t *mi = jl_svecref(specializations, i); + if (mi != jl_nothing) + precompile_enq_specialization_((jl_method_instance_t*)mi, closure); + } + } + if (m->ccallable) + jl_array_ptr_1d_push((jl_array_t*)closure, (jl_value_t*)m->ccallable); + return 1; +} + +static int precompile_enq_all_specializations_(jl_methtable_t *mt, void *env) +{ + return jl_typemap_visitor(jl_atomic_load_relaxed(&mt->defs), precompile_enq_all_specializations__, env); +} + +static void *jl_precompile_(jl_array_t *m, int external_linkage) +{ + jl_array_t *m2 = NULL; + jl_method_instance_t *mi = NULL; + JL_GC_PUSH2(&m2, &mi); + m2 = jl_alloc_vec_any(0); + for (size_t i = 0; i < jl_array_len(m); i++) { + jl_value_t *item = jl_array_ptr_ref(m, i); + if (jl_is_method_instance(item)) { + mi = (jl_method_instance_t*)item; + size_t min_world = 0; + size_t max_world = ~(size_t)0; + if (mi != jl_atomic_load_relaxed(&mi->def.method->unspecialized) && !jl_isa_compileable_sig((jl_tupletype_t*)mi->specTypes, mi->sparam_vals, mi->def.method)) + mi = jl_get_specialization1((jl_tupletype_t*)mi->specTypes, jl_atomic_load_acquire(&jl_world_counter), &min_world, &max_world, 0); + if (mi) + jl_array_ptr_1d_push(m2, (jl_value_t*)mi); + } + else { + assert(jl_is_simplevector(item)); + assert(jl_svec_len(item) == 2); + jl_array_ptr_1d_push(m2, item); + } + } + void *native_code = jl_create_native(m2, NULL, NULL, 0, 1, external_linkage); + JL_GC_POP(); + return native_code; +} + +static void *jl_precompile(int all) +{ + // array of MethodInstances and ccallable aliases to include in the output + jl_array_t *m = jl_alloc_vec_any(0); + JL_GC_PUSH1(&m); + if (all) + jl_compile_all_defs(m); + jl_foreach_reachable_mtable(precompile_enq_all_specializations_, m); + void *native_code = jl_precompile_(m, 0); + JL_GC_POP(); + return native_code; +} + +static void *jl_precompile_worklist(jl_array_t *worklist, jl_array_t *extext_methods, jl_array_t *new_specializations) +{ + if (!worklist) + return NULL; + // this "found" array will contain function + // type signatures that were inferred but haven't been compiled + jl_array_t *m = jl_alloc_vec_any(0); + JL_GC_PUSH1(&m); + size_t i, n = jl_array_len(worklist); + for (i = 0; i < n; i++) { + jl_module_t *mod = (jl_module_t*)jl_array_ptr_ref(worklist, i); + assert(jl_is_module(mod)); + foreach_mtable_in_module(mod, precompile_enq_all_specializations_, m); + } + n = jl_array_len(extext_methods); + for (i = 0; i < n; i++) { + jl_method_t *method = (jl_method_t*)jl_array_ptr_ref(extext_methods, i); + assert(jl_is_method(method)); + jl_svec_t *specializations = jl_atomic_load_relaxed(&method->specializations); + size_t j, l = jl_svec_len(specializations); + for (j = 0; j < l; j++) { + jl_value_t *mi = jl_svecref(specializations, j); + if (mi != jl_nothing) + precompile_enq_specialization_((jl_method_instance_t*)mi, m); + } + } + n = jl_array_len(new_specializations); + for (i = 0; i < n; i++) { + jl_code_instance_t *ci = (jl_code_instance_t*)jl_array_ptr_ref(new_specializations, i); + precompile_enq_specialization_(ci->def, m); + } + void *native_code = jl_precompile_(m, 1); + JL_GC_POP(); + return native_code; +} diff --git a/src/staticdata.c b/src/staticdata.c index 6d306f927dce5..eea59a8cffceb 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -90,6 +90,7 @@ External links: #include "julia_assert.h" #include "staticdata_utils.c" +#include "precompile_utils.c" #ifdef __cplusplus extern "C" { @@ -2577,7 +2578,7 @@ static void jl_write_header_for_incremental(ios_t *f, jl_array_t *worklist, jl_a write_mod_list(f, *mod_array); } -JL_DLLEXPORT void jl_create_system_image(void *_native_data, jl_array_t *worklist, bool_t emit_split, +JL_DLLEXPORT void jl_create_system_image(void **_native_data, jl_array_t *worklist, bool_t emit_split, ios_t **s, ios_t **z, jl_array_t **udeps, int64_t *srctextpos) { jl_gc_collect(JL_GC_FULL); @@ -2617,6 +2618,13 @@ JL_DLLEXPORT void jl_create_system_image(void *_native_data, jl_array_t *worklis jl_gc_enable_finalizers(ct, 0); // make sure we don't run any Julia code concurrently after this point jl_prepare_serialization_data(mod_array, newly_inferred, jl_worklist_key(worklist), &extext_methods, &new_specializations, &method_roots_list, &ext_targets, &edges); + // Generate _native_data` + if (jl_options.outputo || jl_options.outputbc || jl_options.outputunoptbc || jl_options.outputasm) { + jl_precompile_toplevel_module = (jl_module_t*)jl_array_ptr_ref(worklist, jl_array_len(worklist)-1); + *_native_data = jl_precompile_worklist(worklist, extext_methods, new_specializations); + jl_precompile_toplevel_module = NULL; + } + if (!emit_split) { write_int32(f, 0); // No clone_targets write_padding(f, LLT_ALIGN(ios_pos(f), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(f)); @@ -2624,8 +2632,10 @@ JL_DLLEXPORT void jl_create_system_image(void *_native_data, jl_array_t *worklis write_padding(ff, LLT_ALIGN(ios_pos(ff), JL_CACHE_BYTE_ALIGNMENT) - ios_pos(ff)); } datastartpos = ios_pos(ff); + } else { + *_native_data = jl_precompile(jl_options.compile_enabled == JL_OPTIONS_COMPILE_ALL); } - native_functions = _native_data; + native_functions = *_native_data; jl_save_system_image_to_stream(ff, worklist, extext_methods, new_specializations, method_roots_list, ext_targets, edges); native_functions = NULL; if (worklist) { From 6e9a67068da82f66df0e0607c13529758f9bc171 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Sat, 24 Dec 2022 15:50:49 +0100 Subject: [PATCH 21/25] bump Pkg to a version that does not precompile on add --- .../Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 | 1 - .../Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 | 1 - .../Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 | 1 + .../Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 | 1 + stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 create mode 100644 deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 create mode 100644 deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 diff --git a/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 b/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 deleted file mode 100644 index a45343b02aee0..0000000000000 --- a/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -6b0dfd893c25254e303607de4d9bda1f diff --git a/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 b/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 deleted file mode 100644 index bc7943a1aac62..0000000000000 --- a/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -8fcc8f8a8166ca6941ac1be72ec71bce935860cff3fc93609af68ac54b139743d3e3d7fd67ab538fa8ca1ce3279bb451c7c3df77f6fce8d65b11134ccd2581d4 diff --git a/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 b/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 new file mode 100644 index 0000000000000..f1ad4017b3b98 --- /dev/null +++ b/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 @@ -0,0 +1 @@ +f0b82d6954a5ae02783ffa5f3f29d785 diff --git a/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 b/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 new file mode 100644 index 0000000000000..39220c6f97d82 --- /dev/null +++ b/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 @@ -0,0 +1 @@ +173fd9f30d8c6acc231f3fe1680b52fdb2500eb4247f3d2c5514b0e69f09e7214c67bfcf1456650e27010a852f15d2cb731590534a731733605524d5a0e087a1 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index f5db6741ee3c2..d31de68cce414 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = a8ae3c58078f8815718d3edc3842edf3a36adefa +PKG_SHA1 = f4cb866910dbbad6a12c3b0bc3dbec4031590dc1 PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 9f2f0d1c6bfe5b1bf7cbcbc217000cad0879c928 Mon Sep 17 00:00:00 2001 From: Kristoffer Date: Sun, 25 Dec 2022 12:38:25 +0100 Subject: [PATCH 22/25] Revert "bump Pkg to a version that does not precompile on add" This reverts commit 6e9a67068da82f66df0e0607c13529758f9bc171. --- .../Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 | 1 + .../Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 | 1 + .../Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 | 1 - .../Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 | 1 - stdlib/Pkg.version | 2 +- 5 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 create mode 100644 deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 delete mode 100644 deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 delete mode 100644 deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 diff --git a/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 b/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 new file mode 100644 index 0000000000000..a45343b02aee0 --- /dev/null +++ b/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/md5 @@ -0,0 +1 @@ +6b0dfd893c25254e303607de4d9bda1f diff --git a/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 b/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 new file mode 100644 index 0000000000000..bc7943a1aac62 --- /dev/null +++ b/deps/checksums/Pkg-a8ae3c58078f8815718d3edc3842edf3a36adefa.tar.gz/sha512 @@ -0,0 +1 @@ +8fcc8f8a8166ca6941ac1be72ec71bce935860cff3fc93609af68ac54b139743d3e3d7fd67ab538fa8ca1ce3279bb451c7c3df77f6fce8d65b11134ccd2581d4 diff --git a/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 b/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 deleted file mode 100644 index f1ad4017b3b98..0000000000000 --- a/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/md5 +++ /dev/null @@ -1 +0,0 @@ -f0b82d6954a5ae02783ffa5f3f29d785 diff --git a/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 b/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 deleted file mode 100644 index 39220c6f97d82..0000000000000 --- a/deps/checksums/Pkg-f4cb866910dbbad6a12c3b0bc3dbec4031590dc1.tar.gz/sha512 +++ /dev/null @@ -1 +0,0 @@ -173fd9f30d8c6acc231f3fe1680b52fdb2500eb4247f3d2c5514b0e69f09e7214c67bfcf1456650e27010a852f15d2cb731590534a731733605524d5a0e087a1 diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index d31de68cce414..f5db6741ee3c2 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,4 +1,4 @@ PKG_BRANCH = master -PKG_SHA1 = f4cb866910dbbad6a12c3b0bc3dbec4031590dc1 +PKG_SHA1 = a8ae3c58078f8815718d3edc3842edf3a36adefa PKG_GIT_URL := https://github.com/JuliaLang/Pkg.jl.git PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1 From 036552660dca8c3171d9ccb53bb9941cd9329699 Mon Sep 17 00:00:00 2001 From: Tim Holy Date: Mon, 26 Dec 2022 08:43:43 -0600 Subject: [PATCH 23/25] Add MWE for DynamicExpressions failure --- test/precompile.jl | 74 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/test/precompile.jl b/test/precompile.jl index e2db9087fe804..5eea04a46708b 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -516,14 +516,15 @@ precompile_test_harness(false) do dir empty_prefs_hash = Base.get_preferences_hash(nothing, String[]) @test cachefile == Base.compilecache_path(Base.PkgId("FooBar"), empty_prefs_hash) @test isfile(joinpath(cachedir, "FooBar.ji")) - @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tuple{<:Vector, String} + Tsc = Bool(Base.JLOptions().use_pkgimages) ? Tuple{<:Vector, String} : Tuple{<:Vector, Nothing} + @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc @test !isdefined(Main, :FooBar) @test !isdefined(Main, :FooBar1) relFooBar_file = joinpath(dir, "subfolder", "..", "FooBar.jl") @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa (Sys.iswindows() ? Tuple{<:Vector, String} : Bool) # `..` is not a symlink on Windows mkdir(joinpath(dir, "subfolder")) - @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tuple{<:Vector, String} + @test Base.stale_cachefile(relFooBar_file, joinpath(cachedir, "FooBar.ji")) isa Tsc @eval using FooBar fb_uuid = Base.module_build_id(FooBar) @@ -535,7 +536,7 @@ precompile_test_harness(false) do dir @test !isfile(joinpath(cachedir, "FooBar1.ji")) @test isfile(joinpath(cachedir2, "FooBar1.ji")) @test Base.stale_cachefile(FooBar_file, joinpath(cachedir, "FooBar.ji")) === true - @test Base.stale_cachefile(FooBar1_file, joinpath(cachedir2, "FooBar1.ji")) isa Tuple{<:Vector, String} + @test Base.stale_cachefile(FooBar1_file, joinpath(cachedir2, "FooBar1.ji")) isa Tsc @test fb_uuid == Base.module_build_id(FooBar) fb_uuid1 = Base.module_build_id(FooBar1) @test fb_uuid != fb_uuid1 @@ -1671,6 +1672,73 @@ precompile_test_harness("PkgCacheInspector") do load_path end end +precompile_test_harness("DynamicExpressions") do load_path + # https://github.com/JuliaLang/julia/pull/47184#issuecomment-1364716312 + write(joinpath(load_path, "DynamicExpressions.jl"), + """ + module DynamicExpressions + export Node + + mutable struct Node{T} + degree::Int # 0 for constant/variable, 1 for cos/sin, 2 for +/* etc. + val::Union{T,Nothing} # If is a constant, this stores the actual value + # ------------------- (possibly undefined below) + l::Node{T} # Left child node. Only defined for degree=1 or degree=2. + r::Node{T} # Right child node. Only defined for degree=2. + + ################# + ## Constructors: + ################# + Node{T}(d, v) where T = new{T}(d, v) + Node{T}(d, v, l) where T = new{T}(d, v, l) + Node{T}(d, v, l, r) where T = new{T}(d, v, l, r) + end + + Node(d, v::T) where {T} = Node{T}(d, v) + Node(d, v, l::Node{T}) where T = Node{T}(d, v, l) + Node(d, v, l::Node{T}, r::Node{T}) where T = Node{T}(d, v, l, r) + + function Base.convert( + ::Type{Node{T1}}, + tree::Node{T2}, + id_map::IdDict{Node{T2},Node{T1}}=IdDict{Node{T2},Node{T1}}(), + ) where {T1,T2} + if T1 == T2 + return tree + end + get!(id_map, tree) do + if tree.degree == 0 + val = tree.val::T2 + if !(T2 <: T1) + # e.g., we don't want to convert Float32 to Union{Float32,Vector{Float32}}! + val = convert(T1, val) + end + Node{T1}(0, val) + elseif tree.degree == 1 + l = convert(Node{T1}, tree.l, id_map) + Node(1, nothing, l) + else + l = convert(Node{T1}, tree.l, id_map) + r = convert(Node{T1}, tree.r, id_map) + Node(2, nothing, l, r) + end + end + end + + let + Base.Experimental.@force_compile + convert(Node{Float16}, Node(0, -1.2)) + end + + end + """) + Base.compilecache(Base.PkgId("DynamicExpressions")) + (@eval (using DynamicExpressions)) + Base.invokelatest() do + @test convert(Node{Float16}, Node(0, -1.2)).val === Float16(-1.2) + end +end + empty!(Base.DEPOT_PATH) append!(Base.DEPOT_PATH, original_depot_path) empty!(Base.LOAD_PATH) From f33c2a64ffa95e3a153c55b164a3b2acb33069ff Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 26 Dec 2022 09:03:19 -0600 Subject: [PATCH 24/25] Simplify test --- test/precompile.jl | 68 +++++++--------------------------------------- 1 file changed, 10 insertions(+), 58 deletions(-) diff --git a/test/precompile.jl b/test/precompile.jl index 5eea04a46708b..8aa09efe3f417 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -1674,68 +1674,20 @@ end precompile_test_harness("DynamicExpressions") do load_path # https://github.com/JuliaLang/julia/pull/47184#issuecomment-1364716312 - write(joinpath(load_path, "DynamicExpressions.jl"), + write(joinpath(load_path, "Float16MWE.jl"), """ - module DynamicExpressions - export Node - - mutable struct Node{T} - degree::Int # 0 for constant/variable, 1 for cos/sin, 2 for +/* etc. - val::Union{T,Nothing} # If is a constant, this stores the actual value - # ------------------- (possibly undefined below) - l::Node{T} # Left child node. Only defined for degree=1 or degree=2. - r::Node{T} # Right child node. Only defined for degree=2. - - ################# - ## Constructors: - ################# - Node{T}(d, v) where T = new{T}(d, v) - Node{T}(d, v, l) where T = new{T}(d, v, l) - Node{T}(d, v, l, r) where T = new{T}(d, v, l, r) - end - - Node(d, v::T) where {T} = Node{T}(d, v) - Node(d, v, l::Node{T}) where T = Node{T}(d, v, l) - Node(d, v, l::Node{T}, r::Node{T}) where T = Node{T}(d, v, l, r) - - function Base.convert( - ::Type{Node{T1}}, - tree::Node{T2}, - id_map::IdDict{Node{T2},Node{T1}}=IdDict{Node{T2},Node{T1}}(), - ) where {T1,T2} - if T1 == T2 - return tree - end - get!(id_map, tree) do - if tree.degree == 0 - val = tree.val::T2 - if !(T2 <: T1) - # e.g., we don't want to convert Float32 to Union{Float32,Vector{Float32}}! - val = convert(T1, val) - end - Node{T1}(0, val) - elseif tree.degree == 1 - l = convert(Node{T1}, tree.l, id_map) - Node(1, nothing, l) - else - l = convert(Node{T1}, tree.l, id_map) - r = convert(Node{T1}, tree.r, id_map) - Node(2, nothing, l, r) - end - end - end - - let - Base.Experimental.@force_compile - convert(Node{Float16}, Node(0, -1.2)) - end - + module Float16MWE + struct Node{T} + val::T end + doconvert(::Type{<:Node}, val) = convert(Float16, val) + precompile(Tuple{typeof(doconvert), Type{Node{Float16}}, Float64}) + end # module Float16MWE """) - Base.compilecache(Base.PkgId("DynamicExpressions")) - (@eval (using DynamicExpressions)) + Base.compilecache(Base.PkgId("Float16MWE")) + (@eval (using Float16MWE)) Base.invokelatest() do - @test convert(Node{Float16}, Node(0, -1.2)).val === Float16(-1.2) + @test Float16MWE.doconvert(Float16MWE.Node{Float16}, -1.2) === Float16(-1.2) end end From 5fe7c05854256c69e522c2bdbacb28c8b6f97791 Mon Sep 17 00:00:00 2001 From: Valentin Churavy Date: Mon, 26 Dec 2022 10:15:28 -0500 Subject: [PATCH 25/25] Change interposers from weak to internal --- src/aotcompile.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aotcompile.cpp b/src/aotcompile.cpp index 85b923901ad6b..6f9345ee18f82 100644 --- a/src/aotcompile.cpp +++ b/src/aotcompile.cpp @@ -509,7 +509,7 @@ static void injectCRTAlias(Module &M, StringRef name, StringRef alias, FunctionT if (!target) { target = Function::Create(FT, Function::ExternalLinkage, alias, M); } - Function *interposer = Function::Create(FT, Function::WeakAnyLinkage, name, M); + Function *interposer = Function::Create(FT, Function::InternalLinkage, name, M); appendToCompilerUsed(M, {interposer}); llvm::IRBuilder<> builder(BasicBlock::Create(M.getContext(), "top", interposer));