From 4e7ec790b587f4fe26ce7620c83d3e3197b3c871 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 12 Apr 2017 17:41:57 -0400 Subject: [PATCH] make jl_init more friendly to embedded targets this throws the error messages directly when the problem occurs, rather than waiting until late to throw a generic "System image file not found" message --- base/options.jl | 12 ++--- base/replutil.jl | 15 ++++++ base/util.jl | 8 ++- src/dump.c | 64 +++++++++++------------- src/init.c | 117 +++++++++++++++++++++++++------------------- src/jitlayers.cpp | 10 ++++ src/julia.h | 1 + test/cmdlineargs.jl | 41 +++++++++++++++- test/misc.jl | 16 ++++-- 9 files changed, 185 insertions(+), 99 deletions(-) diff --git a/base/options.jl b/base/options.jl index ac8d7643680d5..af18585939ce6 100644 --- a/base/options.jl +++ b/base/options.jl @@ -40,15 +40,15 @@ end JLOptions() = unsafe_load(cglobal(:jl_options, JLOptions)) function show(io::IO, opt::JLOptions) - println(io, "JLOptions(") + print(io, "JLOptions(") fields = fieldnames(opt) nfields = length(fields) - for (i,f) in enumerate(fieldnames(opt)) - v = getfield(opt,f) + for (i, f) in enumerate(fields) + v = getfield(opt, i) if isa(v, Ptr{UInt8}) - v = v != C_NULL ? unsafe_string(v) : "" + v = (v != C_NULL) ? unsafe_string(v) : "" end - println(io, " ", f, " = ", repr(v), i < nfields ? "," : "") + print(io, f, " = ", repr(v), i < nfields ? ", " : "") end - print(io,")") + print(io, ")") end diff --git a/base/replutil.jl b/base/replutil.jl index 33e4f461d081d..e40e194a3762c 100644 --- a/base/replutil.jl +++ b/base/replutil.jl @@ -151,6 +151,21 @@ function show(io::IO, ::MIME"text/plain", s::String) end end +function show(io::IO, ::MIME"text/plain", opt::JLOptions) + println(io, "JLOptions(") + fields = fieldnames(opt) + nfields = length(fields) + for (i, f) in enumerate(fields) + v = getfield(opt, i) + if isa(v, Ptr{UInt8}) + v = (v != C_NULL) ? unsafe_string(v) : "" + end + println(io, " ", f, " = ", repr(v), i < nfields ? "," : "") + end + print(io, ")") +end + + # showing exception objects as descriptive error messages showerror(io::IO, ex) = show(io, ex) diff --git a/base/util.jl b/base/util.jl index 9a50ddf25db31..8e3e7010b3db8 100644 --- a/base/util.jl +++ b/base/util.jl @@ -619,7 +619,13 @@ function julia_cmd(julia=joinpath(JULIA_HOME, julia_exename())) `$julia -C$cpu_target -J$image_file --compile=$compile --depwarn=$depwarn` end -julia_exename() = ccall(:jl_is_debugbuild,Cint,())==0 ? "julia" : "julia-debug" +function julia_exename() + if ccall(:jl_is_debugbuild, Cint, ()) == 0 + return @static is_windows() ? "julia.exe" : "julia" + else + return @static is_windows() ? "julia-debug.exe" : "julia-debug" + end +end """ securezero!(o) diff --git a/src/dump.c b/src/dump.c index d482cd3cf5cdc..c404521b0822a 100644 --- a/src/dump.c +++ b/src/dump.c @@ -222,15 +222,11 @@ JL_DLLEXPORT int jl_running_on_valgrind(void) return RUNNING_ON_VALGRIND; } -static int jl_load_sysimg_so(void) +static void jl_load_sysimg_so(void) { #ifndef _OS_WINDOWS_ Dl_info dlinfo; #endif - // attempt to load the pre-compiled sysimage from jl_sysimg_handle - if (jl_sysimg_handle == 0) - return -1; - int imaging_mode = jl_generating_output() && !jl_options.incremental; // in --build mode only use sysimg data, not precompiled native code if (!imaging_mode && jl_options.use_precompiled==JL_OPTIONS_USE_PRECOMPILED_YES) { @@ -276,13 +272,9 @@ static int jl_load_sysimg_so(void) } #endif } - const char *sysimg_data = (const char*)jl_dlsym_e(jl_sysimg_handle, "jl_system_image_data"); - if (sysimg_data) { - size_t len = *(size_t*)jl_dlsym(jl_sysimg_handle, "jl_system_image_size"); - jl_restore_system_image_data(sysimg_data, len); - return 0; - } - return -1; + const char *sysimg_data = (const char*)jl_dlsym(jl_sysimg_handle, "jl_system_image_data"); + size_t len = *(size_t*)jl_dlsym(jl_sysimg_handle, "jl_system_image_size"); + jl_restore_system_image_data(sysimg_data, len); } static jl_value_t *jl_deserialize_gv(jl_serializer_state *s, jl_value_t *v) @@ -2552,30 +2544,31 @@ extern void jl_get_builtins(void); extern void jl_get_builtin_hooks(void); extern void jl_get_system_hooks(void); -// Takes in a path of the form "usr/lib/julia/sys.{ji,so}", as passed to jl_restore_system_image() +// Takes in a path of the form "usr/lib/julia/sys.so" (jl_restore_system_image should be passed the same string) JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname) { - // If passed NULL, don't even bother - if (!fname) - return; + if (jl_sysimg_handle) + return; // embedded target already called jl_set_sysimg_so - // First, get "sys" from "sys.ji" - char *fname_shlib = (char*)alloca(strlen(fname)+1); - strcpy(fname_shlib, fname); - char *fname_shlib_dot = strrchr(fname_shlib, '.'); - if (fname_shlib_dot != NULL) { - if (!strcmp(fname_shlib_dot, ".ji")) - return; // .ji extension => load .ji file only - *fname_shlib_dot = 0; - } + char *dot = (char*) strrchr(fname, '.'); + int is_ji = (dot && !strcmp(dot, ".ji")); // Get handle to sys.so - jl_sysimg_handle = jl_load_dynamic_library_e(fname_shlib, JL_RTLD_LOCAL | JL_RTLD_NOW); + if (!is_ji) // .ji extension => load .ji file only + jl_set_sysimg_so(jl_load_dynamic_library(fname, JL_RTLD_LOCAL | JL_RTLD_NOW)); +} +// Allow passing in a module handle directly, rather than a path +JL_DLLEXPORT void jl_set_sysimg_so(void *handle) +{ // set cpu target if unspecified by user and available from sysimg // otherwise default to native. - if (jl_sysimg_handle && jl_options.cpu_target == NULL) - jl_options.cpu_target = (const char *)jl_dlsym(jl_sysimg_handle, "jl_sysimg_cpu_target"); + void* *jl_RTLD_DEFAULT_handle_pointer = (void**)jl_dlsym_e(handle, "jl_RTLD_DEFAULT_handle_pointer"); + if (!jl_RTLD_DEFAULT_handle_pointer || (void*)&jl_RTLD_DEFAULT_handle != *jl_RTLD_DEFAULT_handle_pointer) + jl_error("System image file failed consistency check: maybe opened the wrong version?"); + if (jl_options.cpu_target == NULL) + jl_options.cpu_target = (const char *)jl_dlsym(handle, "jl_sysimg_cpu_target"); + jl_sysimg_handle = handle; } static void jl_restore_system_image_from_stream(ios_t *f) @@ -2653,16 +2646,15 @@ static void jl_restore_system_image_from_stream(ios_t *f) JL_DLLEXPORT void jl_restore_system_image(const char *fname) { - char *dot = (char*) strrchr(fname, '.'); +#ifndef NDEBUG + char *dot = fname ? (char*)strrchr(fname, '.') : NULL; int is_ji = (dot && !strcmp(dot, ".ji")); + assert((is_ji || jl_sysimg_handle) && "System image file not preloaded"); +#endif - if (!is_ji) { - int err = jl_load_sysimg_so(); - if (err != 0) { - if (jl_sysimg_handle == 0) - jl_errorf("System image file \"%s\" not found.", fname); - jl_errorf("Library \"%s\" does not contain a valid system image.", fname); - } + if (jl_sysimg_handle) { + // load the pre-compiled sysimage from jl_sysimg_handle + jl_load_sysimg_so(); } else { ios_t f; diff --git a/src/init.c b/src/init.c index 778be20efb38b..765db589cedd5 100644 --- a/src/init.c +++ b/src/init.c @@ -190,10 +190,52 @@ void jl_init_timing(void); void jl_destroy_timing(void); void jl_uv_call_close_callback(jl_value_t *val); +static void jl_close_item_atexit(uv_handle_t *handle) +{ + if (handle->type != UV_FILE && uv_is_closing(handle)) + return; + switch(handle->type) { + case UV_PROCESS: + // cause Julia to forget about the Process object + if (handle->data) + jl_uv_call_close_callback((jl_value_t*)handle->data); + // and make libuv think it is already dead + ((uv_process_t*)handle)->pid = 0; + // fall-through + case UV_TTY: + case UV_UDP: + case UV_TCP: + case UV_NAMED_PIPE: + case UV_POLL: + case UV_TIMER: + case UV_ASYNC: + case UV_FS_EVENT: + case UV_FS_POLL: + case UV_IDLE: + case UV_PREPARE: + case UV_CHECK: + case UV_SIGNAL: + case UV_FILE: + // These will be shutdown as appropriate by jl_close_uv + jl_close_uv(handle); + break; + case UV_HANDLE: + case UV_STREAM: + case UV_UNKNOWN_HANDLE: + case UV_HANDLE_TYPE_MAX: + case UV_RAW_FD: + case UV_RAW_HANDLE: + default: + assert(0); + } +} + JL_DLLEXPORT void jl_atexit_hook(int exitcode) { jl_ptls_t ptls = jl_get_ptls_states(); - if (exitcode == 0) jl_write_compiler_output(); + + if (exitcode == 0) + jl_write_compiler_output(); jl_print_gc_stats(JL_STDERR); if (jl_options.code_coverage) jl_write_coverage_data(); @@ -203,10 +245,10 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) jl_value_t *f = jl_get_global(jl_base_module, jl_symbol("_atexit")); if (f != NULL) { JL_TRY { - size_t last_age = jl_get_ptls_states()->world_age; - jl_get_ptls_states()->world_age = jl_get_world_counter(); + size_t last_age = ptls->world_age; + ptls->world_age = jl_get_world_counter(); jl_apply(&f, 1); - jl_get_ptls_states()->world_age = last_age; + ptls->world_age = last_age; } JL_CATCH { jl_printf(JL_STDERR, "\natexit hook threw an error: "); @@ -231,62 +273,33 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode) struct uv_shutdown_queue queue = {NULL, NULL}; uv_walk(loop, jl_uv_exitcleanup_walk, &queue); struct uv_shutdown_queue_item *item = queue.first; - while (item) { - JL_TRY { - while (item) { - uv_handle_t *handle = item->h; - if (handle->type != UV_FILE && uv_is_closing(handle)) { + if (ptls->current_task != NULL) { + while (item) { + JL_TRY { + while (item) { + jl_close_item_atexit(item->h); item = next_shutdown_queue_item(item); - continue; - } - switch(handle->type) { - case UV_PROCESS: - // cause Julia to forget about the Process object - if (handle->data) - jl_uv_call_close_callback((jl_value_t*)handle->data); - // and make libuv think it is already dead - ((uv_process_t*)handle)->pid = 0; - // fall-through - case UV_TTY: - case UV_UDP: - case UV_TCP: - case UV_NAMED_PIPE: - case UV_POLL: - case UV_TIMER: - case UV_ASYNC: - case UV_FS_EVENT: - case UV_FS_POLL: - case UV_IDLE: - case UV_PREPARE: - case UV_CHECK: - case UV_SIGNAL: - case UV_FILE: - // These will be shutdown as appropriate by jl_close_uv - jl_close_uv(handle); - break; - case UV_HANDLE: - case UV_STREAM: - case UV_UNKNOWN_HANDLE: - case UV_HANDLE_TYPE_MAX: - case UV_RAW_FD: - case UV_RAW_HANDLE: - default: - assert(0); } + } + JL_CATCH { + //error handling -- continue cleanup, as much as possible + uv_unref(item->h); + jl_printf(JL_STDERR, "error during exit cleanup: close: "); + jl_static_show(JL_STDERR, ptls->exception_in_transit); item = next_shutdown_queue_item(item); } } - JL_CATCH { - //error handling -- continue cleanup, as much as possible - uv_unref(item->h); - jl_printf(JL_STDERR, "error during exit cleanup: close: "); - jl_static_show(JL_STDERR, ptls->exception_in_transit); + } + else { + while (item) { + jl_close_item_atexit(item->h); item = next_shutdown_queue_item(item); } } + // force libuv to spin until everything has finished closing loop->stop_flag = 0; - while (uv_run(loop,UV_RUN_DEFAULT)) {} + while (uv_run(loop, UV_RUN_DEFAULT)) { } jl_destroy_timing(); #ifdef ENABLE_TIMINGS @@ -527,9 +540,11 @@ void _julia_init(JL_IMAGE_SEARCH rel) // best to call this first, since it also initializes libuv jl_init_signal_async(); restore_signals(); + jl_resolve_sysimg_location(rel); // loads sysimg if available, and conditionally sets jl_options.cpu_target - jl_preload_sysimg_so(jl_options.image_file); + if (jl_options.image_file) + jl_preload_sysimg_so(jl_options.image_file); if (jl_options.cpu_target == NULL) jl_options.cpu_target = "native"; diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp index 27eca4c7e3c14..ddb71b565c807 100644 --- a/src/jitlayers.cpp +++ b/src/jitlayers.cpp @@ -1121,6 +1121,16 @@ static void jl_gen_llvm_globaldata(llvm::Module *mod, ValueToValueMapTy &VMap, feature_string, "jl_sysimg_cpu_target")); + // reflect the address of the jl_RTLD_DEFAULT_handle variable + // back to the caller, so that we can check for consistency issues + GlobalValue *jlRTLD_DEFAULT_var = mod->getNamedValue("jl_RTLD_DEFAULT_handle"); + addComdat(new GlobalVariable(*mod, + jlRTLD_DEFAULT_var->getType(), + true, + GlobalVariable::ExternalLinkage, + jlRTLD_DEFAULT_var, + "jl_RTLD_DEFAULT_handle_pointer")); + #ifdef HAVE_CPUID // For native also store the cpuid if (strcmp(jl_options.cpu_target,"native") == 0) { diff --git a/src/julia.h b/src/julia.h index f9c520c0d4c59..1727d14f09040 100644 --- a/src/julia.h +++ b/src/julia.h @@ -1324,6 +1324,7 @@ JL_DLLEXPORT void JL_NORETURN jl_exit(int status); JL_DLLEXPORT int jl_deserialize_verify_header(ios_t *s); JL_DLLEXPORT void jl_preload_sysimg_so(const char *fname); +JL_DLLEXPORT void jl_set_sysimg_so(void *handle); JL_DLLEXPORT ios_t *jl_create_system_image(void); JL_DLLEXPORT void jl_save_system_image(const char *fname); JL_DLLEXPORT void jl_restore_system_image(const char *fname); diff --git a/test/cmdlineargs.jl b/test/cmdlineargs.jl index 170b3b8c63a8b..47dda4097da81 100644 --- a/test/cmdlineargs.jl +++ b/test/cmdlineargs.jl @@ -62,8 +62,8 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes --startup-file=no` @test !success(`$exename --load`) # --cpu-target - # NOTE: this test only holds true when there is a sys.{dll,dylib,so} shared library present. - if Libdl.dlopen_e(splitext(unsafe_string(Base.JLOptions().image_file))[1]) != C_NULL + # NOTE: this test only holds true if image_file is a shared library. + if Libdl.dlopen_e(unsafe_string(Base.JLOptions().image_file)) != C_NULL @test !success(`$exename -C invalidtarget --precompiled=yes`) @test !success(`$exename --cpu-target=invalidtarget --precompiled=yes`) else @@ -331,6 +331,43 @@ let exename = `$(Base.julia_cmd()) --precompiled=yes --startup-file=no` end end + +# Find the path of libjulia (or libjulia-debug, as the case may be) +# to use as a dummy shlib to open +libjulia = abspath(Libdl.dlpath((ccall(:jl_is_debugbuild, Cint, ()) != 0) ? "libjulia-debug" : "libjulia")) + +# test error handling code paths of running --sysimage +let exename = joinpath(JULIA_HOME, Base.julia_exename()), + sysname = unsafe_string(Base.JLOptions().image_file) + for nonexist_image in ( + joinpath(@__DIR__, "nonexistent"), + "$sysname.nonexistent", + ) + let stderr = Pipe(), + p = spawn(pipeline(`$exename --sysimage=$nonexist_image`, stderr=stderr)) + close(stderr.in) + let s = readstring(stderr) + @test contains(s, "ERROR: could not load library \"$nonexist_image\"\n") + @test !contains(s, "Segmentation fault") + @test !contains(s, "EXCEPTION_ACCESS_VIOLATION") + end + @test !success(p) + @test !Base.process_signaled(p) + @test p.exitcode == 1 + end + end + let stderr = Pipe(), + p = spawn(pipeline(`$exename --sysimage=$libjulia`, stderr=stderr)) + close(stderr.in) + let s = readstring(stderr) + @test s == "ERROR: System image file failed consistency check: maybe opened the wrong version?\n" + end + @test !success(p) + @test !Base.process_signaled(p) + @test p.exitcode == 1 + end +end + let exename = `$(Base.julia_cmd()) --precompiled=yes` # --startup-file let JL_OPTIONS_STARTUPFILE_ON = 1, diff --git a/test/misc.jl b/test/misc.jl index 27db61156ea91..a1e2b99096f77 100644 --- a/test/misc.jl +++ b/test/misc.jl @@ -514,9 +514,19 @@ if is_windows() end end -optstring = sprint(show, Base.JLOptions()) -@test startswith(optstring, "JLOptions(") -@test endswith(optstring, ")") +let optstring = stringmime(MIME("text/plain"), Base.JLOptions()) + @test startswith(optstring, "JLOptions(\n") + @test !contains(optstring, "Ptr") + @test endswith(optstring, "\n)") + @test contains(optstring, " = \"") +end +let optstring = repr(Base.JLOptions()) + @test startswith(optstring, "JLOptions(") + @test endswith(optstring, ")") + @test !contains(optstring, "\n") + @test !contains(optstring, "Ptr") + @test contains(optstring, " = \"") +end # Base.securezero! functions (#17579) import Base: securezero!, unsafe_securezero!