diff --git a/src/init.c b/src/init.c
index a5632fc66a45ad..9772b7e70fc3fa 100644
--- a/src/init.c
+++ b/src/init.c
@@ -223,7 +223,7 @@ JL_DLLEXPORT void jl_atexit_hook(int exitcode)
         jl_write_coverage_data(jl_options.output_code_coverage);
     if (jl_options.malloc_log)
         jl_write_malloc_log();
-    if (jl_base_module) {
+    if (jl_base_module && jl_error_sym) {
         jl_value_t *f = jl_get_global(jl_base_module, jl_symbol("_atexit"));
         if (f != NULL) {
             JL_TRY {
diff --git a/src/jitlayers.cpp b/src/jitlayers.cpp
index eda841cbadb73a..50316c258d3a04 100644
--- a/src/jitlayers.cpp
+++ b/src/jitlayers.cpp
@@ -814,13 +814,20 @@ namespace {
         SmallVector<std::string, 10> targetFeatures(target.second.begin(), target.second.end());
         std::string errorstr;
         const Target *TheTarget = TargetRegistry::lookupTarget("", TheTriple, errorstr);
-        if (!TheTarget)
-            jl_errorf("%s", errorstr.c_str());
+        if (!TheTarget) {
+            // Note we are explicitly not using `jl_errorf()` here, as it will attempt to
+            // collect a backtrace, but we're too early in LLVM initialization for that.
+            jl_printf(JL_STDERR, "ERROR: %s", errorstr.c_str());
+            exit(1);
+        }
         if (jl_processor_print_help || (target_flags & JL_TARGET_UNKNOWN_NAME)) {
             std::unique_ptr<MCSubtargetInfo> MSTI(
                 TheTarget->createMCSubtargetInfo(TheTriple.str(), "", ""));
-            if (!MSTI->isCPUStringValid(TheCPU))
-                jl_errorf("Invalid CPU name \"%s\".", TheCPU.c_str());
+            if (!MSTI->isCPUStringValid(TheCPU)) {
+                // Same as above, we are too early to use `jl_errorf()` here.
+                jl_printf(JL_STDERR, "ERROR: Invalid CPU name \"%s\".", TheCPU.c_str());
+                exit(1);
+            }
             if (jl_processor_print_help) {
                 // This is the only way I can find to print the help message once.
                 // It'll be nice if we can iterate through the features and print our own help
diff --git a/src/rtutils.c b/src/rtutils.c
index f3a2e745ed651a..bb64355b799a17 100644
--- a/src/rtutils.c
+++ b/src/rtutils.c
@@ -32,7 +32,7 @@ extern "C" {
 
 JL_DLLEXPORT void JL_NORETURN jl_error(const char *str)
 {
-    if (jl_errorexception_type == NULL) {
+    if (jl_errorexception_type == NULL || jl_error_sym == NULL) {
         jl_printf(JL_STDERR, "ERROR: %s\n", str);
         jl_exit(1);
     }
@@ -72,6 +72,13 @@ JL_DLLEXPORT void JL_NORETURN jl_errorf(const char *fmt, ...)
 {
     va_list args;
     va_start(args, fmt);
+    // Don't attempt to backtrace if we're early on in bringup
+    if (jl_errorexception_type == NULL || jl_error_sym == NULL) {
+        jl_printf(JL_STDERR, "ERROR: ");
+        jl_vprintf(JL_STDERR, fmt, args);
+        jl_printf(JL_STDERR, "\n");
+        jl_exit(1);
+    }
     jl_value_t *e = jl_vexceptionf(jl_errorexception_type, fmt, args);
     va_end(args);
     jl_throw(e);