Report StackOverflowException on NativeAOT on Linux

src/coreclr/nativeaot/Runtime/inc/CommonTypes.h

@@ -4,6 +4,7 @@
 #ifndef __COMMON_TYPES_H__
 #define __COMMON_TYPES_H__
 
+#include <assert.h>
 #include <cstddef>
 #include <cstdint>
 #include <stdlib.h>

src/coreclr/nativeaot/Runtime/thread.cpp

@@ -5,6 +5,7 @@
 #include "CommonTypes.h"
 #include "CommonMacros.h"
 #include "daccess.h"
+#include "CommonMacros.inl"
 #include "PalRedhawkCommon.h"
 #include "PalRedhawk.h"
 #include "rhassert.h"
@@ -26,6 +27,8 @@
 #include "stressLog.h"
 #include "RhConfig.h"
 #include "RhVolatile.h"
+#include <signal.h>
+#include <sys/mman.h>
 
 #ifndef DACCESS_COMPILE
 
@@ -281,6 +284,9 @@ void Thread::Construct()
     if (StressLog::StressLogOn(~0u, 0))
         m_pThreadStressLog = StressLog::CreateThreadStressLog(this);
 #endif // STRESS_LOG
+#ifdef TARGET_UNIX
+    EnsureSignalAlternateStack();
+#endif // TARGET_UNIX
 
     // Everything else should be initialized to 0 via the static initialization of tls_CurrentThread.
 
@@ -297,6 +303,89 @@ void Thread::Construct()
     ASSERT(m_interruptedContext == NULL);
 }
 
+#ifdef TARGET_UNIX
+void Thread::FreeSignalAlternateStack()
+{
+    void *altstack = m_alternateStack;
+    m_alternateStack = nullptr;
+
+    if (altstack != nullptr)
+    {
+        stack_t ss, oss;
+        // The man page for sigaltstack says that when the ss.ss_flags is set to SS_DISABLE,
+        // all other ss fields are ignored. However, MUSL implementation checks that the
+        // ss_size is >= MINSIGSTKSZ even in this case.
+        ss.ss_size = MINSIGSTKSZ;
+        ss.ss_flags = SS_DISABLE;
+        ss.ss_sp = NULL;
+        int st = sigaltstack(&ss, &oss);
+        if ((st == 0) && (oss.ss_flags != SS_DISABLE))
+        {
+            // Make sure this altstack is this PAL's before freeing.
+            if (oss.ss_sp == altstack)
+            {
+                int st = munmap(oss.ss_sp, oss.ss_size);
+                _ASSERTE(st == 0);
+            }
+        }
+    }
+}
+
+bool Thread::EnsureSignalAlternateStack()
+{
+    int st = 0;
+
+    stack_t oss;
+
+    // Query the current alternate signal stack
+    st = sigaltstack(NULL, &oss);
+    if ((st == 0) && (oss.ss_flags == SS_DISABLE))
+    {
+        // There is no alternate stack for SIGSEGV handling installed yet so allocate one
+
+        // We include the size of the SignalHandlerWorkerReturnPoint in the alternate stack size since the
+        // context contained in it is large and the SIGSTKSZ was not sufficient on ARM64 during testing.
+        int altStackSize = SIGSTKSZ + ALIGN_UP(sizeof(CONTEXT), 16) + getpagesize();
+#ifdef HAS_ADDRESS_SANITIZER
+        // Asan also uses alternate stack so we increase its size on the SIGSTKSZ * 4 that enough for asan
+        // (see kAltStackSize in compiler-rt/lib/sanitizer_common/sanitizer_posix_libcdep.cc)
+        altStackSize += SIGSTKSZ * 4;
+#endif
+        altStackSize = ALIGN_UP(altStackSize, getpagesize());
+        int flags = MAP_ANONYMOUS | MAP_PRIVATE;
+#ifdef MAP_STACK
+        flags |= MAP_STACK;
+#endif
+        void* altStack = mmap(NULL, altStackSize, PROT_READ | PROT_WRITE, flags, -1, 0);
+        if (altStack != MAP_FAILED)
+        {
+            // create a guard page for the alternate stack
+            st = mprotect(altStack, getpagesize(), PROT_NONE);
+            if (st == 0)
+            {
+                stack_t ss;
+                ss.ss_sp = (char*)altStack;
+                ss.ss_size = altStackSize;
+                ss.ss_flags = 0;
+                st = sigaltstack(&ss, NULL);
+            }
+
+            if (st == 0)
+            {
+                m_alternateStack = altStack;
+            }
+            else
+            {
+                int st2 = munmap(altStack, altStackSize);
+                _ASSERTE(st2 == 0);
+            }
+        }
+    }
+
+    return (st == 0);
+}
+#endif // TARGET_UNIX
+
 bool Thread::IsInitialized()
 {
     return (m_ThreadStateFlags != TSF_Unknown);
@@ -361,6 +450,10 @@ void Thread::Destroy()
     }
 #endif //FEATURE_SUSPEND_REDIRECTION
 
+#ifdef TARGET_UNIX
+    FreeSignalAlternateStack();
+#endif
+
     ASSERT(m_pGCFrameRegistrations == NULL);
 }
 
@@ -866,19 +959,19 @@ void Thread::Unhijack()
 }
 
 // This unhijack routine is called to undo a hijack, that is potentially on a different thread.
-// 
+//
 // Although there are many code sequences (here and in asm) to
 // perform an unhijack operation, they will never execute concurrently:
-// 
+//
 // - A thread may unhijack itself at any time so long as it does that from unmanaged code while in coop mode.
 //   This ensures that coop thread can access its stack synchronously.
 //   Unhijacking from unmanaged code ensures that another thread will not attempt to hijack it,
 //   since we only hijack threads that are executing managed code.
-// 
+//
 // - A GC thread may access a thread asynchronously, including unhijacking it.
 //   Asynchronously accessed thread must be in preemptive mode and should not
 //   access the managed portion of its stack.
-// 
+//
 // - A thread that owns the suspension can access another thread as long as the other thread is
 //   in preemptive mode or suspended in managed code.
 //   Either way the other thread cannot be accessing its hijack.

src/coreclr/nativeaot/Runtime/thread.h

@@ -94,7 +94,7 @@ struct ThreadBuffer
     HANDLE                  m_hPalThread;                           // WARNING: this may legitimately be INVALID_HANDLE_VALUE
     void **                 m_ppvHijackedReturnAddressLocation;
     void *                  m_pvHijackedReturnAddress;
-    uintptr_t               m_uHijackedReturnValueFlags;            
+    uintptr_t               m_uHijackedReturnValueFlags;
     PTR_ExInfo              m_pExInfoStackHead;
     Object*                 m_threadAbortException;                 // ThreadAbortException instance -set only during thread abort
     Object*                 m_pThreadLocalStatics;
@@ -112,6 +112,9 @@ struct ThreadBuffer
 #ifdef FEATURE_GC_STRESS
     uint32_t                m_uRand;                                // current per-thread random number
 #endif // FEATURE_GC_STRESS
+#ifdef TARGET_UNIX
+    void *                   m_alternateStack;                      // ptr to alternate signal stack
+#endif // UNIX
 };
 
 struct ReversePInvokeFrame
@@ -152,7 +155,7 @@ class Thread : private ThreadBuffer
                                                     // For suspension APCs it is mostly harmless, but wasteful and in extreme
                                                     // cases may force the target thread into stack oveflow.
                                                     // We use this flag to avoid sending another APC when one is still going through.
-                                                    // 
+                                                    //
                                                     // On Unix this is an optimization to not queue up more signals when one is
                                                     // still being processed.
     };
@@ -312,6 +315,11 @@ class Thread : private ThreadBuffer
 
     bool                IsActivationPending();
     void                SetActivationPending(bool isPending);
+
+#ifdef TARGET_UNIX
+    bool EnsureSignalAlternateStack();
+    void FreeSignalAlternateStack();
+#endif // TARGET_UNIX
 };
 
 #ifndef __GCENV_BASE_INCLUDED__

src/coreclr/nativeaot/Runtime/unix/HardwareExceptions.cpp

@@ -11,6 +11,9 @@
 #include "HardwareExceptions.h"
 #include "UnixSignals.h"
 #include "PalCreateDump.h"
+#include "thread.h"
+#include "threadstore.h"
+#include <sys/mman.h>
 
 #if defined(HOST_APPLE)
 #include <mach/mach.h>
@@ -545,6 +548,17 @@ bool HardwareExceptionHandler(int code, siginfo_t *siginfo, void *context, void*
 // Handler for the SIGSEGV signal
 void SIGSEGVHandler(int code, siginfo_t *siginfo, void *context)
 {
+    // First check if we have a stack overflow
+    size_t sp = ((UNIX_CONTEXT *)context)->GetSp();
+    size_t failureAddress = (size_t)siginfo->si_addr;
+
+    // If the failure address is at most one page above or below the stack pointer,
+    // we have a stack overflow.
+    if ((failureAddress - (sp - getpagesize())) < 2 * getpagesize())
+    {
+        PalPrintFatalError("\nProcess is terminating due to StackOverflowException.\n");
+        RhFailFast();
+    }
     bool isHandled = HardwareExceptionHandler(code, siginfo, context, siginfo->si_addr);
     if (isHandled)
     {
@@ -589,7 +603,8 @@ void SIGFPEHandler(int code, siginfo_t *siginfo, void *context)
 // Initialize hardware exception handling
 bool InitializeHardwareExceptionHandling()
 {
-    if (!AddSignalHandler(SIGSEGV, SIGSEGVHandler, &g_previousSIGSEGV))
+    // Run SIGSEGV handler on separate stack so we can handle stack overflow. Otherwise, the current (invalid) stack is used and another segfault is raised.
+    if (!AddSignalHandler(SIGSEGV, SIGSEGVHandler, &g_previousSIGSEGV, SA_ONSTACK))
     {
         return false;
     }
@@ -598,26 +613,30 @@ bool InitializeHardwareExceptionHandling()
     {
         return false;
     }
+#ifdef TARGET_UNIX
+    Thread* curThread = ThreadStore::RawGetCurrentThread();
+    curThread->EnsureSignalAlternateStack();
+#endif // TARGET_UNIX
 
 #if defined(HOST_APPLE)
 #ifndef HOST_TVOS // task_set_exception_ports is not supported on tvOS
-	// LLDB installs task-wide Mach exception handlers. XNU dispatches Mach
-	// exceptions first to any registered "activation" handler and then to
-	// any registered task handler before dispatching the exception to a
-	// host-wide Mach exception handler that does translation to POSIX
-	// signals. This makes it impossible to use LLDB with implicit null
+    // LLDB installs task-wide Mach exception handlers. XNU dispatches Mach
+    // exceptions first to any registered "activation" handler and then to
+    // any registered task handler before dispatching the exception to a
+    // host-wide Mach exception handler that does translation to POSIX
+    // signals. This makes it impossible to use LLDB with implicit null
     // checks in NativeAOT; continuing execution after LLDB traps an
     // EXC_BAD_ACCESS will result in LLDB's EXC_BAD_ACCESS handler being
     // invoked again. This also interferes with the translation of SIGFPEs
     // to .NET-level ArithmeticExceptions. Work around this here by
-	// installing a no-op task-wide Mach exception handler for
-	// EXC_BAD_ACCESS and EXC_ARITHMETIC.
-	kern_return_t kr = task_set_exception_ports(
-		mach_task_self(),
-		EXC_MASK_BAD_ACCESS | EXC_MASK_ARITHMETIC, /* SIGSEGV, SIGFPE */
-		MACH_PORT_NULL,
-		EXCEPTION_STATE_IDENTITY,
-		MACHINE_THREAD_STATE);
+    // installing a no-op task-wide Mach exception handler for
+    // EXC_BAD_ACCESS and EXC_ARITHMETIC.
+    kern_return_t kr = task_set_exception_ports(
+        mach_task_self(),
+        EXC_MASK_BAD_ACCESS | EXC_MASK_ARITHMETIC, /* SIGSEGV, SIGFPE */
+        MACH_PORT_NULL,
+        EXCEPTION_STATE_IDENTITY,
+        MACHINE_THREAD_STATE);
     ASSERT(kr == KERN_SUCCESS);
 #endif
 #endif

src/coreclr/nativeaot/Runtime/unix/UnixSignals.cpp

@@ -9,11 +9,11 @@
 #include "UnixSignals.h"
 
 // Add handler for hardware exception signal
-bool AddSignalHandler(int signal, SignalHandler handler, struct sigaction* previousAction)
+bool AddSignalHandler(int signal, SignalHandler handler, struct sigaction* previousAction, int additionalFlags)
 {
     struct sigaction newAction;
 
-    newAction.sa_flags = SA_RESTART;
+    newAction.sa_flags = SA_RESTART | additionalFlags;
     newAction.sa_handler = NULL;
     newAction.sa_sigaction = handler;
     newAction.sa_flags |= SA_SIGINFO;

src/coreclr/nativeaot/Runtime/unix/UnixSignals.h

@@ -14,7 +14,7 @@
 
 typedef void (*SignalHandler)(int code, siginfo_t* siginfo, void* context);
 
-bool AddSignalHandler(int signal, SignalHandler handler, struct sigaction* previousAction);
+bool AddSignalHandler(int signal, SignalHandler handler, struct sigaction* previousAction, int additionalFlags = 0);
 void RestoreSignalHandler(int signal_id, struct sigaction* previousAction);
 
 #endif // __UNIX_SIGNALS_H__

src/tests/nativeaot/SmokeTests/StackOverflowReporting/Program.cs

@@ -0,0 +1,50 @@
+using System;
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+
+public static class StackOverflowReporting
+{
+    public static int Main(string[] args)
+    {
+        if (args.Length == 1 && args[0].ToLowerInvariant() == "recurse")
+        {
+            Recursion(0);
+            return 0;
+        }
+        else
+        {
+            var directory = System.AppContext.BaseDirectory;
+            var exe = System.IO.Path.Combine(directory, "StackOverflowReporting");
+            if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows))
+            {
+                exe += ".exe";
+            }
+            exe = Process.GetCurrentProcess().MainModule.FileName;
+            using (Process p = new Process())
+            {
+                p.StartInfo.FileName = exe;
+                p.StartInfo.Arguments = "recurse";
+                p.StartInfo.RedirectStandardOutput = true;
+                p.StartInfo.RedirectStandardError = true;
+                p.Start();
+                p.WaitForExit();
+                var output = p.StandardOutput.ReadToEnd();
+                var error = p.StandardError.ReadToEnd();
+                if (output.Contains("StackOverflowException") || error.Contains("StackOverflowException"))
+                {
+                    return 100;
+                }
+                else
+                {
+                    return 1;
+                }
+            }
+        }
+    }
+
+    public static void Recursion(int i)
+    {
+        Recursion(i + 1);
+        Recursion(i + 1);
+    }
+}

src/tests/nativeaot/SmokeTests/StackOverflowReporting/StackOverflowReporting.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <CLRTestPriority>0</CLRTestPriority>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <Compile Include="Program.cs" />
+  </ItemGroup>
+
+</Project>