Remove uses of DECODE_RETURN_KIND part of GCInfo (#110799)

* make hijack frames leaf frames

* not using DECODE_RETURN_KIND in NativeAOT

* delete DECODE_RETURN_KIND

* Fix for unix (x64)

* fixes for AOT

* arm64 fixes

* fix for RISC platforms

* fix arm32 build

* remove unnecessary now PTFF_RAX_IS_GCREF and similar.

* fix fir arm32

* revert to the former value of DECODE_HAS_TAILCALLS - in case there are dependencies in diagnostics.

src/coreclr/gcdump/gcdumpnonx86.cpp

@@ -361,7 +361,6 @@ size_t      GCDump::DumpGCTable(PTR_CBYTE      gcInfoBlock,
                                                   | DECODE_GENERICS_INST_CONTEXT
                                                   | DECODE_GC_LIFETIMES
                                                   | DECODE_PROLOG_LENGTH
-                                                  | DECODE_RETURN_KIND
 #if defined(TARGET_ARM) || defined(TARGET_ARM64) || defined(TARGET_RISCV64) || defined(TARGET_LOONGARCH64)
                                                   | DECODE_HAS_TAILCALLS
 #endif
@@ -502,9 +501,6 @@ size_t      GCDump::DumpGCTable(PTR_CBYTE      gcInfoBlock,
     gcPrintf("Size of parameter area: %x\n", hdrdecoder.GetSizeOfStackParameterArea());
 #endif
 
-    ReturnKind returnKind = hdrdecoder.GetReturnKind();
-    gcPrintf("Return Kind: %s\n", ReturnKindToString(returnKind));
-
     UINT32 cbEncodedMethodSize = hdrdecoder.GetCodeLength();
     gcPrintf("Code size: %x\n", cbEncodedMethodSize);
 

src/coreclr/inc/eetwain.h

@@ -306,7 +306,7 @@ virtual size_t GetFunctionSize(GCInfoToken gcInfoToken) = 0;
 *  returns true.
 *  If hijacking is not possible for some reason, it return false.
 */
-virtual bool GetReturnAddressHijackInfo(GCInfoToken gcInfoToken, ReturnKind * returnKind) = 0;
+virtual bool GetReturnAddressHijackInfo(GCInfoToken gcInfoToken X86_ARG(ReturnKind * returnKind)) = 0;
 
 #ifndef USE_GC_INFO_DECODER
 /*
@@ -575,7 +575,7 @@ size_t GetFunctionSize(GCInfoToken gcInfoToken);
 *  returns true.
 *  If hijacking is not possible for some reason, it return false.
 */
-virtual bool GetReturnAddressHijackInfo(GCInfoToken gcInfoToken, ReturnKind * returnKind);
+virtual bool GetReturnAddressHijackInfo(GCInfoToken gcInfoToken X86_ARG(ReturnKind * returnKind));
 
 #ifndef USE_GC_INFO_DECODER
 /*

src/coreclr/inc/gcinfodecoder.h

@@ -222,7 +222,6 @@ enum GcInfoDecoderFlags
     DECODE_PROLOG_LENGTH         = 0x400,   // length of the prolog (used to avoid reporting generics context)
     DECODE_EDIT_AND_CONTINUE     = 0x800,
     DECODE_REVERSE_PINVOKE_VAR   = 0x1000,
-    DECODE_RETURN_KIND           = 0x2000,
 #if defined(TARGET_ARM) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
     DECODE_HAS_TAILCALLS         = 0x4000,
 #endif // TARGET_ARM || TARGET_ARM64 || TARGET_LOONGARCH64
@@ -582,7 +581,6 @@ class GcInfoDecoder
 #if defined(TARGET_ARM) || defined(TARGET_ARM64) || defined(TARGET_LOONGARCH64) || defined(TARGET_RISCV64)
     bool    HasTailCalls();
 #endif // TARGET_ARM || TARGET_ARM64 || TARGET_LOONGARCH64 || defined(TARGET_RISCV64)
-    ReturnKind GetReturnKind();
     UINT32  GetCodeLength();
     UINT32  GetStackBaseRegister();
     UINT32  GetSizeOfEditAndContinuePreservedArea();
@@ -615,7 +613,6 @@ class GcInfoDecoder
 #ifdef TARGET_ARM64
     UINT32  m_SizeOfEditAndContinueFixedStackFrame;
 #endif
-    ReturnKind m_ReturnKind;
 #ifdef PARTIALLY_INTERRUPTIBLE_GC_SUPPORTED
     UINT32  m_NumSafePoints;
     UINT32  m_SafePointIndex;

src/coreclr/nativeaot/Runtime/AsmOffsets.h

@@ -52,9 +52,11 @@ ASM_OFFSET(   30,    48, Thread, m_pTransitionFrame)
 ASM_OFFSET(   34,    50, Thread, m_pDeferredTransitionFrame)
 ASM_OFFSET(   44,    70, Thread, m_ppvHijackedReturnAddressLocation)
 ASM_OFFSET(   48,    78, Thread, m_pvHijackedReturnAddress)
-ASM_OFFSET(   4c,    80, Thread, m_uHijackedReturnValueFlags)
-ASM_OFFSET(   50,    88, Thread, m_pExInfoStackHead)
-ASM_OFFSET(   54,    90, Thread, m_threadAbortException)
+ASM_OFFSET(   4c,    80, Thread, m_pExInfoStackHead)
+ASM_OFFSET(   50,    88, Thread, m_threadAbortException)
+#ifdef TARGET_X86
+ASM_OFFSET(   54,    FF, Thread, m_uHijackedReturnValueFlags)
+#endif
 
 ASM_SIZEOF(   14,    20, EHEnum)
 

src/coreclr/nativeaot/Runtime/ICodeManager.h

@@ -28,92 +28,11 @@ enum GCRefKind : unsigned char
     GCRK_Scalar         = 0x00,
     GCRK_Object         = 0x01,
     GCRK_Byref          = 0x02,
-#ifdef TARGET_64BIT
-    // Composite return kinds for value types returned in two registers (encoded with two bits per register)
-    GCRK_Scalar_Obj     = (GCRK_Object << 2) | GCRK_Scalar,
-    GCRK_Obj_Obj        = (GCRK_Object << 2) | GCRK_Object,
-    GCRK_Byref_Obj      = (GCRK_Object << 2) | GCRK_Byref,
-    GCRK_Scalar_Byref   = (GCRK_Byref  << 2) | GCRK_Scalar,
-    GCRK_Obj_Byref      = (GCRK_Byref  << 2) | GCRK_Object,
-    GCRK_Byref_Byref    = (GCRK_Byref  << 2) | GCRK_Byref,
-
-    GCRK_LastValid      = GCRK_Byref_Byref,
-#else // TARGET_ARM64
     GCRK_LastValid      = GCRK_Byref,
-#endif // TARGET_ARM64
     GCRK_Unknown        = 0xFF,
 };
 
-#ifdef TARGET_ARM64
-// Verify that we can use bitwise shifts to convert from GCRefKind to PInvokeTransitionFrameFlags and back
-C_ASSERT(PTFF_X0_IS_GCREF == ((uint64_t)GCRK_Object << 32));
-C_ASSERT(PTFF_X0_IS_BYREF == ((uint64_t)GCRK_Byref << 32));
-C_ASSERT(PTFF_X1_IS_GCREF == ((uint64_t)GCRK_Scalar_Obj << 32));
-C_ASSERT(PTFF_X1_IS_BYREF == ((uint64_t)GCRK_Scalar_Byref << 32));
-
-inline uint64_t ReturnKindToTransitionFrameFlags(GCRefKind returnKind)
-{
-    // just need to report gc ref bits here.
-    // appropriate PTFF_SAVE_ bits will be added by the frame building routine.
-    return ((uint64_t)returnKind << 32);
-}
-
-inline GCRefKind TransitionFrameFlagsToReturnKind(uint64_t transFrameFlags)
-{
-    GCRefKind returnKind = (GCRefKind)((transFrameFlags & (PTFF_X0_IS_GCREF | PTFF_X0_IS_BYREF | PTFF_X1_IS_GCREF | PTFF_X1_IS_BYREF)) >> 32);
-    ASSERT((returnKind == GCRK_Scalar) || ((transFrameFlags & PTFF_SAVE_X0) && (transFrameFlags & PTFF_SAVE_X1)));
-    return returnKind;
-}
-
-#elif defined(TARGET_LOONGARCH64)
-// Verify that we can use bitwise shifts to convert from GCRefKind to PInvokeTransitionFrameFlags and back
-C_ASSERT(PTFF_R4_IS_GCREF == ((uint64_t)GCRK_Object << 31));
-C_ASSERT(PTFF_R4_IS_BYREF == ((uint64_t)GCRK_Byref << 31));
-C_ASSERT(PTFF_R5_IS_GCREF == ((uint64_t)GCRK_Scalar_Obj << 31));
-C_ASSERT(PTFF_R5_IS_BYREF == ((uint64_t)GCRK_Scalar_Byref << 31));
-
-inline uint64_t ReturnKindToTransitionFrameFlags(GCRefKind returnKind)
-{
-    // just need to report gc ref bits here.
-    // appropriate PTFF_SAVE_ bits will be added by the frame building routine.
-    return ((uint64_t)returnKind << 31);
-}
-
-inline GCRefKind TransitionFrameFlagsToReturnKind(uint64_t transFrameFlags)
-{
-    GCRefKind returnKind = (GCRefKind)((transFrameFlags & (PTFF_R4_IS_GCREF | PTFF_R4_IS_BYREF | PTFF_R5_IS_GCREF | PTFF_R5_IS_BYREF)) >> 31);
-    ASSERT((returnKind == GCRK_Scalar) || ((transFrameFlags & PTFF_SAVE_R4) && (transFrameFlags & PTFF_SAVE_R5)));
-    return returnKind;
-}
-
-#elif defined(TARGET_AMD64)
-
-// Verify that we can use bitwise shifts to convert from GCRefKind to PInvokeTransitionFrameFlags and back
-C_ASSERT(PTFF_RAX_IS_GCREF == ((uint64_t)GCRK_Object << 16));
-C_ASSERT(PTFF_RAX_IS_BYREF == ((uint64_t)GCRK_Byref << 16));
-C_ASSERT(PTFF_RDX_IS_GCREF == ((uint64_t)GCRK_Scalar_Obj << 16));
-C_ASSERT(PTFF_RDX_IS_BYREF == ((uint64_t)GCRK_Scalar_Byref << 16));
-
-inline uint64_t ReturnKindToTransitionFrameFlags(GCRefKind returnKind)
-{
-    // just need to report gc ref bits here.
-    // appropriate PTFF_SAVE_ bits will be added by the frame building routine.
-    return ((uint64_t)returnKind << 16);
-}
-
-inline GCRefKind TransitionFrameFlagsToReturnKind(uint64_t transFrameFlags)
-{
-    GCRefKind returnKind = (GCRefKind)((transFrameFlags & (PTFF_RAX_IS_GCREF | PTFF_RAX_IS_BYREF | PTFF_RDX_IS_GCREF | PTFF_RDX_IS_BYREF)) >> 16);
-#if defined(TARGET_UNIX)
-    ASSERT((returnKind == GCRK_Scalar) || ((transFrameFlags & PTFF_SAVE_RAX) && (transFrameFlags & PTFF_SAVE_RDX)));
-#else
-    ASSERT((returnKind == GCRK_Scalar) || (transFrameFlags & PTFF_SAVE_RAX));
-#endif
-    return returnKind;
-}
-
-#elif defined(TARGET_X86)
-
+#if defined(TARGET_X86)
 // Verify that we can use bitwise shifts to convert from GCRefKind to PInvokeTransitionFrameFlags and back
 C_ASSERT(PTFF_RAX_IS_GCREF == ((uint64_t)GCRK_Object << 16));
 C_ASSERT(PTFF_RAX_IS_BYREF == ((uint64_t)GCRK_Byref << 16));
@@ -132,40 +51,13 @@ inline GCRefKind TransitionFrameFlagsToReturnKind(uintptr_t transFrameFlags)
     return returnKind;
 }
 
-#elif defined(TARGET_ARM)
-
-// Verify that we can use bitwise shifts to convert from GCRefKind to PInvokeTransitionFrameFlags and back
-C_ASSERT(PTFF_R0_IS_GCREF == ((uint64_t)GCRK_Object << 14));
-C_ASSERT(PTFF_R0_IS_BYREF == ((uint64_t)GCRK_Byref << 14));
-
-inline uint64_t ReturnKindToTransitionFrameFlags(GCRefKind returnKind)
-{
-    // just need to report gc ref bits here.
-    // appropriate PTFF_SAVE_ bits will be added by the frame building routine.
-    return ((uint64_t)returnKind << 14);
-}
-
-inline GCRefKind TransitionFrameFlagsToReturnKind(uint64_t transFrameFlags)
-{
-    GCRefKind returnKind = (GCRefKind)((transFrameFlags & (PTFF_R0_IS_GCREF | PTFF_R0_IS_BYREF)) >> 14);
-    ASSERT((returnKind == GCRK_Scalar) || (transFrameFlags & PTFF_SAVE_R0));
-    return returnKind;
-}
-
-#endif
-
 // Extract individual GCRefKind components from a composite return kind
-inline GCRefKind ExtractReg0ReturnKind(GCRefKind returnKind)
+inline GCRefKind ExtractReturnKind(GCRefKind returnKind)
 {
     ASSERT(returnKind <= GCRK_LastValid);
     return (GCRefKind)(returnKind & (GCRK_Object | GCRK_Byref));
 }
-
-inline GCRefKind ExtractReg1ReturnKind(GCRefKind returnKind)
-{
-    ASSERT(returnKind <= GCRK_LastValid);
-    return (GCRefKind)(returnKind >> 2);
-}
+#endif
 
 //
 // MethodInfo is placeholder type used to allocate space for MethodInfo. Maximum size
@@ -272,9 +164,14 @@ class ICodeManager
     virtual bool IsUnwindable(PTR_VOID pvAddress) PURE_VIRTUAL
 
     virtual bool GetReturnAddressHijackInfo(MethodInfo *    pMethodInfo,
-                                            REGDISPLAY *    pRegisterSet,           // in
-                                            PTR_PTR_VOID *  ppvRetAddrLocation,     // out
-                                            GCRefKind *     pRetValueKind) PURE_VIRTUAL     // out
+                                            REGDISPLAY *    pRegisterSet,          // in
+                                            PTR_PTR_VOID *  ppvRetAddrLocation     // out
+                                            ) PURE_VIRTUAL
+
+#ifdef TARGET_X86
+    virtual GCRefKind GetReturnValueKind(MethodInfo *   pMethodInfo,
+                                  REGDISPLAY *   pRegisterSet) PURE_VIRTUAL
+#endif
 
     virtual PTR_VOID RemapHardwareFaultToGCSafePoint(MethodInfo * pMethodInfo, PTR_VOID controlPC) PURE_VIRTUAL
 

src/coreclr/nativeaot/Runtime/StackFrameIterator.cpp

@@ -77,10 +77,18 @@ StackFrameIterator::StackFrameIterator(Thread * pThreadToWalk, PInvokeTransition
     {
         InternalInit(pThreadToWalk, pThreadToWalk->GetInterruptedContext(), GcStackWalkFlags | ActiveStackFrame);
     }
-    else
+    else if (pInitialTransitionFrame == TOP_OF_STACK_MARKER)
     {
         InternalInit(pThreadToWalk, pInitialTransitionFrame, GcStackWalkFlags);
     }
+    else
+    {
+        uint32_t flags = (pInitialTransitionFrame->m_Flags & PTFF_THREAD_HIJACK) == 0 ?
+            GcStackWalkFlags :
+            GcStackWalkFlags | ActiveStackFrame;
+
+        InternalInit(pThreadToWalk, pInitialTransitionFrame, flags);
+    }
 
     PrepareToYieldFrame();
 }
@@ -110,8 +118,10 @@ void StackFrameIterator::EnterInitialInvalidState(Thread * pThreadToWalk)
     m_pThread = pThreadToWalk;
     m_pInstance = GetRuntimeInstance();
     m_pCodeManager = NULL;
+#ifdef TARGET_X86
     m_pHijackedReturnValue = NULL;
     m_HijackedReturnValueKind = GCRK_Unknown;
+#endif
     m_pConservativeStackRangeLowerBound = NULL;
     m_pConservativeStackRangeUpperBound = NULL;
     m_pendingFuncletFramePointer = NULL;
@@ -180,17 +190,6 @@ void StackFrameIterator::InternalInit(Thread * pThreadToWalk, PInvokeTransitionF
     if (pFrame->m_Flags & PTFF_SAVE_R3)  { m_RegDisplay.pR3 = pPreservedRegsCursor++; }
     if (pFrame->m_Flags & PTFF_SAVE_LR)  { m_RegDisplay.pLR = pPreservedRegsCursor++; }
 
-    if (pFrame->m_Flags & PTFF_R0_IS_GCREF)
-    {
-        m_pHijackedReturnValue = (PTR_OBJECTREF) m_RegDisplay.pR0;
-        m_HijackedReturnValueKind = GCRK_Object;
-    }
-    if (pFrame->m_Flags & PTFF_R0_IS_BYREF)
-    {
-        m_pHijackedReturnValue = (PTR_OBJECTREF) m_RegDisplay.pR0;
-        m_HijackedReturnValueKind = GCRK_Byref;
-    }
-
 #elif defined(TARGET_ARM64)
     m_RegDisplay.pFP = (PTR_uintptr_t)PTR_HOST_MEMBER_TADDR(PInvokeTransitionFrame, pFrame, m_FramePointer);
     m_RegDisplay.pLR = (PTR_uintptr_t)PTR_HOST_MEMBER_TADDR(PInvokeTransitionFrame, pFrame, m_RIP);
@@ -232,13 +231,6 @@ void StackFrameIterator::InternalInit(Thread * pThreadToWalk, PInvokeTransitionF
 
     if (pFrame->m_Flags & PTFF_SAVE_LR) { m_RegDisplay.pLR = pPreservedRegsCursor++; }
 
-    GCRefKind retValueKind = TransitionFrameFlagsToReturnKind(pFrame->m_Flags);
-    if (retValueKind != GCRK_Scalar)
-    {
-        m_pHijackedReturnValue = (PTR_OBJECTREF)m_RegDisplay.pX0;
-        m_HijackedReturnValueKind = retValueKind;
-    }
-
 #elif defined(TARGET_LOONGARCH64)
     m_RegDisplay.pFP = (PTR_uintptr_t)PTR_HOST_MEMBER_TADDR(PInvokeTransitionFrame, pFrame, m_FramePointer);
     m_RegDisplay.pRA = (PTR_uintptr_t)PTR_HOST_MEMBER_TADDR(PInvokeTransitionFrame, pFrame, m_RIP);
@@ -280,13 +272,6 @@ void StackFrameIterator::InternalInit(Thread * pThreadToWalk, PInvokeTransitionF
 
     if (pFrame->m_Flags & PTFF_SAVE_RA) { m_RegDisplay.pRA = pPreservedRegsCursor++; }
 
-    GCRefKind retValueKind = TransitionFrameFlagsToReturnKind(pFrame->m_Flags);
-    if (retValueKind != GCRK_Scalar)
-    {
-        m_pHijackedReturnValue = (PTR_OBJECTREF)m_RegDisplay.pR4;
-        m_HijackedReturnValueKind = retValueKind;
-    }
-
 #else // TARGET_ARM
     if (pFrame->m_Flags & PTFF_SAVE_RBX)  { m_RegDisplay.pRbx = pPreservedRegsCursor++; }
     if (pFrame->m_Flags & PTFF_SAVE_RSI)  { m_RegDisplay.pRsi = pPreservedRegsCursor++; }
@@ -314,12 +299,14 @@ void StackFrameIterator::InternalInit(Thread * pThreadToWalk, PInvokeTransitionF
     if (pFrame->m_Flags & PTFF_SAVE_R11)  { m_RegDisplay.pR11 = pPreservedRegsCursor++; }
 #endif // TARGET_AMD64
 
+#ifdef TARGET_X86
     GCRefKind retValueKind = TransitionFrameFlagsToReturnKind(pFrame->m_Flags);
     if (retValueKind != GCRK_Scalar)
     {
         m_pHijackedReturnValue = (PTR_OBJECTREF)m_RegDisplay.pRax;
         m_HijackedReturnValueKind = retValueKind;
     }
+#endif
 
 #endif // TARGET_ARM
 
@@ -1529,8 +1516,10 @@ void StackFrameIterator::NextInternal()
     m_dwFlags &= ~(ExCollide|MethodStateCalculated|UnwoundReversePInvoke|ActiveStackFrame);
     ASSERT(IsValid());
 
+#ifdef TARGET_X86
     m_pHijackedReturnValue = NULL;
     m_HijackedReturnValueKind = GCRK_Unknown;
+#endif
 
 #ifdef _DEBUG
     SetControlPC(dac_cast<PTR_VOID>((void*)666));
@@ -1944,6 +1933,7 @@ void StackFrameIterator::CalculateCurrentMethodState()
     m_dwFlags |= MethodStateCalculated;
 }
 
+#ifdef TARGET_X86
 bool StackFrameIterator::GetHijackedReturnValueLocation(PTR_OBJECTREF * pLocation, GCRefKind * pKind)
 {
     if (GCRK_Unknown == m_HijackedReturnValueKind)
@@ -1955,6 +1945,7 @@ bool StackFrameIterator::GetHijackedReturnValueLocation(PTR_OBJECTREF * pLocatio
     *pKind = m_HijackedReturnValueKind;
     return true;
 }
+#endif
 
 void StackFrameIterator::SetControlPC(PTR_VOID controlPC)
 {

src/coreclr/nativeaot/Runtime/StackFrameIterator.h

@@ -51,7 +51,9 @@ class StackFrameIterator
     PTR_ICodeManager GetCodeManager();
     MethodInfo *     GetMethodInfo();
     bool             IsActiveStackFrame();
+#ifdef TARGET_X86
     bool             GetHijackedReturnValueLocation(PTR_OBJECTREF * pLocation, GCRefKind * pKind);
+#endif
     void             SetControlPC(PTR_VOID controlPC);
 
     static bool     IsValidReturnAddress(PTR_VOID pvAddress);
@@ -221,10 +223,12 @@ class StackFrameIterator
     PTR_ICodeManager    m_pCodeManager;
     MethodInfo          m_methodInfo;
     PTR_VOID            m_effectiveSafePointAddress;
+#ifdef TARGET_X86
     PTR_OBJECTREF       m_pHijackedReturnValue;
     GCRefKind           m_HijackedReturnValueKind;
-    PTR_uintptr_t      m_pConservativeStackRangeLowerBound;
-    PTR_uintptr_t      m_pConservativeStackRangeUpperBound;
+#endif
+    PTR_uintptr_t       m_pConservativeStackRangeLowerBound;
+    PTR_uintptr_t       m_pConservativeStackRangeUpperBound;
     uint32_t            m_dwFlags;
     PTR_ExInfo          m_pNextExInfo;
     PTR_VOID            m_pendingFuncletFramePointer;

src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc

@@ -356,9 +356,8 @@ PTFF_SAVE_ALL_PRESERVED equ 000000F7h   ;; NOTE: RBP is not included in this set
 PTFF_SAVE_RSP           equ 00008000h
 PTFF_SAVE_RAX           equ 00000100h   ;; RAX is saved in hijack handler - in case it contains a GC ref
 PTFF_SAVE_ALL_SCRATCH   equ 00007F00h
-PTFF_RAX_IS_GCREF       equ 00010000h   ;; iff PTFF_SAVE_RAX: set -> eax is Object, clear -> eax is scalar
-PTFF_RAX_IS_BYREF       equ 00020000h   ;; iff PTFF_SAVE_RAX: set -> eax is ByRef, clear -> eax is Object or scalar
 PTFF_THREAD_ABORT       equ 00100000h   ;; indicates that ThreadAbortException should be thrown when returning from the transition
+PTFF_THREAD_HIJACK      equ 00200000h   ;; indicates that this is a frame for a hijacked call
 
 ;; These must match the TrapThreadsFlags enum
 TrapThreadsFlags_None            equ 0