Arm64: review reworks

src/JitInterface/src/CorInfoImpl.cs

@@ -2708,6 +2708,10 @@ private void allocUnwindInfo(byte* pHotCode, byte* pColdCode, uint startOffset,
             _frameInfos[_usedFrameInfos++] = new FrameInfo(flags, (int)startOffset, (int)endOffset, blobData);
         }
 
+        // Get the CFI data in the same shape as clang/LLVM generated one. This improves the compatibility with libunwind and other unwind solutions
+        // - Combine in one single block for the whole prolog instead of one CFI block per assembler instruction
+        // - Store CFA definition first
+        // - Store all used registers in ascending order
         private byte[] CompressARM64CFI(byte[] blobData)
         {
             if (blobData == null || blobData.Length == 0)
@@ -2818,7 +2822,6 @@ private byte[] CompressARM64CFI(byte[] blobData)
                             cfiWriter.Write((byte)CFI_OPCODE.CFI_DEF_CFA);
                             cfiWriter.Write((short)31); 
                             cfiWriter.Write(spOffset);
-                            //storeOffset = -spOffset;
                         }
                     }
 

src/Native/ObjWriter/objwriter.cpp

@@ -310,10 +310,11 @@ void ObjectWriter::SetCodeSectionAttribute(const char *SectionName,
 }
 
 void ObjectWriter::EmitAlignment(int ByteAlignment) {
-  int64_t fillValue = 0x90; //x86 nop
+  int64_t fillValue = 0;
 
-  if (TMachine->getTargetTriple().getArch() == llvm::Triple::ArchType::aarch64) {
-    fillValue = 0; // ARM64 bad
+  if (TMachine->getTargetTriple().getArch() == llvm::Triple::ArchType::x86 ||
+      TMachine->getTargetTriple().getArch() == llvm::Triple::ArchType::x86_64) {
+    fillValue = 0x90; // x86 nop
   }
 
   Streamer->EmitValueToAlignment(ByteAlignment, fillValue);

src/Native/Runtime/arm64/AllocFast.S

@@ -6,8 +6,6 @@
 
 // GC type flags
 GC_ALLOC_FINALIZE               = 1
-GC_ALLOC_ALIGN8_BIAS            = 4
-GC_ALLOC_ALIGN8                 = 8
 
 //
 // Rename fields of nested structs

src/Native/Runtime/arm64/AsmMacros.h

@@ -247,6 +247,15 @@ __SECTIONREL_tls_CurrentThread SETS "$__SECTIONREL_tls_CurrentThread":CC:"_"
 0
     MEND
 
+;; ---------------------------------------------------------------------------- -
+;;
+;; Macro to add a memory barrier. Equal to __sync_synchronize().
+;;
+
+    MACRO ArmInterlockedOperationBarrier
+        dmb ish
+    MEND
+
 ;; -----------------------------------------------------------------------------
 ;;
 ;; Macro used from unmanaged helpers called from managed code where the helper does not transition immediately

src/Native/Runtime/arm64/ExceptionHandling.S

@@ -465,8 +465,8 @@ PopExInfoLoop:
 DonePopping:
         str         x3, [x1, #OFFSETOF__Thread__m_pExInfoStackHead]     // store the new head on the Thread
 
-        adrp x3, RhpTrapThreads
-        add	x3, x3, :lo12:RhpTrapThreads
+        PREPARE_EXTERNAL_VAR RhpTrapThreads, x3 
+
         ldr         w3, [x3]
         tbz         x3, #TrapThreadsFlags_AbortInProgress_Bit, NoAbort
 

src/Native/Runtime/arm64/Interlocked.S

@@ -18,7 +18,9 @@ ALTERNATE_ENTRY RhpLockCmpXchg32AVLocation
     bne     2f              // if (w0 != w2) goto exit
     stlxr   w9, w1, [x8]    // if (w0 == w2) { try *x8 = w1 and goto loop if failed or goto exit }
     cbnz    w9, 1b
+
 2: // exit
+    ArmInterlockedOperationBarrier
     ret
 LEAF_END RhpLockCmpXchg32, _TEXT
 
@@ -38,5 +40,6 @@ ALTERNATE_ENTRY RhpLockCmpXchg64AVLocation
     stlxr   w9, x1, [x8]    // if (x0 == x2) { try *x8 = x1 and goto loop if failed or goto exit }
     cbnz    w9, 1b
 2: // exit
+    ArmInterlockedOperationBarrier
     ret
 LEAF_END RhpLockCmpXchg64, _TEXT

src/Native/Runtime/arm64/PInvoke.S

@@ -8,40 +8,7 @@
 .global RhpTrapThreads
 
 // Note: these must match the defs in PInvokeTransitionFrameFlags defined in rhbinder.h
-PTFF_SAVE_X19           = 0x00000001
-PTFF_SAVE_X20           = 0x00000002
-PTFF_SAVE_X21           = 0x00000004
-PTFF_SAVE_X22           = 0x00000008
-PTFF_SAVE_X23           = 0x00000010
-PTFF_SAVE_X24           = 0x00000020
-PTFF_SAVE_X25           = 0x00000040
-PTFF_SAVE_X26           = 0x00000080
-PTFF_SAVE_X27           = 0x00000100
-PTFF_SAVE_X28           = 0x00000200
 PTFF_SAVE_SP            = 0x00000400
-PTFF_SAVE_ALL_PRESERVED = 0x000003FF  // NOTE: x19-x28
-PTFF_SAVE_X0            = 0x00000800
-PTFF_SAVE_X1            = 0x00001000
-PTFF_SAVE_X2            = 0x00002000
-PTFF_SAVE_X3            = 0x00004000
-PTFF_SAVE_X4            = 0x00008000
-PTFF_SAVE_X5            = 0x00010000
-PTFF_SAVE_X6            = 0x00020000
-PTFF_SAVE_X7            = 0x00040000
-PTFF_SAVE_X8            = 0x00080000
-PTFF_SAVE_X9            = 0x00100000
-PTFF_SAVE_X10           = 0x00200000
-PTFF_SAVE_X11           = 0x00400000
-PTFF_SAVE_X12           = 0x00800000
-PTFF_SAVE_X13           = 0x01000000
-PTFF_SAVE_X14           = 0x02000000
-PTFF_SAVE_X15           = 0x04000000
-PTFF_SAVE_X16           = 0x08000000
-PTFF_SAVE_X17           = 0x10000000
-PTFF_SAVE_X18           = 0x20000000
-PTFF_SAVE_ALL_SCRATCH   = 0x3FFFF800  // NOTE: X0-X18
-PTFF_SAVE_FP            = 0x40000000
-PTFF_SAVE_LR            = 0x80000000
 
 // Bit position for the flags above, to be used with tbz / tbnz instructions
 PTFF_THREAD_ABORT_BIT   = 36
@@ -151,8 +118,8 @@ Done:
 
         PROLOG_SAVE_REG_PAIR_INDEXED    fp, lr, #-0x10
 
-        adrp x10, RhpTrapThreads
-        add	x10, x10, :lo12:RhpTrapThreads
+        PREPARE_EXTERNAL_VAR RhpTrapThreads, x10
+
         ldr         w10, [x10]
         tbz         x10, #TrapThreadsFlags_TrapThreads_Bit, NoWait
         bl          RhpWaitForGCNoAbort
@@ -214,8 +181,8 @@ ThreadAttached:
         str         xzr, [x10, #OFFSETOF__Thread__m_pTransitionFrame] 
         dmb         ish
 
-        adrp x11, RhpTrapThreads
-        add	x11, x11, :lo12:RhpTrapThreads
+        PREPARE_EXTERNAL_VAR RhpTrapThreads, x11
+
         ldr         w11, [x11]
         tbnz        x11, #TrapThreadsFlags_TrapThreads_Bit, TrapThread
 
@@ -327,8 +294,8 @@ NESTED_ENTRY RhpPInvoke, _TEXT, NoHandler
         str     x10, [x0, #OFFSETOF__PInvokeTransitionFrame__m_pThread]
         str     x0, [x10, #OFFSETOF__Thread__m_pTransitionFrame]
 
-        adrp x9, RhpTrapThreads
-        add	x9, x9, :lo12:RhpTrapThreads
+        PREPARE_EXTERNAL_VAR RhpTrapThreads, x9
+
         ldr     w9, [x9]
         cbnz    w9, InvokeRareTrapThread  // TrapThreadsFlags_None = 0
         ret
@@ -343,8 +310,8 @@ LEAF_ENTRY RhpPInvokeReturn, _TEXT
         mov         x10, 0
         str         x10, [x9, #OFFSETOF__Thread__m_pTransitionFrame]
 
-        adrp x9, RhpTrapThreads
-        add	x9, x9, :lo12:RhpTrapThreads
+        PREPARE_EXTERNAL_VAR RhpTrapThreads, x9
+
         ldr     w9, [x9]
         cbnz    w9, 0f  // TrapThreadsFlags_None = 0
         ret

src/Native/Runtime/arm64/StubDispatch.S

@@ -4,8 +4,6 @@
 #include <unixasmmacros.inc>
 #include "AsmOffsets.inc"
 
-#define __tls_array 0
-
 #ifdef FEATURE_CACHED_INTERFACE_DISPATCH
 
     .extern RhpCidResolve

src/Native/Runtime/arm64/WriteBarriers.S

@@ -189,18 +189,14 @@ INVALIDGCVALUE  EQU 0xCCCCCCCD
 // On exit:
 //   x1 : trashed
 //   x9 : trashed
-    LEAF_ENTRY RhpCheckedAssignRef, _TEXT
-    ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
+ LEAF_ENTRY RhpCheckedAssignRef, _TEXT
     ALTERNATE_ENTRY RhpCheckedAssignRefX1
-    ALTERNATE_ENTRY RhpCheckedAssignRefX1AVLocation
 
-        stlr    x1, [x0]
-
-        INSERT_CHECKED_WRITE_BARRIER_CORE x0, x1, 9, x1
-
-        ret
+        mov     x14, x0                     ; x14 = dst
+        mov     x15, x1                     ; x15 = val
+        b       RhpCheckedAssignRefArm64
 
-    LEAF_END RhpCheckedAssignRef, _TEXT
+LEAF_END RhpCheckedAssignRef, _TEXT
 
 // RhpAssignRef(Object** dst, Object* src)
 //
@@ -213,18 +209,14 @@ INVALIDGCVALUE  EQU 0xCCCCCCCD
 // On exit:
 //  x1 : trashed
 //  x9 : trashed
-    LEAF_ENTRY RhpAssignRef, _TEXT
-    ALTERNATE_ENTRY RhpAssignRefAVLocation
+LEAF_ENTRY RhpAssignRef, _TEXT
     ALTERNATE_ENTRY RhpAssignRefX1
-    ALTERNATE_ENTRY RhpAssignRefX1AVLocation
 
-        stlr    x1, [x0]
-
-        INSERT_UNCHECKED_WRITE_BARRIER_CORE x0, x1, 9, x1
-
-        ret
+        mov     x14, x0                     ; x14 = dst
+        mov     x15, x1                     ; x15 = val
+        b       RhpAssignRefArm64
 
-    LEAF_END RhpAssignRef, _TEXT
+LEAF_END RhpAssignRef, _TEXT
 
 // Interlocked operation helpers where the location is an objectref, thus requiring a GC write barrier upon
 // successful updates. 
@@ -260,7 +252,7 @@ CmpXchgRetry:
         stlxr   w9, x1, [x0]
         cbnz    w9, CmpXchgRetry
 
-        // We have successfully updated the value of the objectref so now we need a GC write barrier.
+                // We have successfully updated the value of the objectref so now we need a GC write barrier.
         // The following barrier code takes the destination in x0 and the value in x1 so the arguments are
         // already correctly set up.
 
@@ -269,6 +261,7 @@ CmpXchgRetry:
 CmpXchgNoUpdate:
         // x10 still contains the original value.
         mov     x0, x10
+        ArmInterlockedOperationBarrier
         ret     lr
 
     LEAF_END RhpCheckedLockCmpXchg, _TEXT
@@ -309,11 +302,14 @@ ExchangeRetry:
 
         // x10 still contains the original value.
         mov     x0, x10
+        ArmInterlockedOperationBarrier
         ret
 
     LEAF_END RhpCheckedXchg, _TEXT
 
 LEAF_ENTRY RhpAssignRefArm64, _TEXT
+    ALTERNATE_ENTRY RhpAssignRefAVLocation
+    ALTERNATE_ENTRY RhpAssignRefX1AVLocation
         stlr    x15, [x14]
 
         INSERT_UNCHECKED_WRITE_BARRIER_CORE x14, x15, 12, X14
@@ -333,6 +329,8 @@ LEAF_END RhpAssignRefArm64, _TEXT
 //   x17  : trashed (ip1) if FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
 //
 LEAF_ENTRY RhpCheckedAssignRefArm64, _TEXT
+    ALTERNATE_ENTRY RhpCheckedAssignRefAVLocation
+    ALTERNATE_ENTRY RhpCheckedAssignRefX1AVLocation
 
         stlr    x15, [x14]
 

src/Native/Runtime/arm64/WriteBarriers.asm

@@ -271,6 +271,7 @@ CmpXchgRetry
 CmpXchgNoUpdate
         ;; x10 still contains the original value.
         mov     x0, x10
+        ArmInterlockedOperationBarrier
         ret     lr
 
     LEAF_END RhpCheckedLockCmpXchg
@@ -311,6 +312,7 @@ ExchangeRetry
 
         ;; x10 still contains the original value.
         mov     x0, x10
+        ArmInterlockedOperationBarrier
         ret
 
     LEAF_END RhpCheckedXchg

src/Native/Runtime/unix/unixasmmacrosarm64.inc

@@ -24,6 +24,11 @@ C_FUNC(\Name):
 C_FUNC(\Name):
 .endm
 
+.macro LABELED_RETURN_ADDRESS Name
+        .global C_FUNC(\Name)
+C_FUNC(\Name):
+.endm
+
 .macro LEAF_ENTRY Name, Section
         .global C_FUNC(\Name)
         .type \Name, %function
@@ -37,8 +42,9 @@ C_FUNC(\Name):
 .endm
 
 .macro PREPARE_EXTERNAL_VAR Name, HelperReg
-        ldr \HelperReg, [pc, #C_FUNC(\Name)@GOTPCREL]
-.endm
+        adrp \HelperReg, C_FUNC(\Name)
+        add	\HelperReg, \HelperReg, :lo12:C_FUNC(\Name)
+.endm 
 
 .macro PROLOG_STACK_ALLOC Size
         sub sp, sp, \Size
@@ -152,7 +158,7 @@ C_FUNC(\Name):
 .macro INLINE_GET_TLS_VAR target, var
     mrs	\target, tpidr_el0
     add	\target, \target, #:tprel_hi12:\var, lsl #12
-	add	\target, \target, #:tprel_lo12_nc:\var
+    add	\target, \target, #:tprel_lo12_nc:\var
 .endm
 
 
@@ -164,6 +170,10 @@ C_FUNC(\Name):
     INLINE_GET_TLS_VAR \target, tls_CurrentThread
 .endm
 
+.macro  ArmInterlockedOperationBarrier
+    dmb ish
+.endm
+
 .macro INLINE_THREAD_UNHIJACK threadReg, trashReg1, trashReg2
     //
     // Thread::Unhijack()
@@ -232,12 +242,16 @@ DEFAULT_FRAME_SAVE_FLAGS = PTFF_SAVE_ALL_PRESERVED + PTFF_SAVE_SP
     EPILOG_RESTORE_REG_PAIR_INDEXED   fp, lr, #0x80
 .endm
 
+// Bit position for the flags above, to be used with tbz / tbnz instructions
+PTFF_THREAD_ABORT_BIT = 36
+
 //
 // CONSTANTS -- INTEGER
 //
 #define TSF_Attached                    0x01
 #define TSF_SuppressGcStress            0x08
 #define TSF_DoNotTriggerGc              0x10
+#define TSF_SuppressGcStress__OR__TSF_DoNotTriggerGC 0x18
 
 // Bit position for the flags above, to be used with tbz / tbnz instructions
 TrapThreadsFlags_AbortInProgress_Bit = 0

src/Native/libunwind/src/UnwindCursor.hpp

@@ -1904,7 +1904,7 @@ void UnwindCursor<A, R>::setInfoBasedOnIPRegister(bool isReturnAddress) {
 #if defined(_LIBUNWIND_SUPPORT_DWARF_UNWIND)
     // If there is dwarf unwind info, look there next.
 #if defined(_LIBUNWIND_USE_ONLY_DWARF_INDEX)
-      if (sects.dwarf_index_section != 0) {
+    if (sects.dwarf_index_section != 0) {
 #else
     if (sects.dwarf_section != 0) {
 #endif