Inline TLS field access for linux/osx x64/arm64

src/coreclr/inc/corinfo.h

@@ -1726,8 +1726,11 @@ struct CORINFO_FIELD_INFO
 
 struct CORINFO_THREAD_STATIC_BLOCKS_INFO
 {
-    CORINFO_CONST_LOOKUP tlsIndex;
-    uint32_t offsetOfThreadLocalStoragePointer;
+    CORINFO_CONST_LOOKUP tlsIndex;              // windows specific
+    void* tlsGetAddrFtnPtr;                     // linux/x64 specific - address of __tls_get_addr() function
+    void* tlsIndexObject;                       // linux/x64 specific - address of tls_index object
+    void* threadVarsSection;                    // osx x64/arm64 specific - address of __thread_vars section of `t_ThreadStatics`
+    uint32_t offsetOfThreadLocalStoragePointer; // windows specific
     uint32_t offsetOfMaxThreadStaticBlocks;
     uint32_t offsetOfThreadStaticBlocks;
     uint32_t offsetOfGCDataPointer;

src/coreclr/inc/jiteeversionguid.h

@@ -43,11 +43,11 @@ typedef const GUID *LPCGUID;
 #define GUID_DEFINED
 #endif // !GUID_DEFINED
 
-constexpr GUID JITEEVersionIdentifier = { /* 878d63a7-ffc9-421f-81f7-db4729f0ed5c */
-    0x878d63a7,
-    0xffc9,
-    0x421f,
-    {0x81, 0xf7, 0xdb, 0x47, 0x29, 0xf0, 0xed, 0x5c}
+constexpr GUID JITEEVersionIdentifier = { /* 02e334af-4e6e-4a68-9feb-308d3d2661bc */
+    0x2e334af,
+    0x4e6e,
+    0x4a68,
+    {0x9f, 0xeb, 0x30, 0x8d, 0x3d, 0x26, 0x61, 0xbc}
   };
 
 //////////////////////////////////////////////////////////////////////////////////////////////////////////

src/coreclr/jit/codegenarm64.cpp

@@ -2944,6 +2944,13 @@ void CodeGen::genCodeForStoreLclVar(GenTreeLclVar* lclNode)
                 inst_Mov_Extend(targetType, /* srcInReg */ true, targetReg, dataReg, /* canSkip */ true,
                                 emitActualTypeSize(targetType));
             }
+            else if (TargetOS::IsUnix && data->IsIconHandle(GTF_ICON_TLS_HDL))
+            {
+                assert(data->AsIntCon()->IconValue() == 0);
+                emitAttr attr = emitActualTypeSize(targetType);
+                // On non-windows, need to load the address from system register.
+                emit->emitIns_R(INS_mrs_tpid0, attr, targetReg);
+            }
             else
             {
                 inst_Mov(targetType, targetReg, dataReg, /* canSkip */ true);

src/coreclr/jit/compiler.cpp

@@ -5026,11 +5026,8 @@ void Compiler::compCompile(void** methodCodePtr, uint32_t* methodCodeSize, JitFl
     // Partially inline static initializations
     DoPhase(this, PHASE_EXPAND_STATIC_INIT, &Compiler::fgExpandStaticInit);
 
-    if (TargetOS::IsWindows)
-    {
-        // Currently this is only applicable for Windows
-        DoPhase(this, PHASE_EXPAND_TLS, &Compiler::fgExpandThreadLocalAccess);
-    }
+    // Expand thread local access
+    DoPhase(this, PHASE_EXPAND_TLS, &Compiler::fgExpandThreadLocalAccess);
 
     // Insert GC Polls
     DoPhase(this, PHASE_INSERT_GC_POLLS, &Compiler::fgInsertGCPolls);

src/coreclr/jit/emit.cpp

@@ -10205,9 +10205,7 @@ void emitter::emitRecordCallSite(ULONG                 instrOffset,  /* IN */
 
     if (callSig == nullptr)
     {
-        assert(methodHandle != nullptr);
-
-        if (Compiler::eeGetHelperNum(methodHandle) == CORINFO_HELP_UNDEF)
+        if ((methodHandle != nullptr) && (Compiler::eeGetHelperNum(methodHandle) == CORINFO_HELP_UNDEF))
         {
             emitComp->eeGetMethodSig(methodHandle, &sigInfo);
             callSig = &sigInfo;

src/coreclr/jit/emitarm64.cpp

@@ -937,7 +937,7 @@ void emitter::emitInsSanityCheck(instrDesc* id)
         case IF_SI_0B: // SI_0B   ................ ....bbbb........               imm4 - barrier
             break;
 
-        case IF_SR_1A: // SR_1A   ................ ...........ttttt      Rt       (dc zva)
+        case IF_SR_1A: // SR_1A   ................ ...........ttttt      Rt       (dc zva, mrs)
             datasize = id->idOpSize();
             assert(isGeneralRegister(id->idReg1()));
             assert(datasize == EA_8BYTE);
@@ -3740,7 +3740,11 @@ void emitter::emitIns_R(instruction ins, emitAttr attr, regNumber reg)
             id->idReg1(reg);
             fmt = IF_SR_1A;
             break;
-
+        case INS_mrs_tpid0:
+            id = emitNewInstrSmall(attr);
+            id->idReg1(reg);
+            fmt = IF_SR_1A;
+            break;
         default:
             unreached();
     }
@@ -11793,7 +11797,7 @@ size_t emitter::emitOutputInstr(insGroup* ig, instrDesc* id, BYTE** dp)
             dst += emitOutput_Instr(dst, code);
             break;
 
-        case IF_SR_1A: // SR_1A   ................ ...........ttttt      Rt       (dc zva)
+        case IF_SR_1A: // SR_1A   ................ ...........ttttt      Rt       (dc zva, mrs)
             assert(insOptsNone(id->idInsOpt()));
             code = emitInsCode(ins, fmt);
             code |= insEncodeReg_Rt(id->idReg1()); // ttttt
@@ -13921,8 +13925,16 @@ void emitter::emitDispInsHelp(
             emitDispBarrier((insBarrier)emitGetInsSC(id));
             break;
 
-        case IF_SR_1A: // SR_1A   ................ ...........ttttt      Rt       (dc zva)
-            emitDispReg(id->idReg1(), size, false);
+        case IF_SR_1A: // SR_1A   ................ ...........ttttt      Rt       (dc zva, mrs)
+            if (ins == INS_mrs_tpid0)
+            {
+                emitDispReg(id->idReg1(), size, true);
+                printf("tpidr_el0");
+            }
+            else
+            {
+                emitDispReg(id->idReg1(), size, false);
+            }
             break;
 
         default:

src/coreclr/jit/emitfmtsarm64.h

@@ -227,7 +227,7 @@ IF_DEF(SN_0A, IS_NONE, NONE) // SN_0A   ................ ................
 IF_DEF(SI_0A, IS_NONE, NONE) // SI_0A   ...........iiiii iiiiiiiiiii.....               imm16
 IF_DEF(SI_0B, IS_NONE, NONE) // SI_0B   ................ ....bbbb........               imm4 - barrier
 
-IF_DEF(SR_1A, IS_NONE, NONE) // SR_1A   ................ ...........ttttt      Rt       (dc zva)
+IF_DEF(SR_1A, IS_NONE, NONE) // SR_1A   ................ ...........ttttt      Rt       (dc zva, mrs)
 
 IF_DEF(INVALID, IS_NONE, NONE) //
 

src/coreclr/jit/helperexpansion.cpp

@@ -478,36 +478,49 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement*
         return false;
     }
 
+    assert(!opts.IsReadyToRun());
+
+    if (TargetOS::IsUnix)
+    {
+#if defined(TARGET_ARM) || !defined(TARGET_64BIT)
+        // On Arm, Thread execution blocks are accessed using co-processor registers and instructions such
+        // as MRC and MCR are used to access them. We do not support them and so should never optimize the
+        // field access using TLS.
+        assert(!"Unsupported scenario of optimizing TLS access on Linux Arm32/x86");
+#endif
+    }
+    else
+    {
 #ifdef TARGET_ARM
-    // On Arm, Thread execution blocks are accessed using co-processor registers and instructions such
-    // as MRC and MCR are used to access them. We do not support them and so should never optimize the
-    // field access using TLS.
-    assert(!"Unsupported scenario of optimizing TLS access on Arm32");
+        // On Arm, Thread execution blocks are accessed using co-processor registers and instructions such
+        // as MRC and MCR are used to access them. We do not support them and so should never optimize the
+        // field access using TLS.
+        assert(!"Unsupported scenario of optimizing TLS access on Windows Arm32");
 #endif
+    }
 
     JITDUMP("Expanding thread static local access for [%06d] in " FMT_BB ":\n", dspTreeID(call), block->bbNum);
     DISPTREE(call);
     JITDUMP("\n");
+
     bool isGCThreadStatic =
         eeGetHelperNum(call->gtCallMethHnd) == CORINFO_HELP_GETSHARED_GCTHREADSTATIC_BASE_NOCTOR_OPTIMIZED;
 
     CORINFO_THREAD_STATIC_BLOCKS_INFO threadStaticBlocksInfo;
-    info.compCompHnd->getThreadLocalStaticBlocksInfo(&threadStaticBlocksInfo, isGCThreadStatic);
+    memset(&threadStaticBlocksInfo, 0, sizeof(CORINFO_THREAD_STATIC_BLOCKS_INFO));
 
-    uint32_t offsetOfMaxThreadStaticBlocksVal = 0;
-    uint32_t offsetOfThreadStaticBlocksVal    = 0;
+    info.compCompHnd->getThreadLocalStaticBlocksInfo(&threadStaticBlocksInfo, isGCThreadStatic);
 
     JITDUMP("getThreadLocalStaticBlocksInfo (%s)\n:", isGCThreadStatic ? "GC" : "Non-GC");
-    offsetOfMaxThreadStaticBlocksVal = threadStaticBlocksInfo.offsetOfMaxThreadStaticBlocks;
-    offsetOfThreadStaticBlocksVal    = threadStaticBlocksInfo.offsetOfThreadStaticBlocks;
-
-    JITDUMP("tlsIndex= %u\n", (ssize_t)threadStaticBlocksInfo.tlsIndex.addr);
     JITDUMP("offsetOfThreadLocalStoragePointer= %u\n", threadStaticBlocksInfo.offsetOfThreadLocalStoragePointer);
-    JITDUMP("offsetOfMaxThreadStaticBlocks= %u\n", offsetOfMaxThreadStaticBlocksVal);
-    JITDUMP("offsetOfThreadStaticBlocks= %u\n", offsetOfThreadStaticBlocksVal);
+    JITDUMP("tlsIndex= %u\n", (ssize_t)threadStaticBlocksInfo.tlsIndex.addr);
+    JITDUMP("tlsGetAddrFtnPtr= %u\n", threadStaticBlocksInfo.tlsGetAddrFtnPtr);
+    JITDUMP("tlsIndexObject= %u\n", (size_t)threadStaticBlocksInfo.tlsIndexObject);
+    JITDUMP("threadVarsSection= %u\n", (size_t)threadStaticBlocksInfo.threadVarsSection);
+    JITDUMP("offsetOfMaxThreadStaticBlocks= %u\n", threadStaticBlocksInfo.offsetOfMaxThreadStaticBlocks);
+    JITDUMP("offsetOfThreadStaticBlocks= %u\n", threadStaticBlocksInfo.offsetOfThreadStaticBlocks);
     JITDUMP("offsetOfGCDataPointer= %u\n", threadStaticBlocksInfo.offsetOfGCDataPointer);
 
-    assert(threadStaticBlocksInfo.tlsIndex.accessType == IAT_VALUE);
     assert((eeGetHelperNum(call->gtCallMethHnd) == CORINFO_HELP_GETSHARED_NONGCTHREADSTATIC_BASE_NOCTOR_OPTIMIZED) ||
            (eeGetHelperNum(call->gtCallMethHnd) == CORINFO_HELP_GETSHARED_GCTHREADSTATIC_BASE_NOCTOR_OPTIMIZED));
 
@@ -546,56 +559,135 @@ bool Compiler::fgExpandThreadLocalAccessForCall(BasicBlock** pBlock, Statement*
     gtUpdateStmtSideEffects(stmt);
 
     GenTree* typeThreadStaticBlockIndexValue = call->gtArgs.GetArgByIndex(0)->GetNode();
+    GenTree* tlsValue                        = nullptr;
+    unsigned tlsLclNum                       = lvaGrabTemp(true DEBUGARG("TLS access"));
+    lvaTable[tlsLclNum].lvType               = TYP_I_IMPL;
+    GenTree* maxThreadStaticBlocksValue      = nullptr;
+    GenTree* threadStaticBlocksValue         = nullptr;
+    GenTree* tlsValueDef                     = nullptr;
+
+    if (TargetOS::IsWindows)
+    {
+        size_t   tlsIndexValue = (size_t)threadStaticBlocksInfo.tlsIndex.addr;
+        GenTree* dllRef        = nullptr;
 
-    void** pIdAddr = nullptr;
+        if (tlsIndexValue != 0)
+        {
+            dllRef = gtNewIconHandleNode(tlsIndexValue * TARGET_POINTER_SIZE, GTF_ICON_TLS_HDL);
+        }
 
-    size_t   tlsIndexValue = (size_t)threadStaticBlocksInfo.tlsIndex.addr;
-    GenTree* dllRef        = nullptr;
+        // Mark this ICON as a TLS_HDL, codegen will use FS:[cns] or GS:[cns]
+        tlsValue = gtNewIconHandleNode(threadStaticBlocksInfo.offsetOfThreadLocalStoragePointer, GTF_ICON_TLS_HDL);
+        tlsValue = gtNewIndir(TYP_I_IMPL, tlsValue, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
 
-    if (tlsIndexValue != 0)
-    {
-        dllRef = gtNewIconHandleNode(tlsIndexValue * TARGET_POINTER_SIZE, GTF_ICON_TLS_HDL);
+        if (dllRef != nullptr)
+        {
+            // Add the dllRef to produce thread local storage reference for coreclr
+            tlsValue = gtNewOperNode(GT_ADD, TYP_I_IMPL, tlsValue, dllRef);
+        }
+
+        // Base of coreclr's thread local storage
+        tlsValue = gtNewIndir(TYP_I_IMPL, tlsValue, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
     }
+    else if (TargetOS::IsMacOS)
+    {
+        // For OSX x64/arm64, we need to get the address of relevant __thread_vars section of
+        // the thread local variable `t_ThreadStatics`. Address of `tlv_get_address` is stored
+        // in this entry, which we dereference and invoke it, passing the __thread_vars address
+        // present in `threadVarsSection`.
+        //
+        // Code sequence to access thread local variable on osx/x64:
+        //
+        //      mov rdi, threadVarsSection
+        //      call     [rdi]
+        //
+        // Code sequence to access thread local variable on osx/arm64:
+        //
+        //      mov x0, threadVarsSection
+        //      mov x1, [x0]
+        //      blr x1
+        //
+        size_t   threadVarsSectionVal = (size_t)threadStaticBlocksInfo.threadVarsSection;
+        GenTree* tls_get_addr_val     = gtNewIconHandleNode(threadVarsSectionVal, GTF_ICON_FTN_ADDR);
+
+        tls_get_addr_val = gtNewIndir(TYP_I_IMPL, tls_get_addr_val, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
+
+        tlsValue                = gtNewIndCallNode(tls_get_addr_val, TYP_I_IMPL);
+        GenTreeCall* tlsRefCall = tlsValue->AsCall();
 
-    // Mark this ICON as a TLS_HDL, codegen will use FS:[cns] or GS:[cns]
-    GenTree* tlsRef = gtNewIconHandleNode(threadStaticBlocksInfo.offsetOfThreadLocalStoragePointer, GTF_ICON_TLS_HDL);
+        // This is a call which takes an argument.
+        // Populate and set the ABI appropriately.
+        assert(threadVarsSectionVal != 0);
+        GenTree* tlsArg = gtNewIconNode(threadVarsSectionVal, TYP_I_IMPL);
+        tlsRefCall->gtArgs.InsertAfterThisOrFirst(this, NewCallArg::Primitive(tlsArg));
 
-    tlsRef = gtNewIndir(TYP_I_IMPL, tlsRef, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
+        CallArg* arg0 = tlsRefCall->gtArgs.GetArgByIndex(0);
+        arg0->AbiInfo = CallArgABIInformation();
+        arg0->AbiInfo.SetRegNum(0, REG_ARG_0);
 
-    if (dllRef != nullptr)
+        tlsRefCall->gtFlags |= GTF_EXCEPT | (tls_get_addr_val->gtFlags & GTF_GLOB_EFFECT);
+    }
+    else if (TargetOS::IsUnix)
     {
-        // Add the dllRef to produce thread local storage reference for coreclr
-        tlsRef = gtNewOperNode(GT_ADD, TYP_I_IMPL, tlsRef, dllRef);
+#if defined(TARGET_AMD64)
+        // Code sequence to access thread local variable on linux/x64:
+        //
+        //      mov      rdi, 0x7FE5C418CD28  ; tlsIndexObject
+        //      mov      rax, 0x7FE5C47AFDB0  ; _tls_get_addr
+        //      call     rax
+        //
+        GenTree* tls_get_addr_val =
+            gtNewIconHandleNode((size_t)threadStaticBlocksInfo.tlsGetAddrFtnPtr, GTF_ICON_FTN_ADDR);
+        tlsValue                = gtNewIndCallNode(tls_get_addr_val, TYP_I_IMPL);
+        GenTreeCall* tlsRefCall = tlsValue->AsCall();
+
+        // This is an indirect call which takes an argument.
+        // Populate and set the ABI appropriately.
+        assert(threadStaticBlocksInfo.tlsIndexObject != 0);
+        GenTree* tlsArg = gtNewIconNode((size_t)threadStaticBlocksInfo.tlsIndexObject, TYP_I_IMPL);
+        tlsRefCall->gtArgs.InsertAfterThisOrFirst(this, NewCallArg::Primitive(tlsArg));
+
+        CallArg* arg0 = tlsRefCall->gtArgs.GetArgByIndex(0);
+        arg0->AbiInfo = CallArgABIInformation();
+        arg0->AbiInfo.SetRegNum(0, REG_ARG_0);
+
+        tlsRefCall->gtFlags |= GTF_EXCEPT | (tls_get_addr_val->gtFlags & GTF_GLOB_EFFECT);
+#ifdef UNIX_X86_ABI
+        tlsRefCall->gtFlags &= ~GTF_CALL_POP_ARGS;
+#endif // UNIX_X86_ABI
+#elif defined(TARGET_ARM64)
+        // Code sequence to access thread local variable on linux/arm64:
+        //
+        //      mrs xt, tpidr_elf0
+        //      mov xd, [xt+cns]
+        tlsValue = gtNewIconHandleNode(0, GTF_ICON_TLS_HDL);
+#else
+        assert(!"Unsupported scenario of optimizing TLS access on Linux Arm32/x86");
+#endif
     }
 
-    // Base of coreclr's thread local storage
-    GenTree* tlsValue = gtNewIndir(TYP_I_IMPL, tlsRef, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
-
     // Cache the tls value
-    unsigned tlsLclNum         = lvaGrabTemp(true DEBUGARG("TLS access"));
-    lvaTable[tlsLclNum].lvType = TYP_I_IMPL;
-    GenTree* tlsValueDef       = gtNewStoreLclVarNode(tlsLclNum, tlsValue);
-    GenTree* tlsLclValueUse    = gtNewLclVarNode(tlsLclNum);
+    tlsValueDef             = gtNewStoreLclVarNode(tlsLclNum, tlsValue);
+    GenTree* tlsLclValueUse = gtNewLclVarNode(tlsLclNum);
+
+    size_t offsetOfThreadStaticBlocksVal    = threadStaticBlocksInfo.offsetOfThreadStaticBlocks;
+    size_t offsetOfMaxThreadStaticBlocksVal = threadStaticBlocksInfo.offsetOfMaxThreadStaticBlocks;
 
     // Create tree for "maxThreadStaticBlocks = tls[offsetOfMaxThreadStaticBlocks]"
     GenTree* offsetOfMaxThreadStaticBlocks = gtNewIconNode(offsetOfMaxThreadStaticBlocksVal, TYP_I_IMPL);
     GenTree* maxThreadStaticBlocksRef =
         gtNewOperNode(GT_ADD, TYP_I_IMPL, gtCloneExpr(tlsLclValueUse), offsetOfMaxThreadStaticBlocks);
-    GenTree* maxThreadStaticBlocksValue =
-        gtNewIndir(TYP_INT, maxThreadStaticBlocksRef, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
+    maxThreadStaticBlocksValue = gtNewIndir(TYP_INT, maxThreadStaticBlocksRef, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
+
+    GenTree* threadStaticBlocksRef = gtNewOperNode(GT_ADD, TYP_I_IMPL, gtCloneExpr(tlsLclValueUse),
+                                                   gtNewIconNode(offsetOfThreadStaticBlocksVal, TYP_I_IMPL));
+    threadStaticBlocksValue = gtNewIndir(TYP_I_IMPL, threadStaticBlocksRef, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
 
     // Create tree for "if (maxThreadStaticBlocks < typeIndex)"
     GenTree* maxThreadStaticBlocksCond =
         gtNewOperNode(GT_LT, TYP_INT, maxThreadStaticBlocksValue, gtCloneExpr(typeThreadStaticBlockIndexValue));
     maxThreadStaticBlocksCond = gtNewOperNode(GT_JTRUE, TYP_VOID, maxThreadStaticBlocksCond);
 
-    // Create tree for "threadStaticBlockBase = tls[offsetOfThreadStaticBlocks]"
-    GenTree* offsetOfThreadStaticBlocks = gtNewIconNode(offsetOfThreadStaticBlocksVal, TYP_I_IMPL);
-    GenTree* threadStaticBlocksRef =
-        gtNewOperNode(GT_ADD, TYP_I_IMPL, gtCloneExpr(tlsLclValueUse), offsetOfThreadStaticBlocks);
-    GenTree* threadStaticBlocksValue =
-        gtNewIndir(TYP_I_IMPL, threadStaticBlocksRef, GTF_IND_NONFAULTING | GTF_IND_INVARIANT);
-
     // Create tree to "threadStaticBlockValue = threadStaticBlockBase[typeIndex]"
     typeThreadStaticBlockIndexValue = gtNewOperNode(GT_MUL, TYP_INT, gtCloneExpr(typeThreadStaticBlockIndexValue),
                                                     gtNewIconNode(TARGET_POINTER_SIZE, TYP_INT));

src/coreclr/jit/instrsarm64.h

@@ -1595,6 +1595,9 @@ INST1(isb,         "isb",          0,      IF_SI_0B,  0xD50330DF)
 INST1(dczva,       "dczva",        0,      IF_SR_1A,  0xD50B7420)
                                    //  dc      zva,Rt               SR_1A  1101010100001011 01110100001ttttt   D50B 7420   Rt
 
+INST1(mrs_tpid0,   "mrs",          0,      IF_SR_1A,  0xD53BD040)
+                                   //  mrs     Rt,tpidr_el0         SR_1A  1101010100111011 11010000010ttttt   D53B D040   Rt, tpidr_el0
+
 INST1(umov,        "umov",         0,      IF_DV_2B,  0x0E003C00)
                                    //  umov    Rd,Vn[]              DV_2B  0Q001110000iiiii 001111nnnnnddddd   0E00 3C00   Rd,Vn[]