From 8f0ac5d2041d0c577607e2f778f2fbca1f7d732e Mon Sep 17 00:00:00 2001 From: Koundinya Veluri Date: Tue, 26 Sep 2017 13:14:53 -0700 Subject: [PATCH] =?UTF-8?q?=EF=BB=BFRemove=20Monitor=20asm=20helpers=20(#1?= =?UTF-8?q?4146)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Removed asm helpers on Windows and used portable C++ helpers instead - Rearranged fast path code to improve them a bit and match the asm more closely Perf: - The asm helpers are a bit faster. The code generated for the portable helpers is almost the same now, the remaining differences are: - There were some layout issues where hot paths were in the wrong place and return paths were not cloned. Instrumenting some of the tests below with PGO on x64 resolved all of the layout issues. I couldn't get PGO instrumentation to work on x86 but I imagine it would be the same there. - Register usage - x64: All of the Enter functions are using one or two (TryEnter is using two) callee-saved registers for no apparent reason, forcing them to be saved and restored. r10 and r11 seem to be available but they're not being used. - x86: Similarly to x64, the compiled functions are pushing and popping 2-3 additional registers in the hottest fast paths. - I believe this is the main remaining gap and PGO is not helping with this - On Linux, perf is >= before for the most part - Perf tests used for below are updated in PR https://github.com/dotnet/coreclr/pull/13670 --- src/inc/jithelpers.h | 7 - src/pal/inc/pal.h | 38 +- src/vm/amd64/JitHelpers_InlineGetThread.asm | 1005 ------------------ src/vm/amd64/JitHelpers_Slow.asm | 888 ---------------- src/vm/amd64/asmconstants.h | 33 - src/vm/amd64/cgencpu.h | 10 - src/vm/i386/asmconstants.h | 76 -- src/vm/i386/cgencpu.h | 18 - src/vm/i386/jithelp.asm | 1052 ------------------- src/vm/jithelpers.cpp | 319 +----- src/vm/jitinterfacegen.cpp | 42 - src/vm/syncblk.cpp | 84 ++ src/vm/syncblk.h | 10 +- src/vm/syncblk.inl | 227 ++-- 14 files changed, 212 insertions(+), 3597 deletions(-) diff --git a/src/inc/jithelpers.h b/src/inc/jithelpers.h index b45948ac59ef..43b75293aea4 100644 --- a/src/inc/jithelpers.h +++ b/src/inc/jithelpers.h @@ -134,17 +134,10 @@ JITHELPER(CORINFO_HELP_ENDCATCH, JIT_EndCatch, CORINFO_HELP_SIG_CANNOT_USE_ALIGN_STUB) #endif -#ifdef _TARGET_AMD64_ - DYNAMICJITHELPER(CORINFO_HELP_MON_ENTER, JIT_MonEnterWorker, CORINFO_HELP_SIG_REG_ONLY) - DYNAMICJITHELPER(CORINFO_HELP_MON_EXIT, JIT_MonExitWorker, CORINFO_HELP_SIG_REG_ONLY) - DYNAMICJITHELPER(CORINFO_HELP_MON_ENTER_STATIC, JIT_MonEnterStatic,CORINFO_HELP_SIG_REG_ONLY) - DYNAMICJITHELPER(CORINFO_HELP_MON_EXIT_STATIC, JIT_MonExitStatic,CORINFO_HELP_SIG_REG_ONLY) -#else JITHELPER(CORINFO_HELP_MON_ENTER, JIT_MonEnterWorker, CORINFO_HELP_SIG_REG_ONLY) JITHELPER(CORINFO_HELP_MON_EXIT, JIT_MonExitWorker, CORINFO_HELP_SIG_REG_ONLY) JITHELPER(CORINFO_HELP_MON_ENTER_STATIC, JIT_MonEnterStatic,CORINFO_HELP_SIG_REG_ONLY) JITHELPER(CORINFO_HELP_MON_EXIT_STATIC, JIT_MonExitStatic,CORINFO_HELP_SIG_REG_ONLY) -#endif JITHELPER(CORINFO_HELP_GETCLASSFROMMETHODPARAM, JIT_GetClassFromMethodParam, CORINFO_HELP_SIG_REG_ONLY) JITHELPER(CORINFO_HELP_GETSYNCFROMCLASSHANDLE, JIT_GetSyncFromClassHandle, CORINFO_HELP_SIG_REG_ONLY) diff --git a/src/pal/inc/pal.h b/src/pal/inc/pal.h index 4ae2187b69f3..1f611d092224 100644 --- a/src/pal/inc/pal.h +++ b/src/pal/inc/pal.h @@ -4202,6 +4202,9 @@ InterlockedDecrement( return __sync_sub_and_fetch(lpAddend, (LONG)1); } +#define InterlockedDecrementAcquire InterlockedDecrement +#define InterlockedDecrementRelease InterlockedDecrement + EXTERN_C PALIMPORT inline @@ -4297,39 +4300,8 @@ InterlockedCompareExchange( Exchange /* The value to be stored */); } -EXTERN_C -PALIMPORT -inline -LONG -PALAPI -InterlockedCompareExchangeAcquire( - IN OUT LONG volatile *Destination, - IN LONG Exchange, - IN LONG Comperand) -{ - // TODO: implement the version with only the acquire semantics - return __sync_val_compare_and_swap( - Destination, /* The pointer to a variable whose value is to be compared with. */ - Comperand, /* The value to be compared */ - Exchange /* The value to be stored */); -} - -EXTERN_C -PALIMPORT -inline -LONG -PALAPI -InterlockedCompareExchangeRelease( - IN OUT LONG volatile *Destination, - IN LONG Exchange, - IN LONG Comperand) -{ - // TODO: implement the version with only the release semantics - return __sync_val_compare_and_swap( - Destination, /* The pointer to a variable whose value is to be compared with. */ - Comperand, /* The value to be compared */ - Exchange /* The value to be stored */); -} +#define InterlockedCompareExchangeAcquire InterlockedCompareExchange +#define InterlockedCompareExchangeRelease InterlockedCompareExchange // See the 32-bit variant in interlock2.s EXTERN_C diff --git a/src/vm/amd64/JitHelpers_InlineGetThread.asm b/src/vm/amd64/JitHelpers_InlineGetThread.asm index 022ec675dfbf..40d63bf7298a 100644 --- a/src/vm/amd64/JitHelpers_InlineGetThread.asm +++ b/src/vm/amd64/JitHelpers_InlineGetThread.asm @@ -305,1010 +305,5 @@ endif ; _DEBUG LEAF_END JIT_NewArr1OBJ_MP_InlineGetThread, _TEXT -MON_ENTER_STACK_SIZE equ 00000020h -MON_EXIT_STACK_SIZE equ 00000068h - -ifdef MON_DEBUG -ifdef TRACK_SYNC -MON_ENTER_STACK_SIZE_INLINEGETTHREAD equ 00000020h -MON_EXIT_STACK_SIZE_INLINEGETTHREAD equ 00000068h -endif -endif - -BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX equ 08000000h ; syncblk.h -BIT_SBLK_IS_HASHCODE equ 04000000h ; syncblk.h -BIT_SBLK_SPIN_LOCK equ 10000000h ; syncblk.h - -SBLK_MASK_LOCK_THREADID equ 000003FFh ; syncblk.h -SBLK_LOCK_RECLEVEL_INC equ 00000400h ; syncblk.h -SBLK_MASK_LOCK_RECLEVEL equ 0000FC00h ; syncblk.h - -MASK_SYNCBLOCKINDEX equ 03FFFFFFh ; syncblk.h -STATE_CHECK equ 0FFFFFFFEh - -MT_CTX_PROXY_FLAG equ 10000000h - -g_pSyncTable equ ?g_pSyncTable@@3PEAVSyncTableEntry@@EA -g_SystemInfo equ ?g_SystemInfo@@3U_SYSTEM_INFO@@A -g_SpinConstants equ ?g_SpinConstants@@3USpinConstants@@A - -extern g_pSyncTable:QWORD -extern g_SystemInfo:QWORD -extern g_SpinConstants:QWORD - -; JITutil_MonEnterWorker(Object* obj, BYTE* pbLockTaken) -extern JITutil_MonEnterWorker:proc -; JITutil_MonTryEnter(Object* obj, INT32 timeout, BYTE* pbLockTaken) -extern JITutil_MonTryEnter:proc -; JITutil_MonExitWorker(Object* obj, BYTE* pbLockTaken) -extern JITutil_MonExitWorker:proc -; JITutil_MonSignal(AwareLock* lock, BYTE* pbLockTaken) -extern JITutil_MonSignal:proc -; JITutil_MonContention(AwareLock* lock, BYTE* pbLockTaken) -extern JITutil_MonContention:proc - -ifdef _DEBUG -MON_DEBUG equ 1 -endif - -ifdef MON_DEBUG -ifdef TRACK_SYNC -extern EnterSyncHelper:proc -extern LeaveSyncHelper:proc -endif -endif - - -MON_ENTER_EPILOG_ADJUST_STACK macro -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MON_ENTER_STACK_SIZE_INLINEGETTHREAD -endif -endif - endm - - -MON_ENTER_RETURN_SUCCESS macro - ; This is sensitive to the potential that pbLockTaken is NULL - test rsi, rsi - jz @F - mov byte ptr [rsi], 1 - @@: - MON_ENTER_EPILOG_ADJUST_STACK - pop rsi - ret - - endm - - -; The worker versions of these functions are smart about the potential for pbLockTaken -; to be NULL, and if it is then they treat it as if they don't have a state variable. -; This is because when locking is not inserted by the JIT (instead by explicit calls to -; Monitor.Enter() and Monitor.Exit()) we will call these guys. -; -; This is a frameless helper for entering a monitor on a object. -; The object is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; -; EXTERN_C void JIT_MonEnterWorker_InlineGetThread(Object* obj, /*OUT*/ BYTE* pbLockTaken) -JIT_HELPER_MONITOR_THUNK JIT_MonEnter, _TEXT -NESTED_ENTRY JIT_MonEnterWorker_InlineGetThread, _TEXT - push_nonvol_reg rsi -ifdef MON_DEBUG -ifdef TRACK_SYNC - alloc_stack MON_ENTER_STACK_SIZE_INLINEGETTHREAD - - save_reg_postrsp rcx, MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 10h + 0h - save_reg_postrsp rdx, MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 10h + 8h - save_reg_postrsp r8, MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 10h + 10h - save_reg_postrsp r9, MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 10h + 18h -endif -endif - END_PROLOGUE - - ; Put pbLockTaken in rsi, this can be null - mov rsi, rdx - - ; Check if the instance is NULL - test rcx, rcx - jz FramedLockHelper - - PATCHABLE_INLINE_GETTHREAD r11, JIT_MonEnterWorker_InlineGetThread_GetThread_PatchLabel - - ; Initialize delay value for retry with exponential backoff - mov r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwInitialDuration] - - ; Check if we can abort here - mov eax, dword ptr [r11 + OFFSETOF__Thread__m_State] - and eax, THREAD_CATCHATSAFEPOINT_BITS - ; Go through the slow code path to initiate ThreadAbort - jnz FramedLockHelper - - ; r8 will hold the syncblockindex address - lea r8, [rcx - OFFSETOF__ObjHeader__SyncBlkIndex] - - RetryThinLock: - ; Fetch the syncblock dword - mov eax, dword ptr [r8] - - ; Check whether we have the "thin lock" layout, the lock is free and the spin lock bit is not set - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK + SBLK_MASK_LOCK_THREADID + SBLK_MASK_LOCK_RECLEVEL - jnz NeedMoreTests - - ; Everything is fine - get the thread id to store in the lock - mov edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - - ; If the thread id is too large, we need a syncblock for sure - cmp edx, SBLK_MASK_LOCK_THREADID - ja FramedLockHelper - - ; We want to store a new value with the current thread id set in the low 10 bits - or edx, eax - lock cmpxchg dword ptr [r8], edx - jnz PrepareToWaitThinLock - - ; Everything went fine and we're done - add dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - - ; Done, leave and set pbLockTaken if we have it - MON_ENTER_RETURN_SUCCESS - - NeedMoreTests: - ; OK, not the simple case, find out which case it is - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX - jnz HaveHashOrSyncBlockIndex - - ; The header is transitioning or the lock, treat this as if the lock was taken - test eax, BIT_SBLK_SPIN_LOCK - jnz PrepareToWaitThinLock - - ; Here we know we have the "thin lock" layout, but the lock is not free. - ; It could still be the recursion case, compare the thread id to check - mov edx, eax - and edx, SBLK_MASK_LOCK_THREADID - cmp edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - jne PrepareToWaitThinLock - - ; Ok, the thread id matches, it's the recursion case. - ; Bump up the recursion level and check for overflow - lea edx, [eax + SBLK_LOCK_RECLEVEL_INC] - test edx, SBLK_MASK_LOCK_RECLEVEL - jz FramedLockHelper - - ; Try to put the new recursion level back. If the header was changed in the meantime - ; we need a full retry, because the layout could have changed - lock cmpxchg dword ptr [r8], edx - jnz RetryHelperThinLock - - ; Done, leave and set pbLockTaken if we have it - MON_ENTER_RETURN_SUCCESS - - PrepareToWaitThinLock: - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr [g_SystemInfo + OFFSETOF__g_SystemInfo__dwNumberOfProcessors], 1 - jle FramedLockHelper - - ; Exponential backoff; delay by approximately 2*r10 clock cycles - mov eax, r10d - delayLoopThinLock: - pause ; indicate to the CPU that we are spin waiting - sub eax, 1 - jnz delayLoopThinLock - - ; Next time, wait a factor longer - imul r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwBackoffFactor] - - cmp r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwMaximumDuration] - jle RetryHelperThinLock - - jmp FramedLockHelper - - RetryHelperThinLock: - jmp RetryThinLock - - HaveHashOrSyncBlockIndex: - ; If we have a hash code already, we need to create a sync block - test eax, BIT_SBLK_IS_HASHCODE - jnz FramedLockHelper - - ; OK, we have a sync block index, just and out the top bits and grab the synblock index - and eax, MASK_SYNCBLOCKINDEX - - ; Get the sync block pointer - mov rdx, qword ptr [g_pSyncTable] - shl eax, 4h - mov rdx, [rdx + rax + OFFSETOF__SyncTableEntry__m_SyncBlock] - - ; Check if the sync block has been allocated - test rdx, rdx - jz FramedLockHelper - - ; Get a pointer to the lock object - lea rdx, [rdx + OFFSETOF__SyncBlock__m_Monitor] - - ; Attempt to acquire the lock - RetrySyncBlock: - mov eax, dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld] - test eax, eax - jne HaveWaiters - - ; Common case, lock isn't held and there are no waiters. Attempt to - ; gain ownership ourselves - xor ecx, ecx - inc ecx - - lock cmpxchg dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld], ecx - jnz RetryHelperSyncBlock - - ; Success. Save the thread object in the lock and increment the use count - mov qword ptr [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - add dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - add dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rcx, [rsp + MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 8h] ; return address - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - call EnterSyncHelper -endif -endif - - ; Done, leave and set pbLockTaken if we have it - MON_ENTER_RETURN_SUCCESS - - ; It's possible to get here with waiters by no lock held, but in this - ; case a signal is about to be fired which will wake up the waiter. So - ; for fairness sake we should wait too. - ; Check first for recur11ve lock attempts on the same thread. - HaveWaiters: - ; Is mutex already owned by current thread? - cmp [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - jne PrepareToWait - - ; Yes, bump our use count. - add dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rcx, [rsp + MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 8h] ; return address - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - call EnterSyncHelper -endif -endif - ; Done, leave and set pbLockTaken if we have it - MON_ENTER_RETURN_SUCCESS - - PrepareToWait: - ; If we are on a MP system we try spinning for a certain number of iterations - cmp dword ptr [g_SystemInfo + OFFSETOF__g_SystemInfo__dwNumberOfProcessors], 1 - jle HaveWaiters1 - - ; Exponential backoff: delay by approximately 2*r10 clock cycles - mov eax, r10d - delayLoop: - pause ; indicate to the CPU that we are spin waiting - sub eax, 1 - jnz delayLoop - - ; Next time, wait a factor longer - imul r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwBackoffFactor] - - cmp r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwMaximumDuration] - jle RetrySyncBlock - - HaveWaiters1: - mov rcx, rdx - mov rdx, rsi - MON_ENTER_EPILOG_ADJUST_STACK - pop rsi - ; void JITutil_MonContention(AwareLock* lock, BYTE* pbLockTaken) - jmp JITutil_MonContention - - RetryHelperSyncBlock: - jmp RetrySyncBlock - - FramedLockHelper: - mov rdx, rsi - MON_ENTER_EPILOG_ADJUST_STACK - pop rsi - ; void JITutil_MonEnterWorker(Object* obj, BYTE* pbLockTaken) - jmp JITutil_MonEnterWorker - -NESTED_END JIT_MonEnterWorker_InlineGetThread, _TEXT - - -MON_EXIT_EPILOG_ADJUST_STACK macro -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MON_EXIT_STACK_SIZE_INLINEGETTHREAD -endif -endif - endm - -MON_EXIT_RETURN_SUCCESS macro - ; This is sensitive to the potential that pbLockTaken is null - test r10, r10 - jz @F - mov byte ptr [r10], 0 - @@: - MON_EXIT_EPILOG_ADJUST_STACK - ret - - endm - - -; The worker versions of these functions are smart about the potential for pbLockTaken -; to be NULL, and if it is then they treat it as if they don't have a state variable. -; This is because when locking is not inserted by the JIT (instead by explicit calls to -; Monitor.Enter() and Monitor.Exit()) we will call these guys. -; -; This is a frameless helper for exiting a monitor on a object. -; The object is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; -; void JIT_MonExitWorker_InlineGetThread(Object* obj, BYTE* pbLockTaken) -JIT_HELPER_MONITOR_THUNK JIT_MonExit, _TEXT -NESTED_ENTRY JIT_MonExitWorker_InlineGetThread, _TEXT - .savereg rcx, 0 -ifdef MON_DEBUG -ifdef TRACK_SYNC - alloc_stack MON_EXIT_STACK_SIZE_INLINEGETTHREAD - - save_reg_postrsp rcx, MON_EXIT_STACK_SIZE_INLINEGETTHREAD + 8h + 0h - save_reg_postrsp rdx, MON_EXIT_STACK_SIZE_INLINEGETTHREAD + 8h + 8h - save_reg_postrsp r8, MON_EXIT_STACK_SIZE_INLINEGETTHREAD + 8h + 10h - save_reg_postrsp r9, MON_EXIT_STACK_SIZE_INLINEGETTHREAD + 8h + 18h -endif -endif - END_PROLOGUE - - ; pbLockTaken is stored in r10, this can be null - mov r10, rdx - - ; if pbLockTaken is NULL then we got here without a state variable, avoid the - ; next comparison in that case as it will AV - test rdx, rdx - jz Null_pbLockTaken - - ; If the lock wasn't taken then we bail quickly without doing anything - cmp byte ptr [rdx], 0 - je LockNotTaken - - Null_pbLockTaken: - ; Check is the instance is null - test rcx, rcx - jz FramedLockHelper - - PATCHABLE_INLINE_GETTHREAD r11, JIT_MonExitWorker_InlineGetThread_GetThread_PatchLabel - - ; r8 will hold the syncblockindex address - lea r8, [rcx - OFFSETOF__ObjHeader__SyncBlkIndex] - - RetryThinLock: - ; Fetch the syncblock dword - mov eax, dword ptr [r8] - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK - jnz NeedMoreTests - - ; Ok, we have a "thin lock" layout - check whether the thread id matches - mov edx, eax - and edx, SBLK_MASK_LOCK_THREADID - cmp edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - jne FramedLockHelper - - ; check the recursion level - test eax, SBLK_MASK_LOCK_RECLEVEL - jne DecRecursionLevel - - ; It's zero -- we're leaving the lock. - ; So try to put back a zero thread id. - ; edx and eax match in the thread id bits, and edx is zero else where, so the xor is sufficient - xor edx, eax - lock cmpxchg dword ptr [r8], edx - jnz RetryThinLockHelper1 ; forward jump to avoid mispredict on success - - ; Dec the dwLockCount on the thread - sub dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - - ; Done, leave and set pbLockTaken if we have it - MON_EXIT_RETURN_SUCCESS - - RetryThinLockHelper1: - jmp RetryThinLock - - DecRecursionLevel: - lea edx, [eax - SBLK_LOCK_RECLEVEL_INC] - lock cmpxchg dword ptr [r8], edx - jnz RetryThinLockHelper2 ; forward jump to avoid mispredict on success - - ; We're done, leave and set pbLockTaken if we have it - MON_EXIT_RETURN_SUCCESS - - RetryThinLockHelper2: - jmp RetryThinLock - - NeedMoreTests: - ; Forward all special cases to the slow helper - test eax, BIT_SBLK_IS_HASHCODE + BIT_SBLK_SPIN_LOCK - jnz FramedLockHelper - - ; Get the sync block index and use it to compute the sync block pointer - mov rdx, qword ptr [g_pSyncTable] - and eax, MASK_SYNCBLOCKINDEX - shl eax, 4 - mov rdx, [rdx + rax + OFFSETOF__SyncTableEntry__m_SyncBlock] - - ; Was there a sync block? - test rdx, rdx - jz FramedLockHelper - - ; Get a pointer to the lock object. - lea rdx, [rdx + OFFSETOF__SyncBlock__m_Monitor] - - ; Check if the lock is held. - cmp qword ptr [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - jne FramedLockHelper - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov [rsp + 28h], rcx - mov [rsp + 30h], rdx - mov [rsp + 38h], r10 - mov [rsp + 40h], r11 - - mov rcx, [rsp + MON_EXIT_STACK_SIZE_INLINEGETTHREAD ] ; return address - ; void LeaveSyncHelper(UINT_PTR caller, AwareLock* lock) - call LeaveSyncHelper - - mov rcx, [rsp + 28h] - mov rdx, [rsp + 30h] - mov r10, [rsp + 38h] - mov r11, [rsp + 40h] -endif -endif - - ; Reduce our recursion count - sub dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - jz LastRecursion - - ; Done, leave and set pbLockTaken if we have it - MON_EXIT_RETURN_SUCCESS - - RetryHelperThinLock: - jmp RetryThinLock - - FramedLockHelper: - mov rdx, r10 - MON_EXIT_EPILOG_ADJUST_STACK - ; void JITutil_MonExitWorker(Object* obj, BYTE* pbLockTaken) - jmp JITutil_MonExitWorker - - LastRecursion: -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rax, [rdx + OFFSETOF__AwareLock__m_HoldingThread] -endif -endif - - sub dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - mov qword ptr [rdx + OFFSETOF__AwareLock__m_HoldingThread], 0 - - Retry: - mov eax, dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld] - lea r9d, [eax - 1] - lock cmpxchg dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld], r9d - jne RetryHelper - - test eax, STATE_CHECK - jne MustSignal - - ; Done, leave and set pbLockTaken if we have it - MON_EXIT_RETURN_SUCCESS - - MustSignal: - mov rcx, rdx - mov rdx, r10 - MON_EXIT_EPILOG_ADJUST_STACK - ; void JITutil_MonSignal(AwareLock* lock, BYTE* pbLockTaken) - jmp JITutil_MonSignal - - RetryHelper: - jmp Retry - - LockNotTaken: - MON_EXIT_EPILOG_ADJUST_STACK - REPRET -NESTED_END JIT_MonExitWorker_InlineGetThread, _TEXT - - -; This is a frameless helper for trying to enter a monitor on a object. -; The object is in ARGUMENT_REG1 and a timeout in ARGUMENT_REG2. This tries the -; normal case (no object allocation) in line and calls a framed helper for the -; other cases. -; -; void JIT_MonTryEnter_InlineGetThread(Object* obj, INT32 timeOut, BYTE* pbLockTaken) -NESTED_ENTRY JIT_MonTryEnter_InlineGetThread, _TEXT - ; save rcx, rdx (timeout) in the shadow space - .savereg rcx, 8h - mov [rsp + 8h], rcx - .savereg rdx, 10h - mov [rsp + 10h], rdx -ifdef MON_DEBUG -ifdef TRACK_SYNC - alloc_stack MON_ENTER_STACK_SIZE_INLINEGETTHREAD - -; rcx has already been saved -; save_reg_postrsp rcx, MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 8h + 0h -; rdx has already been saved -; save_reg_postrsp rdx, MON_ENTER_STACK_SIZE + 8h + 8h - save_reg_postrsp r8, MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 8h + 10h - save_reg_postrsp r9, MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 8h + 18h -endif -endif - END_PROLOGUE - - ; Check if the instance is NULL - test rcx, rcx - jz FramedLockHelper - - ; Check if the timeout looks valid - cmp edx, -1 - jl FramedLockHelper - - PATCHABLE_INLINE_GETTHREAD r11, JIT_MonTryEnter_GetThread_PatchLabel - - ; Initialize delay value for retry with exponential backoff - mov r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwInitialDuration] - - ; Check if we can abort here - mov eax, dword ptr [r11 + OFFSETOF__Thread__m_State] - and eax, THREAD_CATCHATSAFEPOINT_BITS - ; Go through the slow code path to initiate THreadAbort - jnz FramedLockHelper - - ; r9 will hold the syncblockindex address - lea r9, [rcx - OFFSETOF__ObjHeader__SyncBlkIndex] - - RetryThinLock: - ; Fetch the syncblock dword - mov eax, dword ptr [r9] - - ; Check whether we have the "thin lock" layout, the lock is free and the spin lock bit is not set - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK + SBLK_MASK_LOCK_THREADID + SBLK_MASK_LOCK_RECLEVEL - jne NeedMoreTests - - ; Everything is fine - get the thread id to store in the lock - mov edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - - ; If the thread id is too large, we need a syncblock for sure - cmp edx, SBLK_MASK_LOCK_THREADID - ja FramedLockHelper - - ; We want to store a new value with the current thread id set in the low 10 bits - or edx, eax - lock cmpxchg dword ptr [r9], edx - jnz RetryHelperThinLock - - ; Got the lock, everything is fine - add dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - ; Return TRUE - mov byte ptr [r8], 1 -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MON_ENTER_STACK_SIZE_INLINEGETTHREAD -endif -endif - ret - - NeedMoreTests: - ; OK, not the simple case, find out which case it is - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX - jnz HaveHashOrSyncBlockIndex - - ; The header is transitioning or the lock - test eax, BIT_SBLK_SPIN_LOCK - jnz RetryHelperThinLock - - ; Here we know we have the "thin lock" layout, but the lock is not free. - ; It could still be the recursion case, compare the thread id to check - mov edx, eax - and edx, SBLK_MASK_LOCK_THREADID - cmp edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - jne PrepareToWaitThinLock - - ; Ok, the thread id matches, it's the recursion case. - ; Dump up the recursion level and check for overflow - lea edx, [eax + SBLK_LOCK_RECLEVEL_INC] - test edx, SBLK_MASK_LOCK_RECLEVEL - jz FramedLockHelper - - ; Try to put the new recursion level back. If the header was changed in the meantime - ; we need a full retry, because the layout could have changed - lock cmpxchg dword ptr [r9], edx - jnz RetryHelperThinLock - - ; Everything went fine and we're done, return TRUE - mov byte ptr [r8], 1 -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MON_ENTER_STACK_SIZE_INLINEGETTHREAD -endif -endif - ret - - PrepareToWaitThinLock: - ; Return failure if timeout is zero - cmp dword ptr [rsp + 10h], 0 - je TimeoutZero - - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr [g_SystemInfo + OFFSETOF__g_SystemInfo__dwNumberOfProcessors], 1 - jle FramedLockHelper - - ; Exponential backoff; delay by approximately 2*r10d clock cycles - mov eax, r10d - DelayLoopThinLock: - pause ; indicate to the CPU that we are spin waiting - sub eax, 1 - jnz DelayLoopThinLock - - ; Next time, wait a factor longer - imul r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwBackoffFactor] - - cmp r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwMaximumDuration] - jle RetryHelperThinLock - - jmp FramedLockHelper - - RetryHelperThinLock: - jmp RetryThinLock - - TimeoutZero: - ; Did not acquire, return FALSE - mov byte ptr [r8], 0 -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MON_ENTER_STACK_SIZE_INLINEGETTHREAD -endif -endif - ret - - HaveHashOrSyncBlockIndex: - ; If we have a hash code already, we need to create a sync block - test eax, BIT_SBLK_IS_HASHCODE - jnz FramedLockHelper - - ; OK, we have a sync block index, just and out the top bits and grab the synblock index - and eax, MASK_SYNCBLOCKINDEX - - ; Get the sync block pointer - mov rdx, qword ptr [g_pSyncTable] - shl eax, 4 - mov rdx, [rdx + rax + OFFSETOF__SyncTableEntry__m_SyncBlock] - - ; Check if the sync block has been allocated - test rdx, rdx - jz FramedLockHelper - - ; Get a pointer to the lock object - lea rdx, [rdx + OFFSETOF__SyncBlock__m_Monitor] - - RetrySyncBlock: - ; Attempt to acuire the lock - mov eax, dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld] - test eax, eax - jne HaveWaiters - - ; Common case, lock isn't held and there are no waiters. Attempt to - ; gain ownership ourselves - xor ecx, ecx - inc ecx - lock cmpxchg dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld], ecx - jnz RetryHelperSyncBlock - - ; Success. Save the thread object in the lock and increment the use count - mov qword ptr [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - add dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - add dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rcx, [rsp + MON_ENTER_STACK_SIZE_INLINEGETTHREAD] ; return address - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - call EnterSyncHelper -endif -endif - - ; Return TRUE - mov byte ptr [r8], 1 -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MON_ENTER_STACK_SIZE_INLINEGETTHREAD -endif -endif - ret - - ; It's possible to get here with waiters by no lock held, but in this - ; case a signal is about to be fired which will wake up the waiter. So - ; for fairness sake we should wait too. - ; Check first for recur11ve lock attempts on the same thread. - HaveWaiters: - ; Is mutex already owned by current thread? - cmp [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - jne PrepareToWait - - ; Yes, bump our use count. - add dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rcx, [rsp + MON_ENTER_STACK_SIZE_INLINEGETTHREAD] ; return address - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - call EnterSyncHelper -endif -endif - - ; Return TRUE - mov byte ptr [r8], 1 -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MON_ENTER_STACK_SIZE_INLINEGETTHREAD -endif -endif - ret - - PrepareToWait: - ; Return failure if timeout is zero - cmp dword ptr [rsp + 10h], 0 -ifdef MON_DEBUG -ifdef TRACK_SYNC - ; if we are using the _DEBUG stuff then rsp has been adjusted - ; so compare the value at the adjusted position - ; there's really little harm in the extra stack read - cmp dword ptr [rsp + MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 10h] -endif -endif - je TimeoutZero - - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr [g_SystemInfo + OFFSETOF__g_SystemInfo__dwNumberOfProcessors], 1 - jle Block - - ; Exponential backoff; delay by approximately 2*r10d clock cycles - mov eax, r10d - DelayLoop: - pause ; indicate to the CPU that we are spin waiting - sub eax, 1 - jnz DelayLoop - - ; Next time, wait a factor longer - imul r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwBackoffFactor] - - cmp r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwMaximumDuration] - jle RetrySyncBlock - - jmp Block - - RetryHelperSyncBlock: - jmp RetrySyncBlock - - Block: - ; In the Block case we've trashed RCX, restore it - mov rcx, [rsp + 8h] -ifdef MON_DEBUG -ifdef TRACK_SYNC - ; if we're tracking this stuff then rcx is at a different offset to RSP, we just - ; overwrite the wrong value which we just got... this is for debug purposes only - ; so there's really no performance issue here - mov rcx, [rsp + MON_ENTER_STACK_SIZE_INLINEGETTHREAD + 8h] -endif -endif - FramedLockHelper: -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MON_ENTER_STACK_SIZE_INLINEGETTHREAD -endif -endif - mov rdx, [rsp + 10h] - ; void JITutil_MonTryEnter(Object* obj, INT32 timeout) - jmp JITutil_MonTryEnter - -NESTED_END JIT_MonTryEnter_InlineGetThread, _TEXT - - -MON_ENTER_STATIC_RETURN_SUCCESS macro - ; pbLockTaken is never null for static helpers - test rdx, rdx - mov byte ptr [rdx], 1 - REPRET - - endm - -MON_EXIT_STATIC_RETURN_SUCCESS macro - ; pbLockTaken is never null for static helpers - mov byte ptr [rdx], 0 - REPRET - - endm - - -; This is a frameless helper for entering a static monitor on a class. -; The methoddesc is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; -; void JIT_MonEnterStatic_InlineGetThread(AwareLock *lock, BYTE *pbLockTaken) -NESTED_ENTRY JIT_MonEnterStatic_InlineGetThread, _TEXT - .savereg rcx, 0 -ifdef MON_DEBUG -ifdef TRACK_SYNC - alloc_stack MIN_SIZE - save_reg_postrsp rcx, MIN_SIZE + 8h + 0h -endif -endif - END_PROLOGUE - - ; Attempt to acquire the lock - Retry: - mov eax, dword ptr [rcx + OFFSETOF__AwareLock__m_MonitorHeld] - test eax, eax - jne HaveWaiters - - ; Common case; lock isn't held and there are no waiters. Attempt to - ; gain ownership by ourselves. - mov r10d, 1 - - lock cmpxchg dword ptr [rcx + OFFSETOF__AwareLock__m_MonitorHeld], r10d - jnz RetryHelper - - PATCHABLE_INLINE_GETTHREAD rax, JIT_MonEnterStaticWorker_InlineGetThread_GetThread_PatchLabel_1 - - mov qword ptr [rcx + OFFSETOF__AwareLock__m_HoldingThread], rax - add dword ptr [rcx + OFFSETOF__AwareLock__m_Recursion], 1 - add dword ptr [rax + OFFSETOF__Thread__m_dwLockCount], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rdx, rcx - mov rcx, [rsp] - add rsp, MIN_SIZE - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - jmp EnterSyncHelper -endif -endif - MON_ENTER_STATIC_RETURN_SUCCESS - - ; It's possible to get here with waiters by with no lock held, in this - ; case a signal is about to be fired which will wake up a waiter. So - ; for fairness sake we should wait too. - ; Check first for recursive lock attempts on the same thread. - HaveWaiters: - PATCHABLE_INLINE_GETTHREAD rax, JIT_MonEnterStaticWorker_InlineGetThread_GetThread_PatchLabel_2 - - ; Is mutex alread owned by current thread? - cmp [rcx + OFFSETOF__AwareLock__m_HoldingThread], rax - jne PrepareToWait - - ; Yes, bump our use count. - add dword ptr [rcx + OFFSETOF__AwareLock__m_Recursion], 1 -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rdx, rcx - mov rcx, [rsp + MIN_SIZE] - add rsp, MIN_SIZE - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - jmp EnterSyncHelper -endif -endif - ret - - PrepareToWait: -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MIN_SIZE -endif -endif - ; void JITutil_MonContention(AwareLock* obj, BYTE* pbLockTaken) - jmp JITutil_MonContention - - RetryHelper: - jmp Retry -NESTED_END JIT_MonEnterStatic_InlineGetThread, _TEXT - -; A frameless helper for exiting a static monitor on a class. -; The methoddesc is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; -; void JIT_MonExitStatic_InlineGetThread(AwareLock *lock, BYTE *pbLockTaken) -NESTED_ENTRY JIT_MonExitStatic_InlineGetThread, _TEXT - .savereg rcx, 0 -ifdef MON_DEBUG -ifdef TRACK_SYNC - alloc_stack MIN_SIZE - save_reg_postrsp rcx, MIN_SIZE + 8h + 0h -endif -endif - END_PROLOGUE - -ifdef MON_DEBUG -ifdef TRACK_SYNC - push rsi - push rdi - mov rsi, rcx - mov rdi, rdx - mov rdx, [rsp + 8] - call LeaveSyncHelper - mov rcx, rsi - mov rdx, rdi - pop rdi - pop rsi -endif -endif - PATCHABLE_INLINE_GETTHREAD rax, JIT_MonExitStaticWorker_InlineGetThread_GetThread_PatchLabel - - ; Check if lock is held - cmp [rcx + OFFSETOF__AwareLock__m_HoldingThread], rax - jne LockError - - ; Reduce our recursion count - sub dword ptr [rcx + OFFSETOF__AwareLock__m_Recursion], 1 - jz LastRecursion - -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MIN_SIZE - ret -endif -endif - REPRET - - ; This is the last count we held on this lock, so release the lock - LastRecursion: - ; Thead* is in rax - sub dword ptr [rax + OFFSETOF__Thread__m_dwLockCount], 1 - mov qword ptr [rcx + OFFSETOF__AwareLock__m_HoldingThread], 0 - - Retry: - mov eax, dword ptr [rcx + OFFSETOF__AwareLock__m_MonitorHeld] - lea r10d, [eax - 1] - lock cmpxchg dword ptr [rcx + OFFSETOF__AwareLock__m_MonitorHeld], r10d - jne RetryHelper - test eax, STATE_CHECK - jne MustSignal - -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MIN_SIZE - ret -endif -endif - MON_EXIT_STATIC_RETURN_SUCCESS - - MustSignal: -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MIN_SIZE -endif -endif - ; void JITutil_MonSignal(AwareLock* lock, BYTE* pbLockTaken) - jmp JITutil_MonSignal - - RetryHelper: - jmp Retry - - LockError: - mov rcx, CORINFO_SynchronizationLockException_ASM -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MIN_SIZE -endif -endif - ; void JIT_InternalThrow(unsigned exceptNum) - jmp JIT_InternalThrow -NESTED_END JIT_MonExitStatic_InlineGetThread, _TEXT - end diff --git a/src/vm/amd64/JitHelpers_Slow.asm b/src/vm/amd64/JitHelpers_Slow.asm index 448bcb2279d9..f86d429e33f7 100644 --- a/src/vm/amd64/JitHelpers_Slow.asm +++ b/src/vm/amd64/JitHelpers_Slow.asm @@ -836,894 +836,6 @@ NESTED_ENTRY JIT_GetSharedGCStaticBaseNoCtor_Slow, _TEXT NESTED_END JIT_GetSharedGCStaticBaseNoCtor_Slow, _TEXT -MON_ENTER_STACK_SIZE equ 00000020h -MON_EXIT_STACK_SIZE equ 00000068h - -ifdef MON_DEBUG -ifdef TRACK_SYNC -MON_ENTER_STACK_SIZE_INLINEGETTHREAD equ 00000020h -MON_EXIT_STACK_SIZE_INLINEGETTHREAD equ 00000068h -endif -endif - -BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX equ 08000000h ; syncblk.h -BIT_SBLK_IS_HASHCODE equ 04000000h ; syncblk.h -BIT_SBLK_SPIN_LOCK equ 10000000h ; syncblk.h - -SBLK_MASK_LOCK_THREADID equ 000003FFh ; syncblk.h -SBLK_LOCK_RECLEVEL_INC equ 00000400h ; syncblk.h -SBLK_MASK_LOCK_RECLEVEL equ 0000FC00h ; syncblk.h - -MASK_SYNCBLOCKINDEX equ 03FFFFFFh ; syncblk.h -STATE_CHECK equ 0FFFFFFFEh - -MT_CTX_PROXY_FLAG equ 10000000h - -g_pSyncTable equ ?g_pSyncTable@@3PEAVSyncTableEntry@@EA -g_SystemInfo equ ?g_SystemInfo@@3U_SYSTEM_INFO@@A -g_SpinConstants equ ?g_SpinConstants@@3USpinConstants@@A - -extern g_pSyncTable:QWORD -extern g_SystemInfo:QWORD -extern g_SpinConstants:QWORD - -; JITutil_MonEnterWorker(Object* obj, BYTE* pbLockTaken) -extern JITutil_MonEnterWorker:proc -; JITutil_MonTryEnter(Object* obj, INT32 timeout, BYTE* pbLockTaken) -extern JITutil_MonTryEnter:proc -; JITutil_MonExitWorker(Object* obj, BYTE* pbLockTaken) -extern JITutil_MonExitWorker:proc -; JITutil_MonSignal(AwareLock* lock, BYTE* pbLockTaken) -extern JITutil_MonSignal:proc -; JITutil_MonContention(AwareLock* lock, BYTE* pbLockTaken) -extern JITutil_MonContention:proc - -ifdef _DEBUG -MON_DEBUG equ 1 -endif - -ifdef MON_DEBUG -ifdef TRACK_SYNC -extern EnterSyncHelper:proc -extern LeaveSyncHelper:proc -endif -endif - - -; This is a frameless helper for entering a monitor on a object. -; The object is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; -; EXTERN_C void JIT_MonEnterWorker_Slow(Object* obj, /*OUT*/ BYTE* pbLockTaken) -NESTED_ENTRY JIT_MonEnterWorker_Slow, _TEXT - push_nonvol_reg rsi - - alloc_stack MON_ENTER_STACK_SIZE - - save_reg_postrsp rcx, MON_ENTER_STACK_SIZE + 10h + 0h - save_reg_postrsp rdx, MON_ENTER_STACK_SIZE + 10h + 8h - save_reg_postrsp r8, MON_ENTER_STACK_SIZE + 10h + 10h - save_reg_postrsp r9, MON_ENTER_STACK_SIZE + 10h + 18h - - END_PROLOGUE - - ; Check if the instance is NULL - test rcx, rcx - jz FramedLockHelper - - ; Put pbLockTaken in rsi, this can be null - mov rsi, rdx - - ; We store the thread object in r11 - CALL_GETTHREAD - mov r11, rax - - ; Initialize delay value for retry with exponential backoff - mov r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwInitialDuration] - - ; Check if we can abort here - mov eax, dword ptr [r11 + OFFSETOF__Thread__m_State] - and eax, THREAD_CATCHATSAFEPOINT_BITS - ; Go through the slow code path to initiate ThreadAbort - jnz FramedLockHelper - - ; r8 will hold the syncblockindex address - lea r8, [rcx - OFFSETOF__ObjHeader__SyncBlkIndex] - - RetryThinLock: - ; Fetch the syncblock dword - mov eax, dword ptr [r8] - - ; Check whether we have the "thin lock" layout, the lock is free and the spin lock bit is not set - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK + SBLK_MASK_LOCK_THREADID + SBLK_MASK_LOCK_RECLEVEL - jnz NeedMoreTests - - ; Everything is fine - get the thread id to store in the lock - mov edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - - ; If the thread id is too large, we need a syncblock for sure - cmp edx, SBLK_MASK_LOCK_THREADID - ja FramedLockHelper - - ; We want to store a new value with the current thread id set in the low 10 bits - or edx, eax - lock cmpxchg dword ptr [r8], edx - jnz PrepareToWaitThinLock - - ; Everything went fine and we're done - add dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - - ; Done, leave and set pbLockTaken if we have it - jmp LockTaken - - NeedMoreTests: - ; OK, not the simple case, find out which case it is - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX - jnz HaveHashOrSyncBlockIndex - - ; The header is transitioning or the lock, treat this as if the lock was taken - test eax, BIT_SBLK_SPIN_LOCK - jnz PrepareToWaitThinLock - - ; Here we know we have the "thin lock" layout, but the lock is not free. - ; It could still be the recursion case, compare the thread id to check - mov edx, eax - and edx, SBLK_MASK_LOCK_THREADID - cmp edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - jne PrepareToWaitThinLock - - ; Ok, the thread id matches, it's the recursion case. - ; Bump up the recursion level and check for overflow - lea edx, [eax + SBLK_LOCK_RECLEVEL_INC] - test edx, SBLK_MASK_LOCK_RECLEVEL - jz FramedLockHelper - - ; Try to put the new recursion level back. If the header was changed in the meantime - ; we need a full retry, because the layout could have changed - lock cmpxchg dword ptr [r8], edx - jnz RetryHelperThinLock - - ; Done, leave and set pbLockTaken if we have it - jmp LockTaken - - PrepareToWaitThinLock: - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr [g_SystemInfo + OFFSETOF__g_SystemInfo__dwNumberOfProcessors], 1 - jle FramedLockHelper - - ; Exponential backoff; delay by approximately 2*r10 clock cycles - mov eax, r10d - delayLoopThinLock: - pause ; indicate to the CPU that we are spin waiting - sub eax, 1 - jnz delayLoopThinLock - - ; Next time, wait a factor longer - imul r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwBackoffFactor] - - cmp r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwMaximumDuration] - jle RetryHelperThinLock - - jmp FramedLockHelper - - RetryHelperThinLock: - jmp RetryThinLock - - HaveHashOrSyncBlockIndex: - ; If we have a hash code already, we need to create a sync block - test eax, BIT_SBLK_IS_HASHCODE - jnz FramedLockHelper - - ; OK, we have a sync block index, just and out the top bits and grab the synblock index - and eax, MASK_SYNCBLOCKINDEX - - ; Get the sync block pointer - mov rdx, qword ptr [g_pSyncTable] - shl eax, 4h - mov rdx, [rdx + rax + OFFSETOF__SyncTableEntry__m_SyncBlock] - - ; Check if the sync block has been allocated - test rdx, rdx - jz FramedLockHelper - - ; Get a pointer to the lock object - lea rdx, [rdx + OFFSETOF__SyncBlock__m_Monitor] - - ; Attempt to acquire the lock - RetrySyncBlock: - mov eax, dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld] - test eax, eax - jne HaveWaiters - - ; Common case, lock isn't held and there are no waiters. Attempt to - ; gain ownership ourselves - xor ecx, ecx - inc ecx - lock cmpxchg dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld], ecx - jnz RetryHelperSyncBlock - - ; Success. Save the thread object in the lock and increment the use count - mov qword ptr [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - add dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - add dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rcx, [rsp + MON_ENTER_STACK_SIZE + 8h] ; return address - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - call EnterSyncHelper -endif -endif - - ; Done, leave and set pbLockTaken if we have it - jmp LockTaken - - ; It's possible to get here with waiters by no lock held, but in this - ; case a signal is about to be fired which will wake up the waiter. So - ; for fairness sake we should wait too. - ; Check first for recur11ve lock attempts on the same thread. - HaveWaiters: - ; Is mutex already owned by current thread? - cmp [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - jne PrepareToWait - - ; Yes, bump our use count. - add dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rcx, [rsp + MON_ENTER_STACK_SIZE + 8h] ; return address - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - call EnterSyncHelper -endif -endif - ; Done, leave and set pbLockTaken if we have it - jmp LockTaken - - PrepareToWait: - ; If we are on a MP system we try spinning for a certain number of iterations - cmp dword ptr [g_SystemInfo + OFFSETOF__g_SystemInfo__dwNumberOfProcessors], 1 - jle HaveWaiters1 - - ; Exponential backoff: delay by approximately 2*r10 clock cycles - mov eax, r10d - delayLoop: - pause ; indicate to the CPU that we are spin waiting - sub eax, 1 - jnz delayLoop - - ; Next time, wait a factor longer - imul r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwBackoffFactor] - - cmp r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwMaximumDuration] - jle RetrySyncBlock - - HaveWaiters1: - mov rcx, rdx - mov rdx, rsi - add rsp, MON_ENTER_STACK_SIZE - pop rsi - ; void JITutil_MonContention(AwareLock* lock, BYTE* pbLockTaken) - jmp JITutil_MonContention - - RetryHelperSyncBlock: - jmp RetrySyncBlock - - FramedLockHelper: - mov rdx, rsi - add rsp, MON_ENTER_STACK_SIZE - pop rsi - ; void JITutil_MonEnterWorker(Object* obj, BYTE* pbLockTaken) - jmp JITutil_MonEnterWorker - - align 16 - ; This is sensitive to the potential that pbLockTaken is NULL - LockTaken: - test rsi, rsi - jz LockTaken_Exit - mov byte ptr [rsi], 1 - LockTaken_Exit: - add rsp, MON_ENTER_STACK_SIZE - pop rsi - ret -NESTED_END JIT_MonEnterWorker_Slow, _TEXT - -; This is a frameless helper for exiting a monitor on a object. -; The object is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; -; void JIT_MonExitWorker_Slow(Object* obj, BYTE* pbLockTaken) -NESTED_ENTRY JIT_MonExitWorker_Slow, _TEXT - alloc_stack MON_EXIT_STACK_SIZE - - save_reg_postrsp rcx, MON_EXIT_STACK_SIZE + 8h + 0h - save_reg_postrsp rdx, MON_EXIT_STACK_SIZE + 8h + 8h - save_reg_postrsp r8, MON_EXIT_STACK_SIZE + 8h + 10h - save_reg_postrsp r9, MON_EXIT_STACK_SIZE + 8h + 18h - - END_PROLOGUE - - ; pbLockTaken is stored in r10 - mov r10, rdx - - ; if pbLockTaken is NULL then we got here without a state variable, avoid the - ; next comparison in that case as it will AV - test rdx, rdx - jz Null_pbLockTaken - - ; If the lock wasn't taken then we bail quickly without doing anything - cmp byte ptr [rdx], 0 - je LockNotTaken - - Null_pbLockTaken: - ; Check is the instance is null - test rcx, rcx - jz FramedLockHelper - - ; The Thread obj address is stored in r11 - CALL_GETTHREAD - mov r11, rax - - ; r8 will hold the syncblockindex address - lea r8, [rcx - OFFSETOF__ObjHeader__SyncBlkIndex] - - RetryThinLock: - ; Fetch the syncblock dword - mov eax, dword ptr [r8] - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK - jnz NeedMoreTests - - ; Ok, we have a "thin lock" layout - check whether the thread id matches - mov edx, eax - and edx, SBLK_MASK_LOCK_THREADID - cmp edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - jne FramedLockHelper - - ; check the recursion level - test eax, SBLK_MASK_LOCK_RECLEVEL - jne DecRecursionLevel - - ; It's zero -- we're leaving the lock. - ; So try to put back a zero thread id. - ; edx and eax match in the thread id bits, and edx is zero else where, so the xor is sufficient - xor edx, eax - lock cmpxchg dword ptr [r8], edx - jnz RetryHelperThinLock - - ; Dec the dwLockCount on the thread - sub dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - - ; Done, leave and set pbLockTaken if we have it - jmp LockReleased - - DecRecursionLevel: - lea edx, [eax - SBLK_LOCK_RECLEVEL_INC] - lock cmpxchg dword ptr [r8], edx - jnz RetryHelperThinLock - - ; We're done, leave and set pbLockTaken if we have it - jmp LockReleased - - NeedMoreTests: - ; Forward all special cases to the slow helper - test eax, BIT_SBLK_IS_HASHCODE + BIT_SBLK_SPIN_LOCK - jnz FramedLockHelper - - ; Get the sync block index and use it to compute the sync block pointer - mov rdx, qword ptr [g_pSyncTable] - and eax, MASK_SYNCBLOCKINDEX - shl eax, 4 - mov rdx, [rdx + rax + OFFSETOF__SyncTableEntry__m_SyncBlock] - - ; Was there a sync block? - test rdx, rdx - jz FramedLockHelper - - ; Get a pointer to the lock object. - lea rdx, [rdx + OFFSETOF__SyncBlock__m_Monitor] - - ; Check if the lock is held. - cmp qword ptr [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - jne FramedLockHelper - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov [rsp + 28h], rcx - mov [rsp + 30h], rdx - mov [rsp + 38h], r10 - mov [rsp + 40h], r11 - - mov rcx, [rsp + MON_EXIT_STACK_SIZE ] ; return address - ; void LeaveSyncHelper(UINT_PTR caller, AwareLock* lock) - call LeaveSyncHelper - - mov rcx, [rsp + 28h] - mov rdx, [rsp + 30h] - mov r10, [rsp + 38h] - mov r11, [rsp + 40h] -endif -endif - - ; Reduce our recursion count - sub dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - jz LastRecursion - - ; Done, leave and set pbLockTaken if we have it - jmp LockReleased - - RetryHelperThinLock: - jmp RetryThinLock - - FramedLockHelper: - mov rdx, r10 - add rsp, MON_EXIT_STACK_SIZE - ; void JITutil_MonExitWorker(Object* obj, BYTE* pbLockTaken) - jmp JITutil_MonExitWorker - - LastRecursion: -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rax, [rdx + OFFSETOF__AwareLock__m_HoldingThread] -endif -endif - - sub dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - mov qword ptr [rdx + OFFSETOF__AwareLock__m_HoldingThread], 0 - - Retry: - mov eax, dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld] - lea r9d, [eax - 1] - lock cmpxchg dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld], r9d - jne RetryHelper - - test eax, STATE_CHECK - jne MustSignal - - ; Done, leave and set pbLockTaken if we have it - jmp LockReleased - - MustSignal: - mov rcx, rdx - mov rdx, r10 - add rsp, MON_EXIT_STACK_SIZE - ; void JITutil_MonSignal(AwareLock* lock, BYTE* pbLockTaken) - jmp JITutil_MonSignal - - RetryHelper: - jmp Retry - - LockNotTaken: - add rsp, MON_EXIT_STACK_SIZE - ret - - align 16 - ; This is sensitive to the potential that pbLockTaken is null - LockReleased: - test r10, r10 - jz LockReleased_Exit - mov byte ptr [r10], 0 - LockReleased_Exit: - add rsp, MON_EXIT_STACK_SIZE - ret -NESTED_END JIT_MonExitWorker_Slow, _TEXT - -; This is a frameless helper for trying to enter a monitor on a object. -; The object is in ARGUMENT_REG1 and a timeout in ARGUMENT_REG2. This tries the -; normal case (no object allocation) in line and calls a framed helper for the -; other cases. -; -; void JIT_MonTryEnter_Slow(Object* obj, INT32 timeOut, BYTE* pbLockTaken) -NESTED_ENTRY JIT_MonTryEnter_Slow, _TEXT - push_nonvol_reg rsi - - alloc_stack MON_ENTER_STACK_SIZE - - save_reg_postrsp rcx, MON_ENTER_STACK_SIZE + 10h + 0h - save_reg_postrsp rdx, MON_ENTER_STACK_SIZE + 10h + 8h - save_reg_postrsp r8, MON_ENTER_STACK_SIZE + 10h + 10h - save_reg_postrsp r9, MON_ENTER_STACK_SIZE + 10h + 18h - - END_PROLOGUE - - mov rsi, rdx - - ; Check if the instance is NULL - test rcx, rcx - jz FramedLockHelper - - ; Check if the timeout looks valid - cmp rdx, -1 - jl FramedLockHelper - - ; We store the thread object in r11 - CALL_GETTHREAD - mov r11, rax - - ; Initialize delay value for retry with exponential backoff - mov r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwInitialDuration] - - ; Check if we can abort here - mov eax, dword ptr [r11 + OFFSETOF__Thread__m_State] - and eax, THREAD_CATCHATSAFEPOINT_BITS - ; Go through the slow code path to initiate THreadAbort - jnz FramedLockHelper - - ; r9 will hold the syncblockindex address - lea r9, [rcx - OFFSETOF__ObjHeader__SyncBlkIndex] - - RetryThinLock: - ; Fetch the syncblock dword - mov eax, dword ptr [r9] - - ; Check whether we have the "thin lock" layout, the lock is free and the spin lock bit is not set - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK + SBLK_MASK_LOCK_THREADID + SBLK_MASK_LOCK_RECLEVEL - jne NeedMoreTests - - ; Everything is fine - get the thread id to store in the lock - mov edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - - ; If the thread id is too large, we need a syncblock for sure - cmp edx, SBLK_MASK_LOCK_THREADID - ja FramedLockHelper - - ; We want to store a new value with the current thread id set in the low 10 bits - or edx, eax - lock cmpxchg dword ptr [r9], edx - jnz RetryHelperThinLock - - ; Got the lock, everything is fine - add dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - ; Return TRUE - mov byte ptr [r8], 1 - add rsp, MON_ENTER_STACK_SIZE - pop rsi - ret - - NeedMoreTests: - ; OK, not the simple case, find out which case it is - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX - jnz HaveHashOrSyncBlockIndex - - ; The header is transitioning or the lock - test eax, BIT_SBLK_SPIN_LOCK - jnz RetryHelperThinLock - - ; Here we know we have the "thin lock" layout, but the lock is not free. - ; It could still be the recursion case, compare the thread id to check - mov edx, eax - and edx, SBLK_MASK_LOCK_THREADID - cmp edx, dword ptr [r11 + OFFSETOF__Thread__m_ThreadId] - jne PrepareToWaitThinLock - - ; Ok, the thread id matches, it's the recursion case. - ; Dump up the recursion level and check for overflow - lea edx, [eax + SBLK_LOCK_RECLEVEL_INC] - test edx, SBLK_MASK_LOCK_RECLEVEL - jz FramedLockHelper - - ; Try to put the new recursion level back. If the header was changed in the meantime - ; we need a full retry, because the layout could have changed - lock cmpxchg dword ptr [r9], edx - jnz RetryHelperThinLock - - ; Everything went fine and we're done, return TRUE - mov byte ptr [r8], 1 - add rsp, MON_ENTER_STACK_SIZE - pop rsi - ret - - PrepareToWaitThinLock: - ; Return failure if timeout is zero - test rsi, rsi - jz TimeoutZero - - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr [g_SystemInfo + OFFSETOF__g_SystemInfo__dwNumberOfProcessors], 1 - jle FramedLockHelper - - ; Exponential backoff; delay by approximately 2*r10d clock cycles - mov eax, r10d - DelayLoopThinLock: - pause ; indicate to the CPU that we are spin waiting - sub eax, 1 - jnz DelayLoopThinLock - - ; Next time, wait a factor longer - imul r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwBackoffFactor] - - cmp r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwMaximumDuration] - jle RetryHelperThinLock - - jmp FramedLockHelper - - RetryHelperThinLock: - jmp RetryThinLock - - HaveHashOrSyncBlockIndex: - ; If we have a hash code already, we need to create a sync block - test eax, BIT_SBLK_IS_HASHCODE - jnz FramedLockHelper - - ; OK, we have a sync block index, just and out the top bits and grab the synblock index - and eax, MASK_SYNCBLOCKINDEX - - ; Get the sync block pointer - mov rdx, qword ptr [g_pSyncTable] - shl eax, 4 - mov rdx, [rdx + rax + OFFSETOF__SyncTableEntry__m_SyncBlock] - - ; Check if the sync block has been allocated - test rdx, rdx - jz FramedLockHelper - - ; Get a pointer to the lock object - lea rdx, [rdx + OFFSETOF__SyncBlock__m_Monitor] - - RetrySyncBlock: - ; Attempt to acuire the lock - mov eax, dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld] - test eax, eax - jne HaveWaiters - - ; Common case, lock isn't held and there are no waiters. Attempt to - ; gain ownership ourselves - xor ecx, ecx - inc ecx - lock cmpxchg dword ptr [rdx + OFFSETOF__AwareLock__m_MonitorHeld], ecx - jnz RetryHelperSyncBlock - - ; Success. Save the thread object in the lock and increment the use count - mov qword ptr [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - add dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - add dword ptr [r11 + OFFSETOF__Thread__m_dwLockCount], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rcx, [rsp + MON_ENTER_STACK_SIZE + 8h] ; return address - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - call EnterSyncHelper -endif -endif - - ; Return TRUE - mov byte ptr [r8], 1 - add rsp, MON_ENTER_STACK_SIZE - pop rsi - ret - - ; It's possible to get here with waiters by no lock held, but in this - ; case a signal is about to be fired which will wake up the waiter. So - ; for fairness sake we should wait too. - ; Check first for recur11ve lock attempts on the same thread. - HaveWaiters: - ; Is mutex already owned by current thread? - cmp [rdx + OFFSETOF__AwareLock__m_HoldingThread], r11 - jne PrepareToWait - - ; Yes, bump our use count. - add dword ptr [rdx + OFFSETOF__AwareLock__m_Recursion], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rcx, [rsp + MON_ENTER_STACK_SIZE + 8h] ; return address - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - call EnterSyncHelper -endif -endif - - ; Return TRUE - mov byte ptr [r8], 1 - add rsp, MON_ENTER_STACK_SIZE - pop rsi - ret - - PrepareToWait: - ; Return failure if timeout is zero - test rsi, rsi - jz TimeoutZero - - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr [g_SystemInfo + OFFSETOF__g_SystemInfo__dwNumberOfProcessors], 1 - jle Block - - ; Exponential backoff; delay by approximately 2*r10d clock cycles - mov eax, r10d - DelayLoop: - pause ; indicate to the CPU that we are spin waiting - sub eax, 1 - jnz DelayLoop - - ; Next time, wait a factor longer - imul r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwBackoffFactor] - - cmp r10d, dword ptr [g_SpinConstants + OFFSETOF__g_SpinConstants__dwMaximumDuration] - jle RetrySyncBlock - - jmp Block - - TimeoutZero: - ; Return FALSE - mov byte ptr [r8], 0 - add rsp, MON_ENTER_STACK_SIZE - pop rsi - ret - - RetryHelperSyncBlock: - jmp RetrySyncBlock - - Block: - ; In the Block case we've trashed RCX, restore it - mov rcx, [rsp + MON_ENTER_STACK_SIZE + 10h] - FramedLockHelper: - mov rdx, rsi - add rsp, MON_ENTER_STACK_SIZE - pop rsi - ; void JITutil_MonTryEnter(Object* obj, UINT32 timeout, BYTE* pbLockTaken) - jmp JITutil_MonTryEnter - -NESTED_END JIT_MonTryEnter_Slow, _TEXT - -MON_ENTER_STATIC_RETURN_SUCCESS macro - ; pbLockTaken is never null for static helpers - mov byte ptr [rdx], 1 - add rsp, MIN_SIZE - ret - - endm - -MON_EXIT_STATIC_RETURN_SUCCESS macro - ; pbLockTaken is never null for static helpers - mov byte ptr [rdx], 0 - add rsp, MIN_SIZE - ret - - endm - - -; This is a frameless helper for entering a static monitor on a class. -; The methoddesc is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; -; void JIT_MonEnterStatic_Slow(AwareLock *lock, BYTE *pbLockTaken) -NESTED_ENTRY JIT_MonEnterStatic_Slow, _TEXT - alloc_stack MIN_SIZE - END_PROLOGUE - - ; Attempt to acquire the lock - Retry: - mov eax, dword ptr [rcx + OFFSETOF__AwareLock__m_MonitorHeld] - test eax, eax - jne HaveWaiters - - ; Common case; lock isn't held and there are no waiters. Attempt to - ; gain ownership by ourselves. - mov r10d, 1 - lock cmpxchg dword ptr [rcx + OFFSETOF__AwareLock__m_MonitorHeld], r10d - jnz RetryHelper - - ; Success. Save the thread object in the lock and increment the use count. - CALL_GETTHREAD - - mov qword ptr [rcx + OFFSETOF__AwareLock__m_HoldingThread], rax - add dword ptr [rcx + OFFSETOF__AwareLock__m_Recursion], 1 - add dword ptr [rax + OFFSETOF__Thread__m_dwLockCount], 1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - add rsp, MIN_SIZE - mov rdx, rcx - mov rcx, [rsp] - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - jmp EnterSyncHelper -endif -endif - MON_ENTER_STATIC_RETURN_SUCCESS - - ; It's possible to get here with waiters by with no lock held, in this - ; case a signal is about to be fired which will wake up a waiter. So - ; for fairness sake we should wait too. - ; Check first for recursive lock attempts on the same thread. - HaveWaiters: - CALL_GETTHREAD - - ; Is mutex alread owned by current thread? - cmp [rcx + OFFSETOF__AwareLock__m_HoldingThread], rax - jne PrepareToWait - - ; Yes, bump our use count. - add dword ptr [rcx + OFFSETOF__AwareLock__m_Recursion], 1 -ifdef MON_DEBUG -ifdef TRACK_SYNC - mov rdx, rcx - mov rcx, [rsp] - ; void EnterSyncHelper(UINT_PTR caller, AwareLock* lock) - add rsp, MIN_SIZE - jmp EnterSyncHelper -endif -endif - MON_ENTER_STATIC_RETURN_SUCCESS - - PrepareToWait: - add rsp, MIN_SIZE - ; void JITutil_MonContention(AwareLock* obj, BYTE* pbLockTaken) - jmp JITutil_MonContention - - RetryHelper: - jmp Retry -NESTED_END JIT_MonEnterStatic_Slow, _TEXT - -; A frameless helper for exiting a static monitor on a class. -; The methoddesc is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; -; void JIT_MonExitStatic_Slow(AwareLock *lock, BYTE *pbLockTaken) -NESTED_ENTRY JIT_MonExitStatic_Slow, _TEXT - alloc_stack MIN_SIZE - END_PROLOGUE - -ifdef MON_DEBUG -ifdef TRACK_SYNC - push rsi - push rdi - mov rsi, rcx - mov rdi, rdx - mov rdx, [rsp + 8] - call LeaveSyncHelper - mov rcx, rsi - mov rdx, rdi - pop rdi - pop rsi -endif -endif - - ; Check if lock is held - CALL_GETTHREAD - - cmp [rcx + OFFSETOF__AwareLock__m_HoldingThread], rax - jne LockError - - ; Reduce our recursion count - sub dword ptr [rcx + OFFSETOF__AwareLock__m_Recursion], 1 - jz LastRecursion - - MON_EXIT_STATIC_RETURN_SUCCESS - - ; This is the last count we held on this lock, so release the lock - LastRecursion: - ; Thead* is in rax - sub dword ptr [rax + OFFSETOF__Thread__m_dwLockCount], 1 - mov qword ptr [rcx + OFFSETOF__AwareLock__m_HoldingThread], 0 - - Retry: - mov eax, dword ptr [rcx + OFFSETOF__AwareLock__m_MonitorHeld] - lea r10d, [eax - 1] - lock cmpxchg dword ptr [rcx + OFFSETOF__AwareLock__m_MonitorHeld], r10d - jne RetryHelper - test eax, STATE_CHECK - jne MustSignal - - MON_EXIT_STATIC_RETURN_SUCCESS - - MustSignal: - add rsp, MIN_SIZE - ; void JITutil_MonSignal(AwareLock* lock, BYTE* pbLockTaken) - jmp JITutil_MonSignal - - RetryHelper: - jmp Retry - - LockError: - mov rcx, CORINFO_SynchronizationLockException_ASM - add rsp, MIN_SIZE - ; void JIT_InternalThrow(unsigned exceptNum) - jmp JIT_InternalThrow -NESTED_END JIT_MonExitStatic_Slow, _TEXT - - ifdef _DEBUG extern Object__DEBUG_SetAppDomain:proc diff --git a/src/vm/amd64/asmconstants.h b/src/vm/amd64/asmconstants.h index 4a100c18234f..1fef80f66ddc 100644 --- a/src/vm/amd64/asmconstants.h +++ b/src/vm/amd64/asmconstants.h @@ -184,37 +184,12 @@ ASMCONSTANTS_C_ASSERT(THREAD_CATCHATSAFEPOINT_BITS == Thread::TS_CatchAtSafePoin #define OFFSETOF__NDirectMethodDesc__m_pWriteableData DBG_FRE(0x48, 0x20) ASMCONSTANTS_C_ASSERT(OFFSETOF__NDirectMethodDesc__m_pWriteableData == offsetof(NDirectMethodDesc, ndirect.m_pWriteableData)); -#define OFFSETOF__ObjHeader__SyncBlkIndex 0x4 -ASMCONSTANTS_C_ASSERT(OFFSETOF__ObjHeader__SyncBlkIndex - == (sizeof(ObjHeader) - offsetof(ObjHeader, m_SyncBlockValue))); - -#define SIZEOF__SyncTableEntry 0x10 -ASMCONSTANT_SIZEOF_ASSERT(SyncTableEntry); - -#define OFFSETOF__SyncTableEntry__m_SyncBlock 0x0 -ASMCONSTANT_OFFSETOF_ASSERT(SyncTableEntry, m_SyncBlock); - -#define OFFSETOF__SyncBlock__m_Monitor 0x0 -ASMCONSTANT_OFFSETOF_ASSERT(SyncBlock, m_Monitor); - #define OFFSETOF__DelegateObject___methodPtr 0x18 ASMCONSTANT_OFFSETOF_ASSERT(DelegateObject, _methodPtr); #define OFFSETOF__DelegateObject___target 0x08 ASMCONSTANT_OFFSETOF_ASSERT(DelegateObject, _target); -#define OFFSETOF__AwareLock__m_MonitorHeld 0x0 -ASMCONSTANTS_C_ASSERT(OFFSETOF__AwareLock__m_MonitorHeld - == offsetof(AwareLock, m_MonitorHeld)); - -#define OFFSETOF__AwareLock__m_Recursion 0x4 -ASMCONSTANTS_C_ASSERT(OFFSETOF__AwareLock__m_Recursion - == offsetof(AwareLock, m_Recursion)); - -#define OFFSETOF__AwareLock__m_HoldingThread 0x8 -ASMCONSTANTS_C_ASSERT(OFFSETOF__AwareLock__m_HoldingThread - == offsetof(AwareLock, m_HoldingThread)); - #define OFFSETOF__g_SystemInfo__dwNumberOfProcessors 0x20 ASMCONSTANTS_C_ASSERT(OFFSETOF__g_SystemInfo__dwNumberOfProcessors == offsetof(SYSTEM_INFO, dwNumberOfProcessors)); @@ -346,10 +321,6 @@ ASMCONSTANTS_C_ASSERT( CORINFO_InvalidCastException_ASM ASMCONSTANTS_C_ASSERT( CORINFO_IndexOutOfRangeException_ASM == CORINFO_IndexOutOfRangeException); -#define CORINFO_SynchronizationLockException_ASM 5 -ASMCONSTANTS_C_ASSERT( CORINFO_SynchronizationLockException_ASM - == CORINFO_SynchronizationLockException); - #define CORINFO_ArrayTypeMismatchException_ASM 6 ASMCONSTANTS_C_ASSERT( CORINFO_ArrayTypeMismatchException_ASM == CORINFO_ArrayTypeMismatchException); @@ -613,10 +584,6 @@ ASMCONSTANTS_C_ASSERT(OFFSETOF__StringObject__m_StringLength ASMCONSTANTS_C_ASSERT(OFFSETOF__ArrayTypeDesc__m_Arg == offsetof(ArrayTypeDesc, m_Arg)); -#define SYNCBLOCKINDEX_OFFSET 0x4 -ASMCONSTANTS_C_ASSERT(SYNCBLOCKINDEX_OFFSET - == (sizeof(ObjHeader) - offsetof(ObjHeader, m_SyncBlockValue))); - #define CallDescrData__pSrc 0x00 #define CallDescrData__numStackSlots 0x08 #ifdef UNIX_AMD64_ABI diff --git a/src/vm/amd64/cgencpu.h b/src/vm/amd64/cgencpu.h index b74e3ca7d3ea..98e9770858b5 100644 --- a/src/vm/amd64/cgencpu.h +++ b/src/vm/amd64/cgencpu.h @@ -544,20 +544,10 @@ inline BOOL ClrFlushInstructionCache(LPCVOID pCodeAddr, size_t sizeOfCode) // // Create alias for optimized implementations of helpers provided on this platform // -#define JIT_MonEnter JIT_MonEnter -#define JIT_MonEnterWorker JIT_MonEnterWorker_InlineGetThread -#define JIT_MonReliableEnter JIT_MonEnterWorker -#define JIT_MonTryEnter JIT_MonTryEnter_InlineGetThread -#define JIT_MonExit JIT_MonExit -#define JIT_MonExitWorker JIT_MonExitWorker_InlineGetThread -#define JIT_MonEnterStatic JIT_MonEnterStatic_InlineGetThread -#define JIT_MonExitStatic JIT_MonExitStatic_InlineGetThread - #define JIT_GetSharedGCStaticBase JIT_GetSharedGCStaticBase_InlineGetAppDomain #define JIT_GetSharedNonGCStaticBase JIT_GetSharedNonGCStaticBase_InlineGetAppDomain #define JIT_GetSharedGCStaticBaseNoCtor JIT_GetSharedGCStaticBaseNoCtor_InlineGetAppDomain #define JIT_GetSharedNonGCStaticBaseNoCtor JIT_GetSharedNonGCStaticBaseNoCtor_InlineGetAppDomain - #endif // FEATURE_IMPLICIT_TLS #ifndef FEATURE_PAL diff --git a/src/vm/i386/asmconstants.h b/src/vm/i386/asmconstants.h index 0a581cffe039..f7d5f709dcaa 100644 --- a/src/vm/i386/asmconstants.h +++ b/src/vm/i386/asmconstants.h @@ -176,9 +176,6 @@ ASMCONSTANTS_C_ASSERT(CORINFO_IndexOutOfRangeException_ASM == CORINFO_IndexOutOf #define CORINFO_OverflowException_ASM 4 ASMCONSTANTS_C_ASSERT(CORINFO_OverflowException_ASM == CORINFO_OverflowException) -#define CORINFO_SynchronizationLockException_ASM 5 -ASMCONSTANTS_C_ASSERT(CORINFO_SynchronizationLockException_ASM == CORINFO_SynchronizationLockException) - #define CORINFO_ArrayTypeMismatchException_ASM 6 ASMCONSTANTS_C_ASSERT(CORINFO_ArrayTypeMismatchException_ASM == CORINFO_ArrayTypeMismatchException) @@ -232,79 +229,6 @@ ASMCONSTANTS_C_ASSERT(Thread::TS_Hijacked == TS_Hijacked_ASM) #define AppDomain__m_dwId 0x4 ASMCONSTANTS_C_ASSERT(AppDomain__m_dwId == offsetof(AppDomain, m_dwId)); -// from clr/src/vm/ceeload.cpp - -// from clr/src/vm/syncblk.h -#define SizeOfSyncTableEntry_ASM 8 -ASMCONSTANTS_C_ASSERT(sizeof(SyncTableEntry) == SizeOfSyncTableEntry_ASM) - -#define SyncBlockIndexOffset_ASM 4 -ASMCONSTANTS_C_ASSERT(sizeof(ObjHeader) - offsetof(ObjHeader, m_SyncBlockValue) == SyncBlockIndexOffset_ASM) - -#ifndef __GNUC__ -#define SyncTableEntry_m_SyncBlock 0 -ASMCONSTANTS_C_ASSERT(offsetof(SyncTableEntry, m_SyncBlock) == SyncTableEntry_m_SyncBlock) - -#define SyncBlock_m_Monitor 0 -ASMCONSTANTS_C_ASSERT(offsetof(SyncBlock, m_Monitor) == SyncBlock_m_Monitor) - -#define AwareLock_m_MonitorHeld 0 -ASMCONSTANTS_C_ASSERT(offsetof(AwareLock, m_MonitorHeld) == AwareLock_m_MonitorHeld) -#else -// The following 3 offsets have value of 0, and must be -// defined to be an empty string. Otherwise, gas may generate assembly -// code with 0 displacement if 0 is left in the displacement field -// of an instruction. -#define SyncTableEntry_m_SyncBlock // 0 -ASMCONSTANTS_C_ASSERT(offsetof(SyncTableEntry, m_SyncBlock) == 0) - -#define SyncBlock_m_Monitor // 0 -ASMCONSTANTS_C_ASSERT(offsetof(SyncBlock, m_Monitor) == 0) - -#define AwareLock_m_MonitorHeld // 0 -ASMCONSTANTS_C_ASSERT(offsetof(AwareLock, m_MonitorHeld) == 0) -#endif // !__GNUC__ - -#define AwareLock_m_HoldingThread 8 -ASMCONSTANTS_C_ASSERT(offsetof(AwareLock, m_HoldingThread) == AwareLock_m_HoldingThread) - -#define AwareLock_m_Recursion 4 -ASMCONSTANTS_C_ASSERT(offsetof(AwareLock, m_Recursion) == AwareLock_m_Recursion) - -#define BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX_ASM 0x08000000 -ASMCONSTANTS_C_ASSERT(BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX_ASM == BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) - -#define BIT_SBLK_SPIN_LOCK_ASM 0x10000000 -ASMCONSTANTS_C_ASSERT(BIT_SBLK_SPIN_LOCK_ASM == BIT_SBLK_SPIN_LOCK) - -#define SBLK_MASK_LOCK_THREADID_ASM 0x000003FF // special value of 0 + 1023 thread ids -ASMCONSTANTS_C_ASSERT(SBLK_MASK_LOCK_THREADID_ASM == SBLK_MASK_LOCK_THREADID) - -#define SBLK_MASK_LOCK_RECLEVEL_ASM 0x0000FC00 // 64 recursion levels -ASMCONSTANTS_C_ASSERT(SBLK_MASK_LOCK_RECLEVEL_ASM == SBLK_MASK_LOCK_RECLEVEL) - -#define SBLK_LOCK_RECLEVEL_INC_ASM 0x00000400 // each level is this much higher than the previous one -ASMCONSTANTS_C_ASSERT(SBLK_LOCK_RECLEVEL_INC_ASM == SBLK_LOCK_RECLEVEL_INC) - -#define BIT_SBLK_IS_HASHCODE_ASM 0x04000000 -ASMCONSTANTS_C_ASSERT(BIT_SBLK_IS_HASHCODE_ASM == BIT_SBLK_IS_HASHCODE) - -#define MASK_SYNCBLOCKINDEX_ASM 0x03ffffff // ((1< -g_SpinConstants TEXTEQU -g_pSyncTable TEXTEQU -JITutil_MonEnterWorker TEXTEQU <@JITutil_MonEnterWorker@4> -JITutil_MonReliableEnter TEXTEQU <@JITutil_MonReliableEnter@8> -JITutil_MonTryEnter TEXTEQU <@JITutil_MonTryEnter@12> -JITutil_MonExitWorker TEXTEQU <@JITutil_MonExitWorker@4> -JITutil_MonContention TEXTEQU <@JITutil_MonContention@4> -JITutil_MonReliableContention TEXTEQU <@JITutil_MonReliableContention@8> -JITutil_MonSignal TEXTEQU <@JITutil_MonSignal@4> -JIT_InternalThrow TEXTEQU <@JIT_InternalThrow@4> -EXTRN g_SystemInfo:BYTE -EXTRN g_SpinConstants:BYTE -EXTRN g_pSyncTable:DWORD -EXTRN JITutil_MonEnterWorker:PROC -EXTRN JITutil_MonReliableEnter:PROC -EXTRN JITutil_MonTryEnter:PROC -EXTRN JITutil_MonExitWorker:PROC -EXTRN JITutil_MonContention:PROC -EXTRN JITutil_MonReliableContention:PROC -EXTRN JITutil_MonSignal:PROC -EXTRN JIT_InternalThrow:PROC - -ifdef MON_DEBUG -ifdef TRACK_SYNC -EnterSyncHelper TEXTEQU <_EnterSyncHelper@8> -LeaveSyncHelper TEXTEQU <_LeaveSyncHelper@8> -EXTRN EnterSyncHelper:PROC -EXTRN LeaveSyncHelper:PROC -endif ;TRACK_SYNC -endif ;MON_DEBUG - -; The following macro is needed because MASM returns -; "instruction prefix not allowed" error message for -; rep nop mnemonic -$repnop MACRO - db 0F3h - db 090h -ENDM - -; Safe ThreadAbort does not abort a thread if it is running finally or has lock counts. -; At the time we call Monitor.Enter, we initiate the abort if we can. -; We do not need to do the same for Monitor.Leave, since most of time, Monitor.Leave is called -; during finally. - -;********************************************************************** -; This is a frameless helper for entering a monitor on a object. -; The object is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; ***** NOTE: if you make any changes to this routine, build with MON_DEBUG undefined -; to make sure you don't break the non-debug build. This is very fragile code. -; Also, propagate the changes to jithelp.s which contains the same helper and assembly code -; (in AT&T syntax) for gnu assembler. -@JIT_MonEnterWorker@4 proc public - ; Initialize delay value for retry with exponential backoff - push ebx - mov ebx, dword ptr g_SpinConstants+SpinConstants_dwInitialDuration - - ; We need yet another register to avoid refetching the thread object - push esi - - ; Check if the instance is NULL. - test ARGUMENT_REG1, ARGUMENT_REG1 - jz MonEnterFramedLockHelper - - call _GetThread@0 - mov esi,eax - - ; Check if we can abort here - mov eax, [esi+Thread_m_State] - and eax, TS_CatchAtSafePoint_ASM - jz MonEnterRetryThinLock - ; go through the slow code path to initiate ThreadAbort. - jmp MonEnterFramedLockHelper - -MonEnterRetryThinLock: - ; Fetch the object header dword - mov eax, [ARGUMENT_REG1-SyncBlockIndexOffset_ASM] - - ; Check whether we have the "thin lock" layout, the lock is free and the spin lock bit not set - ; SBLK_COMBINED_MASK_ASM = BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK + SBLK_MASK_LOCK_THREADID + SBLK_MASK_LOCK_RECLEVEL - test eax, SBLK_COMBINED_MASK_ASM - jnz MonEnterNeedMoreTests - - ; Everything is fine - get the thread id to store in the lock - mov edx, [esi+Thread_m_ThreadId] - - ; If the thread id is too large, we need a syncblock for sure - cmp edx, SBLK_MASK_LOCK_THREADID_ASM - ja MonEnterFramedLockHelper - - ; We want to store a new value with the current thread id set in the low 10 bits - or edx,eax - lock cmpxchg dword ptr [ARGUMENT_REG1-SyncBlockIndexOffset_ASM], edx - jnz MonEnterPrepareToWaitThinLock - - ; Everything went fine and we're done - add [esi+Thread_m_dwLockCount],1 - pop esi - pop ebx - ret - -MonEnterNeedMoreTests: - ; Ok, it's not the simple case - find out which case it is - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX_ASM - jnz MonEnterHaveHashOrSyncBlockIndex - - ; The header is transitioning or the lock - treat this as if the lock was taken - test eax, BIT_SBLK_SPIN_LOCK_ASM - jnz MonEnterPrepareToWaitThinLock - - ; Here we know we have the "thin lock" layout, but the lock is not free. - ; It could still be the recursion case - compare the thread id to check - mov edx,eax - and edx, SBLK_MASK_LOCK_THREADID_ASM - cmp edx, [esi+Thread_m_ThreadId] - jne MonEnterPrepareToWaitThinLock - - ; Ok, the thread id matches, it's the recursion case. - ; Bump up the recursion level and check for overflow - lea edx, [eax+SBLK_LOCK_RECLEVEL_INC_ASM] - test edx, SBLK_MASK_LOCK_RECLEVEL_ASM - jz MonEnterFramedLockHelper - - ; Try to put the new recursion level back. If the header was changed in the meantime, - ; we need a full retry, because the layout could have changed. - lock cmpxchg [ARGUMENT_REG1-SyncBlockIndexOffset_ASM], edx - jnz MonEnterRetryHelperThinLock - - ; Everything went fine and we're done - pop esi - pop ebx - ret - -MonEnterPrepareToWaitThinLock: - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr g_SystemInfo+SYSTEM_INFO_dwNumberOfProcessors,1 - jle MonEnterFramedLockHelper - - ; exponential backoff: delay by approximately 2*ebx clock cycles (on a PIII) - mov eax, ebx -MonEnterdelayLoopThinLock: - $repnop ; indicate to the CPU that we are spin waiting (useful for some Intel P4 multiprocs) - dec eax - jnz MonEnterdelayLoopThinLock - - ; next time, wait a factor longer - imul ebx, dword ptr g_SpinConstants+SpinConstants_dwBackoffFactor - - cmp ebx, dword ptr g_SpinConstants+SpinConstants_dwMaximumDuration - jle MonEnterRetryHelperThinLock - - jmp MonEnterFramedLockHelper - -MonEnterRetryHelperThinLock: - jmp MonEnterRetryThinLock - -MonEnterHaveHashOrSyncBlockIndex: - ; If we have a hash code already, we need to create a sync block - test eax, BIT_SBLK_IS_HASHCODE_ASM - jnz MonEnterFramedLockHelper - - ; Ok, we have a sync block index - just and out the top bits and grab the syncblock index - and eax, MASK_SYNCBLOCKINDEX_ASM - - ; Get the sync block pointer. - mov ARGUMENT_REG2, dword ptr g_pSyncTable - mov ARGUMENT_REG2, [ARGUMENT_REG2+eax*SizeOfSyncTableEntry_ASM+SyncTableEntry_m_SyncBlock] - - ; Check if the sync block has been allocated. - test ARGUMENT_REG2, ARGUMENT_REG2 - jz MonEnterFramedLockHelper - - ; Get a pointer to the lock object. - lea ARGUMENT_REG2, [ARGUMENT_REG2+SyncBlock_m_Monitor] - - ; Attempt to acquire the lock. -MonEnterRetrySyncBlock: - mov eax, [ARGUMENT_REG2+AwareLock_m_MonitorHeld] - test eax,eax - jne MonEnterHaveWaiters - - ; Common case, lock isn't held and there are no waiters. Attempt to - ; gain ownership ourselves. - mov ARGUMENT_REG1,1 - lock cmpxchg [ARGUMENT_REG2+AwareLock_m_MonitorHeld], ARGUMENT_REG1 - jnz MonEnterRetryHelperSyncBlock - - ; Success. Save the thread object in the lock and increment the use count. - mov dword ptr [ARGUMENT_REG2+AwareLock_m_HoldingThread],esi - inc dword ptr [esi+Thread_m_dwLockCount] - inc dword ptr [ARGUMENT_REG2+AwareLock_m_Recursion] - -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG2 ; AwareLock - push [esp+4] ; return address - call EnterSyncHelper -endif ;TRACK_SYNC -endif ;MON_DEBUG - pop esi - pop ebx - ret - - ; It's possible to get here with waiters but no lock held, but in this - ; case a signal is about to be fired which will wake up a waiter. So - ; for fairness sake we should wait too. - ; Check first for recursive lock attempts on the same thread. -MonEnterHaveWaiters: - ; Is mutex already owned by current thread? - cmp [ARGUMENT_REG2+AwareLock_m_HoldingThread],esi - jne MonEnterPrepareToWait - - ; Yes, bump our use count. - inc dword ptr [ARGUMENT_REG2+AwareLock_m_Recursion] -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG2 ; AwareLock - push [esp+4] ; return address - call EnterSyncHelper -endif ;TRACK_SYNC -endif ;MON_DEBUG - pop esi - pop ebx - ret - -MonEnterPrepareToWait: - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr g_SystemInfo+SYSTEM_INFO_dwNumberOfProcessors,1 - jle MonEnterHaveWaiters1 - - ; exponential backoff: delay by approximately 2*ebx clock cycles (on a PIII) - mov eax,ebx -MonEnterdelayLoop: - $repnop ; indicate to the CPU that we are spin waiting (useful for some Intel P4 multiprocs) - dec eax - jnz MonEnterdelayLoop - - ; next time, wait a factor longer - imul ebx, dword ptr g_SpinConstants+SpinConstants_dwBackoffFactor - - cmp ebx, dword ptr g_SpinConstants+SpinConstants_dwMaximumDuration - jle MonEnterRetrySyncBlock - -MonEnterHaveWaiters1: - - pop esi - pop ebx - - ; Place AwareLock in arg1 then call contention helper. - mov ARGUMENT_REG1, ARGUMENT_REG2 - jmp JITutil_MonContention - -MonEnterRetryHelperSyncBlock: - jmp MonEnterRetrySyncBlock - - ; ECX has the object to synchronize on -MonEnterFramedLockHelper: - pop esi - pop ebx - jmp JITutil_MonEnterWorker - -@JIT_MonEnterWorker@4 endp - -;********************************************************************** -; This is a frameless helper for entering a monitor on a object, and -; setting a flag to indicate that the lock was taken. -; The object is in ARGUMENT_REG1. The flag is in ARGUMENT_REG2. -; This tries the normal case (no blocking or object allocation) in line -; and calls a framed helper for the other cases. -; ***** NOTE: if you make any changes to this routine, build with MON_DEBUG undefined -; to make sure you don't break the non-debug build. This is very fragile code. -; Also, propagate the changes to jithelp.s which contains the same helper and assembly code -; (in AT&T syntax) for gnu assembler. -@JIT_MonReliableEnter@8 proc public - ; Initialize delay value for retry with exponential backoff - push ebx - mov ebx, dword ptr g_SpinConstants+SpinConstants_dwInitialDuration - - ; Put pbLockTaken in edi - push edi - mov edi, ARGUMENT_REG2 - - ; We need yet another register to avoid refetching the thread object - push esi - - ; Check if the instance is NULL. - test ARGUMENT_REG1, ARGUMENT_REG1 - jz MonReliableEnterFramedLockHelper - - call _GetThread@0 - mov esi,eax - - ; Check if we can abort here - mov eax, [esi+Thread_m_State] - and eax, TS_CatchAtSafePoint_ASM - jz MonReliableEnterRetryThinLock - ; go through the slow code path to initiate ThreadAbort. - jmp MonReliableEnterFramedLockHelper - -MonReliableEnterRetryThinLock: - ; Fetch the object header dword - mov eax, [ARGUMENT_REG1-SyncBlockIndexOffset_ASM] - - ; Check whether we have the "thin lock" layout, the lock is free and the spin lock bit not set - ; SBLK_COMBINED_MASK_ASM = BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK + SBLK_MASK_LOCK_THREADID + SBLK_MASK_LOCK_RECLEVEL - test eax, SBLK_COMBINED_MASK_ASM - jnz MonReliableEnterNeedMoreTests - - ; Everything is fine - get the thread id to store in the lock - mov edx, [esi+Thread_m_ThreadId] - - ; If the thread id is too large, we need a syncblock for sure - cmp edx, SBLK_MASK_LOCK_THREADID_ASM - ja MonReliableEnterFramedLockHelper - - ; We want to store a new value with the current thread id set in the low 10 bits - or edx,eax - lock cmpxchg dword ptr [ARGUMENT_REG1-SyncBlockIndexOffset_ASM], edx - jnz MonReliableEnterPrepareToWaitThinLock - - ; Everything went fine and we're done - add [esi+Thread_m_dwLockCount],1 - ; Set *pbLockTaken=true - mov byte ptr [edi],1 - pop esi - pop edi - pop ebx - ret - -MonReliableEnterNeedMoreTests: - ; Ok, it's not the simple case - find out which case it is - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX_ASM - jnz MonReliableEnterHaveHashOrSyncBlockIndex - - ; The header is transitioning or the lock - treat this as if the lock was taken - test eax, BIT_SBLK_SPIN_LOCK_ASM - jnz MonReliableEnterPrepareToWaitThinLock - - ; Here we know we have the "thin lock" layout, but the lock is not free. - ; It could still be the recursion case - compare the thread id to check - mov edx,eax - and edx, SBLK_MASK_LOCK_THREADID_ASM - cmp edx, [esi+Thread_m_ThreadId] - jne MonReliableEnterPrepareToWaitThinLock - - ; Ok, the thread id matches, it's the recursion case. - ; Bump up the recursion level and check for overflow - lea edx, [eax+SBLK_LOCK_RECLEVEL_INC_ASM] - test edx, SBLK_MASK_LOCK_RECLEVEL_ASM - jz MonReliableEnterFramedLockHelper - - ; Try to put the new recursion level back. If the header was changed in the meantime, - ; we need a full retry, because the layout could have changed. - lock cmpxchg [ARGUMENT_REG1-SyncBlockIndexOffset_ASM], edx - jnz MonReliableEnterRetryHelperThinLock - - ; Everything went fine and we're done - ; Set *pbLockTaken=true - mov byte ptr [edi],1 - pop esi - pop edi - pop ebx - ret - -MonReliableEnterPrepareToWaitThinLock: - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr g_SystemInfo+SYSTEM_INFO_dwNumberOfProcessors,1 - jle MonReliableEnterFramedLockHelper - - ; exponential backoff: delay by approximately 2*ebx clock cycles (on a PIII) - mov eax, ebx -MonReliableEnterdelayLoopThinLock: - $repnop ; indicate to the CPU that we are spin waiting (useful for some Intel P4 multiprocs) - dec eax - jnz MonReliableEnterdelayLoopThinLock - - ; next time, wait a factor longer - imul ebx, dword ptr g_SpinConstants+SpinConstants_dwBackoffFactor - - cmp ebx, dword ptr g_SpinConstants+SpinConstants_dwMaximumDuration - jle MonReliableEnterRetryHelperThinLock - - jmp MonReliableEnterFramedLockHelper - -MonReliableEnterRetryHelperThinLock: - jmp MonReliableEnterRetryThinLock - -MonReliableEnterHaveHashOrSyncBlockIndex: - ; If we have a hash code already, we need to create a sync block - test eax, BIT_SBLK_IS_HASHCODE_ASM - jnz MonReliableEnterFramedLockHelper - - ; Ok, we have a sync block index - just and out the top bits and grab the syncblock index - and eax, MASK_SYNCBLOCKINDEX_ASM - - ; Get the sync block pointer. - mov ARGUMENT_REG2, dword ptr g_pSyncTable - mov ARGUMENT_REG2, [ARGUMENT_REG2+eax*SizeOfSyncTableEntry_ASM+SyncTableEntry_m_SyncBlock] - - ; Check if the sync block has been allocated. - test ARGUMENT_REG2, ARGUMENT_REG2 - jz MonReliableEnterFramedLockHelper - - ; Get a pointer to the lock object. - lea ARGUMENT_REG2, [ARGUMENT_REG2+SyncBlock_m_Monitor] - - ; Attempt to acquire the lock. -MonReliableEnterRetrySyncBlock: - mov eax, [ARGUMENT_REG2+AwareLock_m_MonitorHeld] - test eax,eax - jne MonReliableEnterHaveWaiters - - ; Common case, lock isn't held and there are no waiters. Attempt to - ; gain ownership ourselves. - mov ARGUMENT_REG1,1 - lock cmpxchg [ARGUMENT_REG2+AwareLock_m_MonitorHeld], ARGUMENT_REG1 - jnz MonReliableEnterRetryHelperSyncBlock - - ; Success. Save the thread object in the lock and increment the use count. - mov dword ptr [ARGUMENT_REG2+AwareLock_m_HoldingThread],esi - inc dword ptr [esi+Thread_m_dwLockCount] - inc dword ptr [ARGUMENT_REG2+AwareLock_m_Recursion] - ; Set *pbLockTaken=true - mov byte ptr [edi],1 - -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG2 ; AwareLock - push [esp+4] ; return address - call EnterSyncHelper -endif ;TRACK_SYNC -endif ;MON_DEBUG - pop esi - pop edi - pop ebx - ret - - ; It's possible to get here with waiters but no lock held, but in this - ; case a signal is about to be fired which will wake up a waiter. So - ; for fairness sake we should wait too. - ; Check first for recursive lock attempts on the same thread. -MonReliableEnterHaveWaiters: - ; Is mutex already owned by current thread? - cmp [ARGUMENT_REG2+AwareLock_m_HoldingThread],esi - jne MonReliableEnterPrepareToWait - - ; Yes, bump our use count. - inc dword ptr [ARGUMENT_REG2+AwareLock_m_Recursion] - ; Set *pbLockTaken=true - mov byte ptr [edi],1 -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG2 ; AwareLock - push [esp+4] ; return address - call EnterSyncHelper -endif ;TRACK_SYNC -endif ;MON_DEBUG - pop esi - pop edi - pop ebx - ret - -MonReliableEnterPrepareToWait: - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr g_SystemInfo+SYSTEM_INFO_dwNumberOfProcessors,1 - jle MonReliableEnterHaveWaiters1 - - ; exponential backoff: delay by approximately 2*ebx clock cycles (on a PIII) - mov eax,ebx -MonReliableEnterdelayLoop: - $repnop ; indicate to the CPU that we are spin waiting (useful for some Intel P4 multiprocs) - dec eax - jnz MonReliableEnterdelayLoop - - ; next time, wait a factor longer - imul ebx, dword ptr g_SpinConstants+SpinConstants_dwBackoffFactor - - cmp ebx, dword ptr g_SpinConstants+SpinConstants_dwMaximumDuration - jle MonReliableEnterRetrySyncBlock - -MonReliableEnterHaveWaiters1: - - ; Place AwareLock in arg1, pbLockTaken in arg2, then call contention helper. - mov ARGUMENT_REG1, ARGUMENT_REG2 - mov ARGUMENT_REG2, edi - - pop esi - pop edi - pop ebx - - jmp JITutil_MonReliableContention - -MonReliableEnterRetryHelperSyncBlock: - jmp MonReliableEnterRetrySyncBlock - - ; ECX has the object to synchronize on -MonReliableEnterFramedLockHelper: - mov ARGUMENT_REG2, edi - pop esi - pop edi - pop ebx - jmp JITutil_MonReliableEnter - -@JIT_MonReliableEnter@8 endp - -;************************************************************************ -; This is a frameless helper for trying to enter a monitor on a object. -; The object is in ARGUMENT_REG1 and a timeout in ARGUMENT_REG2. This tries the -; normal case (no object allocation) in line and calls a framed helper for the -; other cases. -; ***** NOTE: if you make any changes to this routine, build with MON_DEBUG undefined -; to make sure you don't break the non-debug build. This is very fragile code. -; Also, propagate the changes to jithelp.s which contains the same helper and assembly code -; (in AT&T syntax) for gnu assembler. -@JIT_MonTryEnter@12 proc public - ; Save the timeout parameter. - push ARGUMENT_REG2 - - ; Initialize delay value for retry with exponential backoff - push ebx - mov ebx, dword ptr g_SpinConstants+SpinConstants_dwInitialDuration - - ; The thin lock logic needs another register to store the thread - push esi - - ; Check if the instance is NULL. - test ARGUMENT_REG1, ARGUMENT_REG1 - jz MonTryEnterFramedLockHelper - - ; Check if the timeout looks valid - cmp ARGUMENT_REG2,-1 - jl MonTryEnterFramedLockHelper - - ; Get the thread right away, we'll need it in any case - call _GetThread@0 - mov esi,eax - - ; Check if we can abort here - mov eax, [esi+Thread_m_State] - and eax, TS_CatchAtSafePoint_ASM - jz MonTryEnterRetryThinLock - ; go through the slow code path to initiate ThreadAbort. - jmp MonTryEnterFramedLockHelper - -MonTryEnterRetryThinLock: - ; Get the header dword and check its layout - mov eax, [ARGUMENT_REG1-SyncBlockIndexOffset_ASM] - - ; Check whether we have the "thin lock" layout, the lock is free and the spin lock bit not set - ; SBLK_COMBINED_MASK_ASM = BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK + SBLK_MASK_LOCK_THREADID + SBLK_MASK_LOCK_RECLEVEL - test eax, SBLK_COMBINED_MASK_ASM - jnz MonTryEnterNeedMoreTests - - ; Ok, everything is fine. Fetch the thread id and make sure it's small enough for thin locks - mov edx, [esi+Thread_m_ThreadId] - cmp edx, SBLK_MASK_LOCK_THREADID_ASM - ja MonTryEnterFramedLockHelper - - ; Try to put our thread id in there - or edx,eax - lock cmpxchg [ARGUMENT_REG1-SyncBlockIndexOffset_ASM],edx - jnz MonTryEnterRetryHelperThinLock - - ; Got the lock - everything is fine" - add [esi+Thread_m_dwLockCount],1 - pop esi - - ; Delay value no longer needed - pop ebx - - ; Timeout parameter not needed, ditch it from the stack. - add esp,4 - - mov eax, [esp+4] - mov byte ptr [eax], 1 - ret 4 - -MonTryEnterNeedMoreTests: - ; Ok, it's not the simple case - find out which case it is - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX_ASM - jnz MonTryEnterHaveSyncBlockIndexOrHash - - ; The header is transitioning or the lock is taken - test eax, BIT_SBLK_SPIN_LOCK_ASM - jnz MonTryEnterRetryHelperThinLock - - mov edx, eax - and edx, SBLK_MASK_LOCK_THREADID_ASM - cmp edx, [esi+Thread_m_ThreadId] - jne MonTryEnterPrepareToWaitThinLock - - ; Ok, the thread id matches, it's the recursion case. - ; Bump up the recursion level and check for overflow - lea edx, [eax+SBLK_LOCK_RECLEVEL_INC_ASM] - test edx, SBLK_MASK_LOCK_RECLEVEL_ASM - jz MonTryEnterFramedLockHelper - - ; Try to put the new recursion level back. If the header was changed in the meantime, - ; we need a full retry, because the layout could have changed. - lock cmpxchg [ARGUMENT_REG1-SyncBlockIndexOffset_ASM],edx - jnz MonTryEnterRetryHelperThinLock - - ; Everything went fine and we're done - pop esi - pop ebx - - ; Timeout parameter not needed, ditch it from the stack. - add esp, 4 - mov eax, [esp+4] - mov byte ptr [eax], 1 - ret 4 - -MonTryEnterPrepareToWaitThinLock: - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr g_SystemInfo+SYSTEM_INFO_dwNumberOfProcessors,1 - jle MonTryEnterFramedLockHelper - - ; exponential backoff: delay by approximately 2*ebx clock cycles (on a PIII) - mov eax, ebx -MonTryEnterdelayLoopThinLock: - $repnop ; indicate to the CPU that we are spin waiting (useful for some Intel P4 multiprocs) - dec eax - jnz MonTryEnterdelayLoopThinLock - - ; next time, wait a factor longer - imul ebx, dword ptr g_SpinConstants+SpinConstants_dwBackoffFactor - - cmp ebx, dword ptr g_SpinConstants+SpinConstants_dwMaximumDuration - jle MonTryEnterRetryHelperThinLock - - jmp MonTryEnterWouldBlock - -MonTryEnterRetryHelperThinLock: - jmp MonTryEnterRetryThinLock - - -MonTryEnterHaveSyncBlockIndexOrHash: - ; If we have a hash code already, we need to create a sync block - test eax, BIT_SBLK_IS_HASHCODE_ASM - jnz MonTryEnterFramedLockHelper - - ; Just and out the top bits and grab the syncblock index - and eax, MASK_SYNCBLOCKINDEX_ASM - - ; Get the sync block pointer. - mov ARGUMENT_REG2, dword ptr g_pSyncTable - mov ARGUMENT_REG2, [ARGUMENT_REG2+eax*SizeOfSyncTableEntry_ASM+SyncTableEntry_m_SyncBlock] - - ; Check if the sync block has been allocated. - test ARGUMENT_REG2, ARGUMENT_REG2 - jz MonTryEnterFramedLockHelper - - ; Get a pointer to the lock object. - lea ARGUMENT_REG2, [ARGUMENT_REG2+SyncBlock_m_Monitor] - -MonTryEnterRetrySyncBlock: - ; Attempt to acquire the lock. - mov eax, [ARGUMENT_REG2+AwareLock_m_MonitorHeld] - test eax,eax - jne MonTryEnterHaveWaiters - - ; We need another scratch register for what follows, so save EBX now so" - ; we can use it for that purpose." - push ebx - - ; Common case, lock isn't held and there are no waiters. Attempt to - ; gain ownership ourselves. - mov ebx,1 - lock cmpxchg [ARGUMENT_REG2+AwareLock_m_MonitorHeld],ebx - - pop ebx - - jnz MonTryEnterRetryHelperSyncBlock - - ; Success. Save the thread object in the lock and increment the use count. - mov dword ptr [ARGUMENT_REG2+AwareLock_m_HoldingThread],esi - inc dword ptr [ARGUMENT_REG2+AwareLock_m_Recursion] - inc dword ptr [esi+Thread_m_dwLockCount] - -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG2 ; AwareLock - push [esp+4] ; return address - call EnterSyncHelper -endif ;TRACK_SYNC -endif ;MON_DEBUG - - pop esi - pop ebx - - ; Timeout parameter not needed, ditch it from the stack." - add esp,4 - - mov eax, [esp+4] - mov byte ptr [eax], 1 - ret 4 - - ; It's possible to get here with waiters but no lock held, but in this - ; case a signal is about to be fired which will wake up a waiter. So - ; for fairness sake we should wait too. - ; Check first for recursive lock attempts on the same thread. -MonTryEnterHaveWaiters: - ; Is mutex already owned by current thread? - cmp [ARGUMENT_REG2+AwareLock_m_HoldingThread],esi - jne MonTryEnterPrepareToWait - - ; Yes, bump our use count. - inc dword ptr [ARGUMENT_REG2+AwareLock_m_Recursion] -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG2 ; AwareLock - push [esp+4] ; return address - call EnterSyncHelper -endif ;TRACK_SYNC -endif ;MON_DEBUG - pop esi - pop ebx - - ; Timeout parameter not needed, ditch it from the stack. - add esp,4 - - mov eax, [esp+4] - mov byte ptr [eax], 1 - ret 4 - -MonTryEnterPrepareToWait: - ; If we are on an MP system, we try spinning for a certain number of iterations - cmp dword ptr g_SystemInfo+SYSTEM_INFO_dwNumberOfProcessors,1 - jle MonTryEnterWouldBlock - - ; exponential backoff: delay by approximately 2*ebx clock cycles (on a PIII) - mov eax, ebx -MonTryEnterdelayLoop: - $repnop ; indicate to the CPU that we are spin waiting (useful for some Intel P4 multiprocs) - dec eax - jnz MonTryEnterdelayLoop - - ; next time, wait a factor longer - imul ebx, dword ptr g_SpinConstants+SpinConstants_dwBackoffFactor - - cmp ebx, dword ptr g_SpinConstants+SpinConstants_dwMaximumDuration - jle MonTryEnterRetrySyncBlock - - ; We would need to block to enter the section. Return failure if - ; timeout is zero, else call the framed helper to do the blocking - ; form of TryEnter." -MonTryEnterWouldBlock: - pop esi - pop ebx - pop ARGUMENT_REG2 - test ARGUMENT_REG2, ARGUMENT_REG2 - jnz MonTryEnterBlock - mov eax, [esp+4] - mov byte ptr [eax], 0 - ret 4 - -MonTryEnterRetryHelperSyncBlock: - jmp MonTryEnterRetrySyncBlock - -MonTryEnterFramedLockHelper: - ; ARGUMENT_REG1 has the object to synchronize on, must retrieve the - ; timeout parameter from the stack. - pop esi - pop ebx - pop ARGUMENT_REG2 -MonTryEnterBlock: - jmp JITutil_MonTryEnter - -@JIT_MonTryEnter@12 endp - -;********************************************************************** -; This is a frameless helper for exiting a monitor on a object. -; The object is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; ***** NOTE: if you make any changes to this routine, build with MON_DEBUG undefined -; to make sure you don't break the non-debug build. This is very fragile code. -; Also, propagate the changes to jithelp.s which contains the same helper and assembly code -; (in AT&T syntax) for gnu assembler. -@JIT_MonExitWorker@4 proc public - ; The thin lock logic needs an additional register to hold the thread, unfortunately - push esi - - ; Check if the instance is NULL. - test ARGUMENT_REG1, ARGUMENT_REG1 - jz MonExitFramedLockHelper - - call _GetThread@0 - mov esi,eax - -MonExitRetryThinLock: - ; Fetch the header dword and check its layout and the spin lock bit - mov eax, [ARGUMENT_REG1-SyncBlockIndexOffset_ASM] - ;BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX_SPIN_LOCK_ASM = BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_SPIN_LOCK - test eax, BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX_SPIN_LOCK_ASM - jnz MonExitNeedMoreTests - - ; Ok, we have a "thin lock" layout - check whether the thread id matches - mov edx,eax - and edx, SBLK_MASK_LOCK_THREADID_ASM - cmp edx, [esi+Thread_m_ThreadId] - jne MonExitFramedLockHelper - - ; Check the recursion level - test eax, SBLK_MASK_LOCK_RECLEVEL_ASM - jne MonExitDecRecursionLevel - - ; It's zero - we're leaving the lock. - ; So try to put back a zero thread id. - ; edx and eax match in the thread id bits, and edx is zero elsewhere, so the xor is sufficient - xor edx,eax - lock cmpxchg [ARGUMENT_REG1-SyncBlockIndexOffset_ASM],edx - jnz MonExitRetryHelperThinLock - - ; We're done - sub [esi+Thread_m_dwLockCount],1 - pop esi - ret - -MonExitDecRecursionLevel: - lea edx, [eax-SBLK_LOCK_RECLEVEL_INC_ASM] - lock cmpxchg [ARGUMENT_REG1-SyncBlockIndexOffset_ASM],edx - jnz MonExitRetryHelperThinLock - - ; We're done - pop esi - ret - -MonExitNeedMoreTests: - ;Forward all special cases to the slow helper - ;BIT_SBLK_IS_HASHCODE_OR_SPIN_LOCK_ASM = BIT_SBLK_IS_HASHCODE + BIT_SBLK_SPIN_LOCK - test eax, BIT_SBLK_IS_HASHCODE_OR_SPIN_LOCK_ASM - jnz MonExitFramedLockHelper - - ; Get the sync block index and use it to compute the sync block pointer - mov ARGUMENT_REG2, dword ptr g_pSyncTable - and eax, MASK_SYNCBLOCKINDEX_ASM - mov ARGUMENT_REG2, [ARGUMENT_REG2+eax*SizeOfSyncTableEntry_ASM+SyncTableEntry_m_SyncBlock] - - ; was there a sync block? - test ARGUMENT_REG2, ARGUMENT_REG2 - jz MonExitFramedLockHelper - - ; Get a pointer to the lock object. - lea ARGUMENT_REG2, [ARGUMENT_REG2+SyncBlock_m_Monitor] - - ; Check if lock is held. - cmp [ARGUMENT_REG2+AwareLock_m_HoldingThread],esi - jne MonExitFramedLockHelper - -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG1 ; preserve regs - push ARGUMENT_REG2 - - push ARGUMENT_REG2 ; AwareLock - push [esp+8] ; return address - call LeaveSyncHelper - - pop ARGUMENT_REG2 ; restore regs - pop ARGUMENT_REG1 -endif ;TRACK_SYNC -endif ;MON_DEBUG - ; Reduce our recursion count. - dec dword ptr [ARGUMENT_REG2+AwareLock_m_Recursion] - jz MonExitLastRecursion - - pop esi - ret - -MonExitRetryHelperThinLock: - jmp MonExitRetryThinLock - -MonExitFramedLockHelper: - pop esi - jmp JITutil_MonExitWorker - - ; This is the last count we held on this lock, so release the lock. -MonExitLastRecursion: - dec dword ptr [esi+Thread_m_dwLockCount] - mov dword ptr [ARGUMENT_REG2+AwareLock_m_HoldingThread],0 - -MonExitRetry: - mov eax, [ARGUMENT_REG2+AwareLock_m_MonitorHeld] - lea esi, [eax-1] - lock cmpxchg [ARGUMENT_REG2+AwareLock_m_MonitorHeld], esi - jne MonExitRetryHelper - pop esi - test eax,0FFFFFFFEh - jne MonExitMustSignal - - ret - -MonExitMustSignal: - mov ARGUMENT_REG1, ARGUMENT_REG2 - jmp JITutil_MonSignal - -MonExitRetryHelper: - jmp MonExitRetry - -@JIT_MonExitWorker@4 endp - -;********************************************************************** -; This is a frameless helper for entering a static monitor on a class. -; The methoddesc is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; Note we are changing the methoddesc parameter to a pointer to the -; AwareLock. -; ***** NOTE: if you make any changes to this routine, build with MON_DEBUG undefined -; to make sure you don't break the non-debug build. This is very fragile code. -; Also, propagate the changes to jithelp.s which contains the same helper and assembly code -; (in AT&T syntax) for gnu assembler. -@JIT_MonEnterStatic@4 proc public - ; We need another scratch register for what follows, so save EBX now so - ; we can use it for that purpose. - push ebx - - ; Attempt to acquire the lock -MonEnterStaticRetry: - mov eax, [ARGUMENT_REG1+AwareLock_m_MonitorHeld] - test eax,eax - jne MonEnterStaticHaveWaiters - - ; Common case, lock isn't held and there are no waiters. Attempt to - ; gain ownership ourselves. - mov ebx,1 - lock cmpxchg [ARGUMENT_REG1+AwareLock_m_MonitorHeld],ebx - jnz MonEnterStaticRetryHelper - - pop ebx - - ; Success. Save the thread object in the lock and increment the use count. - call _GetThread@0 - mov [ARGUMENT_REG1+AwareLock_m_HoldingThread], eax - inc dword ptr [ARGUMENT_REG1+AwareLock_m_Recursion] - inc dword ptr [eax+Thread_m_dwLockCount] - -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG1 ; AwareLock - push [esp+4] ; return address - call EnterSyncHelper -endif ;TRACK_SYNC -endif ;MON_DEBUG - ret - - ; It's possible to get here with waiters but no lock held, but in this - ; case a signal is about to be fired which will wake up a waiter. So - ; for fairness sake we should wait too. - ; Check first for recursive lock attempts on the same thread. -MonEnterStaticHaveWaiters: - ; Get thread but preserve EAX (contains cached contents of m_MonitorHeld). - push eax - call _GetThread@0 - mov ebx,eax - pop eax - - ; Is mutex already owned by current thread? - cmp [ARGUMENT_REG1+AwareLock_m_HoldingThread],ebx - jne MonEnterStaticPrepareToWait - - ; Yes, bump our use count. - inc dword ptr [ARGUMENT_REG1+AwareLock_m_Recursion] -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG1 ; AwareLock - push [esp+4] ; return address - call EnterSyncHelper -endif ;TRACK_SYNC -endif ;MON_DEBUG - pop ebx - ret - -MonEnterStaticPrepareToWait: - pop ebx - - ; ARGUMENT_REG1 should have AwareLock. Call contention helper. - jmp JITutil_MonContention - -MonEnterStaticRetryHelper: - jmp MonEnterStaticRetry -@JIT_MonEnterStatic@4 endp - -;********************************************************************** -; A frameless helper for exiting a static monitor on a class. -; The methoddesc is in ARGUMENT_REG1. This tries the normal case (no -; blocking or object allocation) in line and calls a framed helper -; for the other cases. -; Note we are changing the methoddesc parameter to a pointer to the -; AwareLock. -; ***** NOTE: if you make any changes to this routine, build with MON_DEBUG undefined -; to make sure you don't break the non-debug build. This is very fragile code. -; Also, propagate the changes to jithelp.s which contains the same helper and assembly code -; (in AT&T syntax) for gnu assembler. -@JIT_MonExitStatic@4 proc public - -ifdef MON_DEBUG -ifdef TRACK_SYNC - push ARGUMENT_REG1 ; preserve regs - - push ARGUMENT_REG1 ; AwareLock - push [esp+8] ; return address - call LeaveSyncHelper - - pop [ARGUMENT_REG1] ; restore regs -endif ;TRACK_SYNC -endif ;MON_DEBUG - - ; Check if lock is held. - call _GetThread@0 - cmp [ARGUMENT_REG1+AwareLock_m_HoldingThread],eax - jne MonExitStaticLockError - - ; Reduce our recursion count. - dec dword ptr [ARGUMENT_REG1+AwareLock_m_Recursion] - jz MonExitStaticLastRecursion - - ret - - ; This is the last count we held on this lock, so release the lock. -MonExitStaticLastRecursion: - ; eax must have the thread object - dec dword ptr [eax+Thread_m_dwLockCount] - mov dword ptr [ARGUMENT_REG1+AwareLock_m_HoldingThread],0 - push ebx - -MonExitStaticRetry: - mov eax, [ARGUMENT_REG1+AwareLock_m_MonitorHeld] - lea ebx, [eax-1] - lock cmpxchg [ARGUMENT_REG1+AwareLock_m_MonitorHeld],ebx - jne MonExitStaticRetryHelper - pop ebx - test eax,0FFFFFFFEh - jne MonExitStaticMustSignal - - ret - -MonExitStaticMustSignal: - jmp JITutil_MonSignal - -MonExitStaticRetryHelper: - jmp MonExitStaticRetry - ; Throw a synchronization lock exception. -MonExitStaticLockError: - mov ARGUMENT_REG1, CORINFO_SynchronizationLockException_ASM - jmp JIT_InternalThrow - -@JIT_MonExitStatic@4 endp - ; PatchedCodeStart and PatchedCodeEnd are used to determine bounds of patched code. ; diff --git a/src/vm/jithelpers.cpp b/src/vm/jithelpers.cpp index 32be77823ce1..8cff03ae60c5 100644 --- a/src/vm/jithelpers.cpp +++ b/src/vm/jithelpers.cpp @@ -4472,19 +4472,18 @@ HCIMPL_MONHELPER(JIT_MonEnterWorker_Portable, Object* obj) result = obj->EnterObjMonitorHelper(pCurThread); if (result == AwareLock::EnterHelperResult_Entered) { - MONHELPER_STATE(*pbLockTaken = 1;) + MONHELPER_STATE(*pbLockTaken = 1); return; } - else if (result == AwareLock::EnterHelperResult_Contention) { - AwareLock::EnterHelperResult resultSpin = obj->EnterObjMonitorHelperSpin(pCurThread); - if (resultSpin == AwareLock::EnterHelperResult_Entered) + result = obj->EnterObjMonitorHelperSpin(pCurThread); + if (result == AwareLock::EnterHelperResult_Entered) { - MONHELPER_STATE(*pbLockTaken = 1;) + MONHELPER_STATE(*pbLockTaken = 1); return; } - if (resultSpin == AwareLock::EnterHelperResult_Contention) + if (result == AwareLock::EnterHelperResult_Contention) { FC_INNER_RETURN_VOID(JIT_MonContention_Helper(obj, MONHELPER_ARG, GetEEFuncEntryPointMacro(JIT_MonEnter))); } @@ -4519,15 +4518,14 @@ HCIMPL1(void, JIT_MonEnter_Portable, Object* obj) { return; } - else if (result == AwareLock::EnterHelperResult_Contention) { - AwareLock::EnterHelperResult resultSpin = obj->EnterObjMonitorHelperSpin(pCurThread); - if (resultSpin == AwareLock::EnterHelperResult_Entered) + result = obj->EnterObjMonitorHelperSpin(pCurThread); + if (result == AwareLock::EnterHelperResult_Entered) { return; } - if (resultSpin == AwareLock::EnterHelperResult_Contention) + if (result == AwareLock::EnterHelperResult_Contention) { FC_INNER_RETURN_VOID(JIT_MonContention_Helper(obj, NULL, GetEEFuncEntryPointMacro(JIT_MonEnter))); } @@ -4563,16 +4561,15 @@ HCIMPL2(void, JIT_MonReliableEnter_Portable, Object* obj, BYTE* pbLockTaken) *pbLockTaken = 1; return; } - else if (result == AwareLock::EnterHelperResult_Contention) { - AwareLock::EnterHelperResult resultSpin = obj->EnterObjMonitorHelperSpin(pCurThread); - if (resultSpin == AwareLock::EnterHelperResult_Entered) + result = obj->EnterObjMonitorHelperSpin(pCurThread); + if (result == AwareLock::EnterHelperResult_Entered) { *pbLockTaken = 1; return; } - if (resultSpin == AwareLock::EnterHelperResult_Contention) + if (result == AwareLock::EnterHelperResult_Contention) { FC_INNER_RETURN_VOID(JIT_MonContention_Helper(obj, pbLockTaken, GetEEFuncEntryPointMacro(JIT_MonReliableEnter))); } @@ -4649,14 +4646,15 @@ HCIMPL3(void, JIT_MonTryEnter_Portable, Object* obj, INT32 timeOut, BYTE* pbLock *pbLockTaken = 1; return; } - else if (result == AwareLock::EnterHelperResult_Contention) { if (timeOut == 0) + { return; + } - AwareLock::EnterHelperResult resultSpin = obj->EnterObjMonitorHelperSpin(pCurThread); - if (resultSpin == AwareLock::EnterHelperResult_Entered) + result = obj->EnterObjMonitorHelperSpin(pCurThread); + if (result == AwareLock::EnterHelperResult_Entered) { *pbLockTaken = 1; return; @@ -4741,7 +4739,6 @@ FCIMPL1(void, JIT_MonExit_Portable, Object* obj) { return; } - else if (action == AwareLock::LeaveHelperAction_Signal) { FC_INNER_RETURN_VOID(JIT_MonExit_Signal(obj)); @@ -4773,7 +4770,6 @@ HCIMPL_MONHELPER(JIT_MonExitWorker_Portable, Object* obj) MONHELPER_STATE(*pbLockTaken = 0;) return; } - else if (action == AwareLock::LeaveHelperAction_Signal) { MONHELPER_STATE(*pbLockTaken = 0;) @@ -4821,7 +4817,7 @@ HCIMPL_MONHELPER(JIT_MonEnterStatic_Portable, AwareLock *lock) goto FramedLockHelper; } - if (lock->EnterHelper(pCurThread) == AwareLock::EnterHelperResult_Entered) + if (lock->EnterHelper(pCurThread, true /* checkRecursiveCase */)) { #if defined(_DEBUG) && defined(TRACK_SYNC) // The best place to grab this is from the ECall frame @@ -4909,289 +4905,6 @@ HCIMPL_MONHELPER(JIT_MonExitStatic_Portable, AwareLock *lock) HCIMPLEND #include -/*********************************************************************/ -// JITutil_Mon* are helpers than handle slow paths for JIT_Mon* methods -// implemented in assembly. They are not doing any spinning compared -// to the full fledged portable implementations above. -/*********************************************************************/ - -/*********************************************************************/ -HCIMPL_MONHELPER(JITutil_MonEnterWorker, Object* obj) -{ - CONTRACTL - { - FCALL_CHECK; - } - CONTRACTL_END; - - OBJECTREF objRef = ObjectToOBJECTREF(obj); - - // The following makes sure that Monitor.Enter shows up on thread abort - // stack walks (otherwise Monitor.Enter called within a CER can block a - // thread abort indefinitely). Setting the __me internal variable (normally - // only set for fcalls) will cause the helper frame below to be able to - // backtranslate into the method desc for the Monitor.Enter fcall. - // - // Note that we need explicitly initialize Monitor.Enter fcall in - // code:SystemDomain::LoadBaseSystemClasses to make this work in the case - // where the first call ever to Monitor.Enter is done as JIT helper - // for synchronized method. - __me = GetEEFuncEntryPointMacro(JIT_MonEnter); - - // Monitor helpers are used as both hcalls and fcalls, thus we need exact depth. - HELPER_METHOD_FRAME_BEGIN_ATTRIB_1(Frame::FRAME_ATTR_EXACT_DEPTH, objRef); - - if (objRef == NULL) - COMPlusThrow(kArgumentNullException); - - MONHELPER_STATE(GCPROTECT_BEGININTERIOR(pbLockTaken);) - -#ifdef _DEBUG - Thread *pThread = GetThread(); - DWORD lockCount = pThread->m_dwLockCount; -#endif - if (GET_THREAD()->CatchAtSafePointOpportunistic()) - { - GET_THREAD()->PulseGCMode(); - } - objRef->EnterObjMonitor(); - _ASSERTE ((objRef->GetSyncBlock()->GetMonitor()->m_Recursion == 1 && pThread->m_dwLockCount == lockCount + 1) || - pThread->m_dwLockCount == lockCount); - MONHELPER_STATE(if (pbLockTaken != 0) *pbLockTaken = 1;) - - MONHELPER_STATE(GCPROTECT_END();) - HELPER_METHOD_FRAME_END(); -} -HCIMPLEND - -/*********************************************************************/ - -// This helper is only ever used as part of FCall, but it is implemented using HCIMPL macro -// so that it can be tail called from assembly helper without triggering asserts in debug. -HCIMPL2(void, JITutil_MonReliableEnter, Object* obj, BYTE* pbLockTaken) -{ - CONTRACTL - { - FCALL_CHECK; - } - CONTRACTL_END; - - OBJECTREF objRef = ObjectToOBJECTREF(obj); - - // The following makes sure that Monitor.Enter shows up on thread abort - // stack walks (otherwise Monitor.Enter called within a CER can block a - // thread abort indefinitely). Setting the __me internal variable (normally - // only set for fcalls) will cause the helper frame below to be able to - // backtranslate into the method desc for the Monitor.Enter fcall. - __me = GetEEFuncEntryPointMacro(JIT_MonReliableEnter); - - // Monitor helpers are used as both hcalls and fcalls, thus we need exact depth. - HELPER_METHOD_FRAME_BEGIN_ATTRIB_1(Frame::FRAME_ATTR_EXACT_DEPTH, objRef); - - if (objRef == NULL) - COMPlusThrow(kArgumentNullException); - - GCPROTECT_BEGININTERIOR(pbLockTaken); - -#ifdef _DEBUG - Thread *pThread = GetThread(); - DWORD lockCount = pThread->m_dwLockCount; -#endif - if (GET_THREAD()->CatchAtSafePointOpportunistic()) - { - GET_THREAD()->PulseGCMode(); - } - objRef->EnterObjMonitor(); - _ASSERTE ((objRef->GetSyncBlock()->GetMonitor()->m_Recursion == 1 && pThread->m_dwLockCount == lockCount + 1) || - pThread->m_dwLockCount == lockCount); - *pbLockTaken = 1; - - GCPROTECT_END(); - HELPER_METHOD_FRAME_END(); -} -HCIMPLEND - - -/*********************************************************************/ - -// This helper is only ever used as part of FCall, but it is implemented using HCIMPL macro -// so that it can be tail called from assembly helper without triggering asserts in debug. -HCIMPL3(void, JITutil_MonTryEnter, Object* obj, INT32 timeOut, BYTE* pbLockTaken) -{ - CONTRACTL - { - FCALL_CHECK; - } - CONTRACTL_END; - - BOOL result = FALSE; - - OBJECTREF objRef = ObjectToOBJECTREF(obj); - - // The following makes sure that Monitor.TryEnter shows up on thread - // abort stack walks (otherwise Monitor.TryEnter called within a CER can - // block a thread abort for long periods of time). Setting the __me internal - // variable (normally only set for fcalls) will cause the helper frame below - // to be able to backtranslate into the method desc for the Monitor.TryEnter - // fcall. - __me = GetEEFuncEntryPointMacro(JIT_MonTryEnter); - - // Monitor helpers are used as both hcalls and fcalls, thus we need exact depth. - HELPER_METHOD_FRAME_BEGIN_ATTRIB_1(Frame::FRAME_ATTR_EXACT_DEPTH, objRef); - - if (objRef == NULL) - COMPlusThrow(kArgumentNullException); - - if (timeOut < -1) - COMPlusThrow(kArgumentOutOfRangeException); - - GCPROTECT_BEGININTERIOR(pbLockTaken); - - if (GET_THREAD()->CatchAtSafePointOpportunistic()) - { - GET_THREAD()->PulseGCMode(); - } - - result = objRef->TryEnterObjMonitor(timeOut); - *pbLockTaken = result != FALSE; - - GCPROTECT_END(); - HELPER_METHOD_FRAME_END(); -} -HCIMPLEND - -/*********************************************************************/ -HCIMPL_MONHELPER(JITutil_MonExitWorker, Object* obj) -{ - CONTRACTL - { - FCALL_CHECK; - } - CONTRACTL_END; - - MONHELPER_STATE(if (pbLockTaken != NULL && *pbLockTaken == 0) return;) - - OBJECTREF objRef = ObjectToOBJECTREF(obj); - - // Monitor helpers are used as both hcalls and fcalls, thus we need exact depth. - HELPER_METHOD_FRAME_BEGIN_ATTRIB_1(Frame::FRAME_ATTR_NO_THREAD_ABORT|Frame::FRAME_ATTR_EXACT_DEPTH, objRef); - - if (objRef == NULL) - COMPlusThrow(kArgumentNullException); - - if (!objRef->LeaveObjMonitor()) - COMPlusThrow(kSynchronizationLockException); - - MONHELPER_STATE(if (pbLockTaken != 0) *pbLockTaken = 0;) - - TESTHOOKCALL(AppDomainCanBeUnloaded(GET_THREAD()->GetDomain()->GetId().m_dwId,FALSE)); - - if (GET_THREAD()->IsAbortRequested()) { - GET_THREAD()->HandleThreadAbort(); - } - - HELPER_METHOD_FRAME_END(); -} -HCIMPLEND - -/*********************************************************************/ -// A helper for JIT_MonEnter that is on the callee side of an ecall -// frame and handles the contention case. - -HCIMPL_MONHELPER(JITutil_MonContention, AwareLock* lock) -{ - CONTRACTL - { - FCALL_CHECK; - } - CONTRACTL_END; - - // The following makes sure that Monitor.Enter shows up on thread abort - // stack walks (otherwise Monitor.Enter called within a CER can block a - // thread abort indefinitely). Setting the __me internal variable (normally - // only set for fcalls) will cause the helper frame below to be able to - // backtranslate into the method desc for the Monitor.Enter fcall. - __me = GetEEFuncEntryPointMacro(JIT_MonEnter); - - // Monitor helpers are used as both hcalls and fcalls, thus we need exact depth. - HELPER_METHOD_FRAME_BEGIN_ATTRIB(Frame::FRAME_ATTR_EXACT_DEPTH); - MONHELPER_STATE(GCPROTECT_BEGININTERIOR(pbLockTaken);) - -#ifdef _DEBUG - Thread *pThread = GetThread(); - DWORD lockCount = pThread->m_dwLockCount; -#endif - lock->Contention(); - _ASSERTE (pThread->m_dwLockCount == lockCount + 1); - MONHELPER_STATE(if (pbLockTaken != 0) *pbLockTaken = 1;) - - MONHELPER_STATE(GCPROTECT_END();) - HELPER_METHOD_FRAME_END(); -} -HCIMPLEND - -// This helper is only ever used as part of FCall, but it is implemented using HCIMPL macro -// so that it can be tail called from assembly helper without triggering asserts in debug. -HCIMPL2(void, JITutil_MonReliableContention, AwareLock* lock, BYTE* pbLockTaken) -{ - CONTRACTL - { - FCALL_CHECK; - } - CONTRACTL_END; - - // The following makes sure that Monitor.Enter shows up on thread abort - // stack walks (otherwise Monitor.Enter called within a CER can block a - // thread abort indefinitely). Setting the __me internal variable (normally - // only set for fcalls) will cause the helper frame below to be able to - // backtranslate into the method desc for the Monitor.Enter fcall. - __me = GetEEFuncEntryPointMacro(JIT_MonReliableEnter); - - // Monitor helpers are used as both hcalls and fcalls, thus we need exact depth. - HELPER_METHOD_FRAME_BEGIN_ATTRIB(Frame::FRAME_ATTR_EXACT_DEPTH); - GCPROTECT_BEGININTERIOR(pbLockTaken); - -#ifdef _DEBUG - Thread *pThread = GetThread(); - DWORD lockCount = pThread->m_dwLockCount; -#endif - lock->Contention(); - _ASSERTE (pThread->m_dwLockCount == lockCount + 1); - *pbLockTaken = 1; - - GCPROTECT_END(); - HELPER_METHOD_FRAME_END(); -} -HCIMPLEND - -/*********************************************************************/ -// A helper for JIT_MonExit and JIT_MonExitStatic that is on the -// callee side of an ecall frame and handles cases that might allocate, -// throw or block. -HCIMPL_MONHELPER(JITutil_MonSignal, AwareLock* lock) -{ - CONTRACTL - { - FCALL_CHECK; - } - CONTRACTL_END; - - // Monitor helpers are used as both hcalls and fcalls, thus we need exact depth. - HELPER_METHOD_FRAME_BEGIN_ATTRIB(Frame::FRAME_ATTR_EXACT_DEPTH | Frame::FRAME_ATTR_NO_THREAD_ABORT); - - lock->Signal(); - MONHELPER_STATE(if (pbLockTaken != 0) *pbLockTaken = 0;) - - TESTHOOKCALL(AppDomainCanBeUnloaded(GET_THREAD()->GetDomain()->GetId().m_dwId,FALSE)); - - if (GET_THREAD()->IsAbortRequested()) { - GET_THREAD()->HandleThreadAbort(); - } - - HELPER_METHOD_FRAME_END(); -} -HCIMPLEND - HCIMPL1(void *, JIT_GetSyncFromClassHandle, CORINFO_CLASS_HANDLE typeHnd_) CONTRACTL { FCALL_CHECK; diff --git a/src/vm/jitinterfacegen.cpp b/src/vm/jitinterfacegen.cpp index 26387405b745..38f1a7436a0e 100644 --- a/src/vm/jitinterfacegen.cpp +++ b/src/vm/jitinterfacegen.cpp @@ -47,15 +47,6 @@ EXTERN_C Object* JIT_NewArr1OBJ_UP (CORINFO_CLASS_HANDLE arrayMT, INT_PTR size); EXTERN_C Object* JIT_NewArr1VC_MP (CORINFO_CLASS_HANDLE arrayMT, INT_PTR size); EXTERN_C Object* JIT_NewArr1VC_UP (CORINFO_CLASS_HANDLE arrayMT, INT_PTR size); -//For the optimized JIT_Mon helpers -#if defined(_TARGET_AMD64_) -EXTERN_C void JIT_MonEnterWorker_Slow(Object* obj, BYTE* pbLockTaken); -EXTERN_C void JIT_MonExitWorker_Slow(Object* obj, BYTE* pbLockTaken); -EXTERN_C void JIT_MonTryEnter_Slow(Object* obj, INT32 timeOut, BYTE* pbLockTaken); -EXTERN_C void JIT_MonEnterStatic_Slow(AwareLock* lock, BYTE* pbLockTaken); -EXTERN_C void JIT_MonExitStatic_Slow(AwareLock* lock, BYTE* pbLockTaken); -#endif // _TARGET_AMD64_ - extern "C" void* JIT_GetSharedNonGCStaticBase_Slow(SIZE_T moduleDomainID, DWORD dwModuleClassID); extern "C" void* JIT_GetSharedNonGCStaticBaseNoCtor_Slow(SIZE_T moduleDomainID, DWORD dwModuleClassID); extern "C" void* JIT_GetSharedGCStaticBase_Slow(SIZE_T moduleDomainID, DWORD dwModuleClassID); @@ -114,12 +105,6 @@ EXTERN_C void JIT_BoxFastMPIGT__PatchTLSLabel(); EXTERN_C void AllocateStringFastMP_InlineGetThread__PatchTLSOffset(); EXTERN_C void JIT_NewArr1VC_MP_InlineGetThread__PatchTLSOffset(); EXTERN_C void JIT_NewArr1OBJ_MP_InlineGetThread__PatchTLSOffset(); -EXTERN_C void JIT_MonEnterWorker_InlineGetThread_GetThread_PatchLabel(); -EXTERN_C void JIT_MonExitWorker_InlineGetThread_GetThread_PatchLabel(); -EXTERN_C void JIT_MonTryEnter_GetThread_PatchLabel(); -EXTERN_C void JIT_MonEnterStaticWorker_InlineGetThread_GetThread_PatchLabel_1(); -EXTERN_C void JIT_MonEnterStaticWorker_InlineGetThread_GetThread_PatchLabel_2(); -EXTERN_C void JIT_MonExitStaticWorker_InlineGetThread_GetThread_PatchLabel(); static const LPVOID InlineGetThreadLocations[] = { @@ -128,12 +113,6 @@ static const LPVOID InlineGetThreadLocations[] = { (PVOID)AllocateStringFastMP_InlineGetThread__PatchTLSOffset, (PVOID)JIT_NewArr1VC_MP_InlineGetThread__PatchTLSOffset, (PVOID)JIT_NewArr1OBJ_MP_InlineGetThread__PatchTLSOffset, - (PVOID)JIT_MonEnterWorker_InlineGetThread_GetThread_PatchLabel, - (PVOID)JIT_MonExitWorker_InlineGetThread_GetThread_PatchLabel, - (PVOID)JIT_MonTryEnter_GetThread_PatchLabel, - (PVOID)JIT_MonEnterStaticWorker_InlineGetThread_GetThread_PatchLabel_1, - (PVOID)JIT_MonEnterStaticWorker_InlineGetThread_GetThread_PatchLabel_2, - (PVOID)JIT_MonExitStaticWorker_InlineGetThread_GetThread_PatchLabel, }; EXTERN_C void JIT_GetSharedNonGCStaticBase__PatchTLSLabel(); @@ -187,11 +166,6 @@ void FixupInlineGetters(DWORD tlsSlot, const LPVOID * pLocations, int nLocations } #endif // defined(_WIN64) && !defined(FEATURE_IMPLICIT_TLS) -#if defined(_TARGET_AMD64_) -EXTERN_C void JIT_MonEnterStaticWorker(); -EXTERN_C void JIT_MonExitStaticWorker(); -#endif - void InitJITHelpers1() { STANDARD_VM_CONTRACT; @@ -274,22 +248,6 @@ void InitJITHelpers1() } } -#ifndef FEATURE_IMPLICIT_TLS - if (gThreadTLSIndex >= TLS_MINIMUM_AVAILABLE) - { - // We need to patch the helpers for FCalls - MakeIntoJumpStub(JIT_MonEnterWorker_InlineGetThread, JIT_MonEnterWorker_Slow); - MakeIntoJumpStub(JIT_MonExitWorker_InlineGetThread, JIT_MonExitWorker_Slow); - MakeIntoJumpStub(JIT_MonTryEnter_InlineGetThread, JIT_MonTryEnter_Slow); - - SetJitHelperFunction(CORINFO_HELP_MON_ENTER, JIT_MonEnterWorker_Slow); - SetJitHelperFunction(CORINFO_HELP_MON_EXIT, JIT_MonExitWorker_Slow); - - SetJitHelperFunction(CORINFO_HELP_MON_ENTER_STATIC, JIT_MonEnterStatic_Slow); - SetJitHelperFunction(CORINFO_HELP_MON_EXIT_STATIC, JIT_MonExitStatic_Slow); - } -#endif - if(IsSingleAppDomain()) { SetJitHelperFunction(CORINFO_HELP_GETSHARED_GCSTATIC_BASE, JIT_GetSharedGCStaticBase_SingleAppDomain); diff --git a/src/vm/syncblk.cpp b/src/vm/syncblk.cpp index 50eec9b068a5..9c512675b6a0 100644 --- a/src/vm/syncblk.cpp +++ b/src/vm/syncblk.cpp @@ -1901,6 +1901,90 @@ BOOL ObjHeader::TryEnterObjMonitor(INT32 timeOut) return GetSyncBlock()->TryEnterMonitor(timeOut); } +AwareLock::EnterHelperResult ObjHeader::EnterObjMonitorHelperSpin(Thread* pCurThread) +{ + CONTRACTL{ + SO_TOLERANT; + NOTHROW; + GC_NOTRIGGER; + MODE_COOPERATIVE; + } CONTRACTL_END; + + // Note: EnterObjMonitorHelper must be called before this function (see below) + + if (g_SystemInfo.dwNumberOfProcessors == 1) + { + return AwareLock::EnterHelperResult_Contention; + } + + for (DWORD spinCount = g_SpinConstants.dwInitialDuration; AwareLock::SpinWaitAndBackOffBeforeOperation(&spinCount);) + { + LONG oldValue = m_SyncBlockValue.LoadWithoutBarrier(); + + // Since spinning has begun, chances are good that the monitor has already switched to AwareLock mode, so check for that + // case first + if (oldValue & BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) + { + // If we have a hash code already, we need to create a sync block + if (oldValue & BIT_SBLK_IS_HASHCODE) + { + return AwareLock::EnterHelperResult_UseSlowPath; + } + + // Check the recursive case once before the spin loop. If it's not the recursive case in the beginning, it will not + // be in the future, so the spin loop can avoid checking the recursive case. + SyncBlock *syncBlock = g_pSyncTable[oldValue & MASK_SYNCBLOCKINDEX].m_SyncBlock; + _ASSERTE(syncBlock != NULL); + AwareLock *awareLock = &syncBlock->m_Monitor; + if (awareLock->EnterHelper(pCurThread, true /* checkRecursiveCase */)) + { + return AwareLock::EnterHelperResult_Entered; + } + while (AwareLock::SpinWaitAndBackOffBeforeOperation(&spinCount)) + { + if (awareLock->EnterHelper(pCurThread, false /* checkRecursiveCase */)) + { + return AwareLock::EnterHelperResult_Entered; + } + } + break; + } + + DWORD tid = pCurThread->GetThreadId(); + if ((oldValue & (BIT_SBLK_SPIN_LOCK + + SBLK_MASK_LOCK_THREADID + + SBLK_MASK_LOCK_RECLEVEL)) == 0) + { + if (tid > SBLK_MASK_LOCK_THREADID) + { + return AwareLock::EnterHelperResult_UseSlowPath; + } + + LONG newValue = oldValue | tid; + if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) + { + pCurThread->IncLockCount(); + return AwareLock::EnterHelperResult_Entered; + } + + continue; + } + + // EnterObjMonitorHelper handles the thin lock recursion case. If it's not that case, it won't become that case. If + // EnterObjMonitorHelper failed to increment the recursion level, it will go down the slow path and won't come here. So, + // no need to check the recursion case here. + _ASSERTE( + // The header is transitioning - treat this as if the lock was taken + oldValue & BIT_SBLK_SPIN_LOCK || + // Here we know we have the "thin lock" layout, but the lock is not free. + // It can't be the recursion case though, because the call to EnterObjMonitorHelper prior to this would have taken + // the slow path in the recursive case. + tid != (DWORD)(oldValue & SBLK_MASK_LOCK_THREADID)); + } + + return AwareLock::EnterHelperResult_Contention; +} + BOOL ObjHeader::LeaveObjMonitor() { CONTRACTL diff --git a/src/vm/syncblk.h b/src/vm/syncblk.h index c6c63aefc7b0..9dcd57c49776 100644 --- a/src/vm/syncblk.h +++ b/src/vm/syncblk.h @@ -251,9 +251,10 @@ class AwareLock LeaveHelperAction_Error, }; + static bool SpinWaitAndBackOffBeforeOperation(DWORD *spinCountRef); + // Helper encapsulating the fast path entering monitor. Returns what kind of result was achieved. - AwareLock::EnterHelperResult EnterHelper(Thread* pCurThread); - AwareLock::EnterHelperResult EnterHelperSpin(Thread* pCurThread, INT32 timeOut = -1); + bool EnterHelper(Thread* pCurThread, bool checkRecursiveCase); // Helper encapsulating the core logic for leaving monitor. Returns what kind of // follow up action is necessary @@ -1265,8 +1266,11 @@ class ObjHeader // non-blocking version of above BOOL TryEnterObjMonitor(INT32 timeOut = 0); - // Inlineable fast path of EnterObjMonitor/TryEnterObjMonitor + // Inlineable fast path of EnterObjMonitor/TryEnterObjMonitor. Must be called before EnterObjMonitorHelperSpin. AwareLock::EnterHelperResult EnterObjMonitorHelper(Thread* pCurThread); + + // Typically non-inlined spin loop for some fast paths of EnterObjMonitor/TryEnterObjMonitor. EnterObjMonitorHelper must be + // called before this function. AwareLock::EnterHelperResult EnterObjMonitorHelperSpin(Thread* pCurThread); // leaves the monitor of an object diff --git a/src/vm/syncblk.inl b/src/vm/syncblk.inl index cb6b2802282f..376cd4c2e7c8 100644 --- a/src/vm/syncblk.inl +++ b/src/vm/syncblk.inl @@ -8,157 +8,141 @@ #ifndef DACCESS_COMPILE -FORCEINLINE AwareLock::EnterHelperResult AwareLock::EnterHelper(Thread* pCurThread) +FORCEINLINE bool AwareLock::SpinWaitAndBackOffBeforeOperation(DWORD *spinCountRef) { - CONTRACTL { + CONTRACTL{ SO_TOLERANT; NOTHROW; GC_NOTRIGGER; - MODE_ANY; + MODE_COOPERATIVE; } CONTRACTL_END; - for (;;) + _ASSERTE(spinCountRef != nullptr); + DWORD &spinCount = *spinCountRef; + _ASSERTE(g_SystemInfo.dwNumberOfProcessors != 1); + + if (spinCount > g_SpinConstants.dwMaximumDuration) { - LONG state = m_MonitorHeld.LoadWithoutBarrier(); + return false; + } - if (state == 0) - { - if (InterlockedCompareExchangeAcquire((LONG*)&m_MonitorHeld, 1, 0) == 0) - { - m_HoldingThread = pCurThread; - m_Recursion = 1; - pCurThread->IncLockCount(); - return AwareLock::EnterHelperResult_Entered; - } - } - else - { - if (GetOwningThread() == pCurThread) /* monitor is held, but it could be a recursive case */ - { - m_Recursion++; - return AwareLock::EnterHelperResult_Entered; - } + for (DWORD i = 0; i < spinCount; i++) + { + YieldProcessor(); + } + + spinCount *= g_SpinConstants.dwBackoffFactor; + return true; +} + +FORCEINLINE bool AwareLock::EnterHelper(Thread* pCurThread, bool checkRecursiveCase) +{ + CONTRACTL{ + SO_TOLERANT; + NOTHROW; + GC_NOTRIGGER; + MODE_ANY; + } CONTRACTL_END; - return AwareLock::EnterHelperResult_Contention; + LONG state = m_MonitorHeld.LoadWithoutBarrier(); + if (state == 0) + { + if (InterlockedCompareExchangeAcquire((LONG*)&m_MonitorHeld, 1, 0) == 0) + { + m_HoldingThread = pCurThread; + m_Recursion = 1; + pCurThread->IncLockCount(); + return true; } } + else if (checkRecursiveCase && GetOwningThread() == pCurThread) /* monitor is held, but it could be a recursive case */ + { + m_Recursion++; + return true; + } + return false; } FORCEINLINE AwareLock::EnterHelperResult ObjHeader::EnterObjMonitorHelper(Thread* pCurThread) { - CONTRACTL { + CONTRACTL{ SO_TOLERANT; NOTHROW; GC_NOTRIGGER; MODE_COOPERATIVE; } CONTRACTL_END; - DWORD tid = pCurThread->GetThreadId(); + LONG oldValue = m_SyncBlockValue.LoadWithoutBarrier(); - LONG oldvalue = m_SyncBlockValue.LoadWithoutBarrier(); - - if ((oldvalue & (BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + - BIT_SBLK_SPIN_LOCK + - SBLK_MASK_LOCK_THREADID + - SBLK_MASK_LOCK_RECLEVEL)) == 0) + if ((oldValue & (BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + + BIT_SBLK_SPIN_LOCK + + SBLK_MASK_LOCK_THREADID + + SBLK_MASK_LOCK_RECLEVEL)) == 0) { + DWORD tid = pCurThread->GetThreadId(); if (tid > SBLK_MASK_LOCK_THREADID) { return AwareLock::EnterHelperResult_UseSlowPath; } - LONG newvalue = oldvalue | tid; - if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newvalue, oldvalue) == oldvalue) + LONG newValue = oldValue | tid; + if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) { pCurThread->IncLockCount(); return AwareLock::EnterHelperResult_Entered; } - } - else - if (oldvalue & BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) - { - // If we have a hash code already, we need to create a sync block - if (oldvalue & BIT_SBLK_IS_HASHCODE) - { - return AwareLock::EnterHelperResult_UseSlowPath; - } - - SyncBlock *syncBlock = g_pSyncTable [oldvalue & MASK_SYNCBLOCKINDEX].m_SyncBlock; - _ASSERTE(syncBlock != NULL); - return syncBlock->m_Monitor.EnterHelper(pCurThread); + return AwareLock::EnterHelperResult_Contention; } - else + + if (oldValue & BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) { - // The header is transitioning - treat this as if the lock was taken - if (oldvalue & BIT_SBLK_SPIN_LOCK) + // If we have a hash code already, we need to create a sync block + if (oldValue & BIT_SBLK_IS_HASHCODE) { - return AwareLock::EnterHelperResult_Contention; + return AwareLock::EnterHelperResult_UseSlowPath; } - // Here we know we have the "thin lock" layout, but the lock is not free. - // It could still be the recursion case - compare the thread id to check - if (tid == (DWORD) (oldvalue & SBLK_MASK_LOCK_THREADID)) + SyncBlock *syncBlock = g_pSyncTable[oldValue & MASK_SYNCBLOCKINDEX].m_SyncBlock; + _ASSERTE(syncBlock != NULL); + if (syncBlock->m_Monitor.EnterHelper(pCurThread, true /* checkRecursiveCase */)) { - // Ok, the thread id matches, it's the recursion case. - // Bump up the recursion level and check for overflow - LONG newvalue = oldvalue + SBLK_LOCK_RECLEVEL_INC; - - if ((newvalue & SBLK_MASK_LOCK_RECLEVEL) == 0) - { - return AwareLock::EnterHelperResult_UseSlowPath; - } - - if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newvalue, oldvalue) == oldvalue) - { - return AwareLock::EnterHelperResult_Entered; - } + return AwareLock::EnterHelperResult_Entered; } - } - return AwareLock::EnterHelperResult_Contention; -} + return AwareLock::EnterHelperResult_Contention; + } -inline AwareLock::EnterHelperResult ObjHeader::EnterObjMonitorHelperSpin(Thread* pCurThread) -{ - CONTRACTL { - SO_TOLERANT; - NOTHROW; - GC_NOTRIGGER; - MODE_COOPERATIVE; - } CONTRACTL_END; + // The header is transitioning - treat this as if the lock was taken + if (oldValue & BIT_SBLK_SPIN_LOCK) + { + return AwareLock::EnterHelperResult_Contention; + } - if (1 == g_SystemInfo.dwNumberOfProcessors) + // Here we know we have the "thin lock" layout, but the lock is not free. + // It could still be the recursion case - compare the thread id to check + if (pCurThread->GetThreadId() != (DWORD)(oldValue & SBLK_MASK_LOCK_THREADID)) { return AwareLock::EnterHelperResult_Contention; } - DWORD spincount = g_SpinConstants.dwInitialDuration; + // Ok, the thread id matches, it's the recursion case. + // Bump up the recursion level and check for overflow + LONG newValue = oldValue + SBLK_LOCK_RECLEVEL_INC; - for (;;) + if ((newValue & SBLK_MASK_LOCK_RECLEVEL) == 0) { - // - // exponential backoff - // - for (DWORD i = 0; i < spincount; i++) - { - YieldProcessor(); - } - - AwareLock::EnterHelperResult result = EnterObjMonitorHelper(pCurThread); - if (result != AwareLock::EnterHelperResult_Contention) - { - return result; - } + return AwareLock::EnterHelperResult_UseSlowPath; + } - spincount *= g_SpinConstants.dwBackoffFactor; - if (spincount > g_SpinConstants.dwMaximumDuration) - { - break; - } + if (InterlockedCompareExchangeAcquire((LONG*)&m_SyncBlockValue, newValue, oldValue) == oldValue) + { + return AwareLock::EnterHelperResult_Entered; } - return AwareLock::EnterHelperResult_Contention; + // Use the slow path instead of spinning. The compare-exchange above would not fail often, and it's not worth forcing the + // spin loop that typically follows the call to this function to check the recursive case, so just bail to the slow path. + return AwareLock::EnterHelperResult_UseSlowPath; } // Helper encapsulating the core logic for releasing monitor. Returns what kind of @@ -185,31 +169,20 @@ FORCEINLINE AwareLock::LeaveHelperAction AwareLock::LeaveHelper(Thread* pCurThre pCurThread->m_pTrackSync->LeaveSync(caller, this); #endif - if (--m_Recursion != 0) - { - return AwareLock::LeaveHelperAction_None; - } - - m_HoldingThread->DecLockCount(); - m_HoldingThread = NULL; - - for (;;) + if (--m_Recursion == 0) { - // Read existing lock state - LONG state = m_MonitorHeld.LoadWithoutBarrier(); + m_HoldingThread->DecLockCount(); + m_HoldingThread = NULL; // Clear lock bit. - if (InterlockedCompareExchangeRelease((LONG*)&m_MonitorHeld, state - 1, state) == state) + LONG state = InterlockedDecrementRelease((LONG*)&m_MonitorHeld); + + // If wait count is non-zero on successful clear, we must signal the event. + if (state & ~1) { - // If wait count is non-zero on successful clear, we must signal the event. - if (state & ~1) - { - return AwareLock::LeaveHelperAction_Signal; - } - break; + return AwareLock::LeaveHelperAction_Signal; } } - return AwareLock::LeaveHelperAction_None; } @@ -234,34 +207,34 @@ FORCEINLINE AwareLock::LeaveHelperAction ObjHeader::LeaveObjMonitorHelper(Thread return AwareLock::LeaveHelperAction_Error; } - if (syncBlockValue & SBLK_MASK_LOCK_RECLEVEL) + if (!(syncBlockValue & SBLK_MASK_LOCK_RECLEVEL)) { - // recursion and ThinLock - DWORD newValue = syncBlockValue - SBLK_LOCK_RECLEVEL_INC; + // We are leaving the lock + DWORD newValue = (syncBlockValue & (~SBLK_MASK_LOCK_THREADID)); if (InterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue) { return AwareLock::LeaveHelperAction_Yield; } + pCurThread->DecLockCount(); } else { - // We are leaving the lock - DWORD newValue = (syncBlockValue & (~SBLK_MASK_LOCK_THREADID)); + // recursion and ThinLock + DWORD newValue = syncBlockValue - SBLK_LOCK_RECLEVEL_INC; if (InterlockedCompareExchangeRelease((LONG*)&m_SyncBlockValue, newValue, syncBlockValue) != (LONG)syncBlockValue) { return AwareLock::LeaveHelperAction_Yield; } - pCurThread->DecLockCount(); } return AwareLock::LeaveHelperAction_None; } - if ((syncBlockValue & (BIT_SBLK_SPIN_LOCK + BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX + BIT_SBLK_IS_HASHCODE)) == BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) + if ((syncBlockValue & (BIT_SBLK_SPIN_LOCK + BIT_SBLK_IS_HASHCODE)) == 0) { - SyncBlock *syncBlock = g_pSyncTable [syncBlockValue & MASK_SYNCBLOCKINDEX].m_SyncBlock; + _ASSERTE((syncBlockValue & BIT_SBLK_IS_HASH_OR_SYNCBLKINDEX) != 0); + SyncBlock *syncBlock = g_pSyncTable[syncBlockValue & MASK_SYNCBLOCKINDEX].m_SyncBlock; _ASSERTE(syncBlock != NULL); - return syncBlock->m_Monitor.LeaveHelper(pCurThread); }