Merge pull request eclipse-openj9#2298 from ebadyano/master

Initial changes to refactor snapshot at the beginning barrier.

runtime/compiler/control/J9Options.cpp

@@ -1106,7 +1106,8 @@ J9::Options::fePreProcess(void * base)
          case j9gc_modron_wrtbar_cardmark:               wrtbarMode = TR_WrtbarCardMark; break;
          case j9gc_modron_wrtbar_cardmark_and_oldcheck:  wrtbarMode = TR_WrtbarCardMarkAndOldCheck; break;
          case j9gc_modron_wrtbar_cardmark_incremental:   wrtbarMode = TR_WrtbarCardMarkIncremental; break;
-         case j9gc_modron_wrtbar_realtime:               wrtbarMode = TR_WrtbarRealTime; break;
+         case j9gc_modron_wrtbar_satb:
+         case j9gc_modron_wrtbar_satb_and_oldcheck:      wrtbarMode = TR_WrtbarRealTime; break;
          }
 
 #if defined(J9VM_GC_HEAP_CARD_TABLE)

runtime/compiler/env/VMJ9.cpp

@@ -1677,7 +1677,7 @@ TR_J9VMBase::thisThreadRememberedSetFragmentOffset()
    {
 
 #if defined(J9VM_GC_REALTIME)
-      return offsetof(J9VMThread, staccatoRememberedSetFragment);
+      return offsetof(J9VMThread, sATBBarrierRememberedSetFragment);
 #endif
    return 0;
    }

runtime/gc_base/modron.h

@@ -1,6 +1,6 @@
 
 /*******************************************************************************
- * Copyright (c) 1991, 2017 IBM Corp. and others
+ * Copyright (c) 1991, 2018 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -123,7 +123,7 @@ extern "C" fj9object_t j9gc_objaccess_tokenFromPointer(J9VMThread *vmThread, mm_
 
 #if defined(J9VM_GC_STACCATO)
 /* Note that the "reserved" index is used for 2 different purposes with the
- * staccatoRememberedSet:
+ * sATBBarrierRememberedSet:
  * 1) As a per-thread flag indicating the double barrier is on.
  * 2) As a global flag indicating the barrier is disabled.
  * 

runtime/gc_glue_java/ConcurrentMarkingDelegate.cpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2017 IBM Corp. and others
+ * Copyright (c) 1991, 2018 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -93,6 +93,37 @@ MM_ConcurrentMarkingDelegate::signalThreadsToTraceStacks(MM_EnvironmentBase *env
 	_collector->getConcurrentGCStats()->setThreadsToScanCount(threadCount);
 }
 
+void
+MM_ConcurrentMarkingDelegate::signalThreadsToActivateWriteBarrier(MM_EnvironmentBase *env)
+{
+	MM_GCExtensions *extensions = MM_GCExtensions::getExtensions(env);
+	GC_VMInterface::lockVMThreadList(extensions);
+
+	J9VMThread *walkThread;
+	GC_VMThreadListIterator vmThreadListIterator(_javaVM);
+	while ((walkThread = vmThreadListIterator.nextVMThread()) != NULL) {
+		walkThread->privateFlags |= J9_PRIVATE_FLAGS_CONCURRENT_MARK_ACTIVE;
+	}
+	GC_VMInterface::unlockVMThreadList(extensions);
+}
+
+void
+MM_ConcurrentMarkingDelegate::signalThreadsToDeactivateWriteBarrier(MM_EnvironmentBase *env)
+{
+	MM_GCExtensions *extensions = MM_GCExtensions::getExtensions(_javaVM);
+	if (extensions->optimizeConcurrentWB) {
+		GC_VMInterface::lockVMThreadList(extensions);
+		GC_VMThreadListIterator vmThreadListIterator(_javaVM);
+		J9VMThread *walkThread;
+
+		/* Reset vmThread flag so mutators don't dirty cards or run write barriers until next concurrent KO */
+		while ((walkThread = vmThreadListIterator.nextVMThread()) != NULL) {
+			walkThread->privateFlags &= ~J9_PRIVATE_FLAGS_CONCURRENT_MARK_ACTIVE;
+		}
+		GC_VMInterface::unlockVMThreadList(extensions);
+	}
+}
+
 void
 MM_ConcurrentMarkingDelegate::signalThreadsToDirtyCards(MM_EnvironmentBase *env)
 {

runtime/gc_glue_java/ConcurrentMarkingDelegate.hpp

@@ -1,5 +1,5 @@
 /*******************************************************************************
- * Copyright (c) 1991, 2017 IBM Corp. and others
+ * Copyright (c) 1991, 2018 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -245,10 +245,16 @@ class MM_ConcurrentMarkingDelegate
 	 */
 	void signalThreadsToTraceStacks(MM_EnvironmentBase *env);
 
+	void signalThreadsToActivateWriteBarrier(MM_EnvironmentBase *env);
+
+	void signalThreadsToDeactivateWriteBarrier(MM_EnvironmentBase *env);
+
+	//TODO: STAB remove two methods bellow after the initial changes are in
 	void signalThreadsToDirtyCards(MM_EnvironmentBase *env);
 
 	void signalThreadsToStopDirtyingCards(MM_EnvironmentBase *env);
 
+
 	/**
 	 * This method is called during card cleaning for each object associated with an uncleaned, dirty card in the card
 	 * table. No client actions are necessary but this method may be overridden if desired to hook into card cleaning.

runtime/gc_glue_java/ConfigurationDelegate.hpp

@@ -88,6 +88,7 @@ class MM_ConfigurationDelegate
 		if (extensions->alwaysCallWriteBarrier) {
 			writeBarrierType = gc_modron_wrtbar_always;
 		}
+
 		Assert_MM_true(gc_modron_wrtbar_illegal != writeBarrierType);
 		javaVM->gcWriteBarrierType = writeBarrierType;
 

runtime/gc_include/ObjectAccessBarrierAPI.hpp

@@ -2625,8 +2625,9 @@ class MM_ObjectAccessBarrierAPI
 	VMINLINE void
 	internalPreStoreObject(J9VMThread *vmThread, j9object_t object, fj9object_t *destAddress, j9object_t value)
 	{
-		if (j9gc_modron_wrtbar_realtime == _writeBarrierType) {
-			internalPreStoreObjectRealtime(vmThread, object, destAddress, value);
+		if ((j9gc_modron_wrtbar_satb == _writeBarrierType) ||
+				(j9gc_modron_wrtbar_satb_and_oldcheck == _writeBarrierType)) {
+			internalPreStoreObjectSATB(vmThread, object, destAddress, value);
 		}
 	}
 
@@ -2644,8 +2645,9 @@ class MM_ObjectAccessBarrierAPI
 	VMINLINE void
 	internalStaticPreStoreObject(J9VMThread *vmThread, j9object_t object, j9object_t *destAddress, j9object_t value)
 	{
-		if (j9gc_modron_wrtbar_realtime == _writeBarrierType) {
-			internalStaticPreStoreObjectRealtime(vmThread, object, destAddress, value);
+		if ((j9gc_modron_wrtbar_satb == _writeBarrierType) ||
+				(j9gc_modron_wrtbar_satb_and_oldcheck == _writeBarrierType)) {
+			internalStaticPreStoreObjectSATB(vmThread, object, destAddress, value);
 		}
 	}
 
@@ -2658,10 +2660,10 @@ class MM_ObjectAccessBarrierAPI
 	 *
 	 */
 	VMINLINE void
-	internalPreStoreObjectRealtime(J9VMThread *vmThread, j9object_t object, fj9object_t *destAddress, j9object_t value)
+	internalPreStoreObjectSATB(J9VMThread *vmThread, j9object_t object, fj9object_t *destAddress, j9object_t value)
 	{
 #if defined(J9VM_GC_REALTIME)
-		J9VMGCRememberedSetFragment *fragment =  &vmThread->staccatoRememberedSetFragment;
+		J9VMGCRememberedSetFragment *fragment =  &vmThread->sATBBarrierRememberedSetFragment;
 		J9VMGCRememberedSet *parent = fragment->fragmentParent;
 		/* Check if the barrier is enabled.  No work if barrier is not enabled */
 		if (0 != parent->globalFragmentIndex) {
@@ -2689,10 +2691,10 @@ class MM_ObjectAccessBarrierAPI
 	 *
 	 */
 	VMINLINE void
-	internalStaticPreStoreObjectRealtime(J9VMThread *vmThread, j9object_t object, j9object_t *destAddress, j9object_t value)
+	internalStaticPreStoreObjectSATB(J9VMThread *vmThread, j9object_t object, j9object_t *destAddress, j9object_t value)
 	{
 #if defined(J9VM_GC_REALTIME)
-		J9VMGCRememberedSetFragment *fragment =  &vmThread->staccatoRememberedSetFragment;
+		J9VMGCRememberedSetFragment *fragment =  &vmThread->sATBBarrierRememberedSetFragment;
 		J9VMGCRememberedSet *parent = fragment->fragmentParent;
 		/* Check if the barrier is enabled.  No work if barrier is not enabled */
 		if (0 != parent->globalFragmentIndex) {
@@ -2763,7 +2765,9 @@ class MM_ObjectAccessBarrierAPI
 			internalPostObjectStoreCardTable(vmThread, object, value);
 			break;
 		case j9gc_modron_wrtbar_none:
-		case j9gc_modron_wrtbar_realtime:
+		case j9gc_modron_wrtbar_satb:
+		case j9gc_modron_wrtbar_satb_and_oldcheck:
+			//TODO SATB change to handle gencon, decide where to do it in pre/post store
 			break;
 		default:
 			/* Should assert as all real types are handled.  Should never get here
@@ -2790,7 +2794,8 @@ class MM_ObjectAccessBarrierAPI
 			internalPostBatchStoreObjectCardTable(vmThread, object);
 			break;
 		case j9gc_modron_wrtbar_none:
-		case j9gc_modron_wrtbar_realtime:
+		case j9gc_modron_wrtbar_satb:
+		case j9gc_modron_wrtbar_satb_and_oldcheck:
 			break;
 		default:
 			/* Should assert as all real types are handled.  Should never get here

runtime/gc_include/j9modron.h

@@ -47,7 +47,8 @@ typedef enum {
 	j9gc_modron_wrtbar_cardmark = gc_modron_wrtbar_cardmark,
 	j9gc_modron_wrtbar_cardmark_incremental = gc_modron_wrtbar_cardmark_incremental,
 	j9gc_modron_wrtbar_cardmark_and_oldcheck = gc_modron_wrtbar_cardmark_and_oldcheck,
-	j9gc_modron_wrtbar_realtime = gc_modron_wrtbar_realtime,
+	j9gc_modron_wrtbar_satb = gc_modron_wrtbar_satb,
+	j9gc_modron_wrtbar_satb_and_oldcheck = gc_modron_wrtbar_satb_and_oldcheck,
 	j9gc_modron_wrtbar_count = gc_modron_wrtbar_count
 } J9WriteBarrierType;
 

runtime/gc_modron_standard/StandardAccessBarrier.cpp

@@ -63,6 +63,29 @@ MM_StandardAccessBarrier::newInstance(MM_EnvironmentBase *env)
 	return barrier;
 }
 
+/**
+ * Enables the double barrier on the provided thread.
+ */
+void
+MM_StandardAccessBarrier::setDoubleBarrierActiveOnThread(MM_EnvironmentBase* env)
+{
+	MM_GCExtensions::getExtensions(env)->sATBBarrierRememberedSet->preserveLocalFragmentIndex(env, &(((J9VMThread *)env->getLanguageVMThread())->sATBBarrierRememberedSetFragment));
+}
+
+void
+MM_StandardAccessBarrier::initializeForNewThread(MM_EnvironmentBase* env)
+{
+#if defined(OMR_GC_REALTIME)
+	if (_extensions->configuration->isSnapshotAtTheBeginningBarrierEnabled()) {
+
+		_extensions->sATBBarrierRememberedSet->initializeFragment(env, &(((J9VMThread *)env->getLanguageVMThread())->sATBBarrierRememberedSetFragment));
+		if (isDoubleBarrierActive()) {
+			setDoubleBarrierActiveOnThread(env);
+		}
+	}
+#endif /* OMR_GC_REALTIME */
+}
+
 bool 
 MM_StandardAccessBarrier::initialize(MM_EnvironmentBase *env)
 {
@@ -92,13 +115,122 @@ MM_StandardAccessBarrier::tearDown(MM_EnvironmentBase *env)
 	MM_ObjectAccessBarrier::tearDown(env);
 }
 
+/**
+ * Unmarked, heap reference, about to be deleted (or overwritten), while marking
+ * is in progress is to be remembered for later marking and scanning.
+ */
+void
+MM_StandardAccessBarrier::rememberObjectToRescan(MM_EnvironmentBase *env, J9Object *object)
+{
+	MM_ParallelGlobalGC *globalCollector = (MM_ParallelGlobalGC *)_extensions->getGlobalCollector();
+
+	if (globalCollector->getMarkingScheme()->markObject(env, object, true)) {
+		rememberObjectImpl(env, object);
+	}
+}
+
+/**
+ * Unmarked, heap reference, about to be deleted (or overwritten), while marking
+ * is in progress is to be remembered for later marking and scanning.
+ * This method is called by MM_StandardAccessBarrier::rememberObject()
+ */
+void
+MM_StandardAccessBarrier::rememberObjectImpl(MM_EnvironmentBase *env, J9Object* object)
+{
+	J9VMThread *vmThread = (J9VMThread *)env->getLanguageVMThread();
+	_extensions->sATBBarrierRememberedSet->storeInFragment(env, &vmThread->sATBBarrierRememberedSetFragment, (UDATA *)object);
+}
+
+
+bool
+MM_StandardAccessBarrier::preObjectStoreImpl(J9VMThread *vmThread, J9Object *destObject, fj9object_t *destAddress, J9Object *value, bool isVolatile)
+{
+	MM_EnvironmentBase* env = MM_EnvironmentBase::getEnvironment(vmThread->omrVMThread);
+
+	if (isSATBBarrierActive(env)) {
+		if (NULL != destObject) {
+			if (isDoubleBarrierActiveOnThread(vmThread)) {
+				rememberObjectToRescan(env, value);
+			}
+
+			J9Object *oldObject = NULL;
+			protectIfVolatileBefore(vmThread, isVolatile, true, false);
+			oldObject = mmPointerFromToken(vmThread, *destAddress);
+			protectIfVolatileAfter(vmThread, isVolatile, true, false);
+			rememberObjectToRescan(env, oldObject);
+		}
+	}
+
+	return true;
+}
+
+bool
+MM_StandardAccessBarrier::preObjectStoreImpl(J9VMThread *vmThread, J9Object **destAddress, J9Object *value, bool isVolatile)
+{
+	MM_EnvironmentBase* env = MM_EnvironmentBase::getEnvironment(vmThread->omrVMThread);
+
+	if (isSATBBarrierActive(env)) {
+		if (isDoubleBarrierActiveOnThread(vmThread)) {
+			rememberObjectToRescan(env, value);
+		}
+		J9Object* oldObject = NULL;
+		protectIfVolatileBefore(vmThread, isVolatile, true, false);
+		oldObject = *destAddress;
+		protectIfVolatileAfter(vmThread, isVolatile, true, false);
+		rememberObjectToRescan(env, oldObject);
+	}
+
+	return true;
+}
+/**
+ * @copydoc MM_ObjectAccessBarrier::preObjectStore()
+ *
+ * Metronome uses a snapshot-at-the-beginning algorithm, but with a fuzzy snapshot in the
+ * sense that threads are allowed to run during the root scan.  This requires a "double
+ * barrier."  The barrier is active from the start of root scanning through the end of
+ * tracing.  For an unscanned thread performing a store, the new value is remembered by
+ * the collector.  For any thread performing a store (whether scanned or not), the old
+ * value is remembered by the collector before being overwritten (thus this barrier must be
+ * positioned as a pre-store barrier).  For the latter ("Yuasa barrier") aspect of the
+ * double barrier, only the first overwritten value needs to be remembered (remembering
+ * others is harmless but not needed), and so we omit synchronization on the reading of the
+ * old value.
+ **/
+bool
+MM_StandardAccessBarrier::preObjectStore(J9VMThread *vmThread, J9Object *destObject, fj9object_t *destAddress, J9Object *value, bool isVolatile)
+{
+	return preObjectStoreImpl(vmThread, destObject, destAddress, value, isVolatile);
+}
+
+/**
+ * @copydoc MM_MetronomeAccessBarrier::preObjectStore()
+ *
+ * Used for stores into classes
+ */
+bool
+MM_StandardAccessBarrier::preObjectStore(J9VMThread *vmThread, J9Object *destClass, J9Object **destAddress, J9Object *value, bool isVolatile)
+{
+	return preObjectStoreImpl(vmThread, destAddress, value, isVolatile);
+}
+
+/**
+ * @copydoc MM_MetronomeAccessBarrier::preObjectStore()
+ *
+ * Used for stores into internal structures
+ */
+bool
+MM_StandardAccessBarrier::preObjectStore(J9VMThread *vmThread, J9Object **destAddress, J9Object *value, bool isVolatile)
+{
+	return preObjectStoreImpl(vmThread, destAddress, value, isVolatile);
+}
 /**
  * Called after an object is stored into another object.
  */
 void
 MM_StandardAccessBarrier::postObjectStore(J9VMThread *vmThread, J9Object *destObject, fj9object_t *destAddress, J9Object *value, bool isVolatile)
 {
 	postObjectStoreImpl(vmThread, destObject, value);
+
 }
 
 /**
@@ -159,9 +291,7 @@ MM_StandardAccessBarrier::postObjectStoreImpl(J9VMThread *vmThread, J9Object *ds
 
 #if defined(OMR_GC_MODRON_CONCURRENT_MARK)
 		/* Call the concurrent write barrier if required */
-		if(_extensions->concurrentMark &&
-				(vmThread->privateFlags & J9_PRIVATE_FLAGS_CONCURRENT_MARK_ACTIVE) &&
-				_extensions->isOld(dstObject)) {
+		if(isIncrementalUpdateBarrierActive(vmThread) && _extensions->isOld(dstObject)) {
 			J9ConcurrentWriteBarrierStore(vmThread->omrVMThread, dstObject, srcObject);
 		}
 #endif /* OMR_GC_MODRON_CONCURRENT_MARK */
@@ -481,6 +611,10 @@ I_32
 MM_StandardAccessBarrier::backwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots)
 {
 	I_32 retValue = ARRAY_COPY_NOT_DONE;
+	//TODO SATB re-enable opt?
+	if (_extensions->configuration->isSnapshotAtTheBeginningBarrierEnabled()) {
+		return retValue;
+	}
 
 	if(0 == lengthInSlots) {
 		retValue = ARRAY_COPY_SUCCESSFUL;
@@ -507,8 +641,13 @@ MM_StandardAccessBarrier::backwardReferenceArrayCopyIndex(J9VMThread *vmThread,
 I_32
 MM_StandardAccessBarrier::forwardReferenceArrayCopyIndex(J9VMThread *vmThread, J9IndexableObject *srcObject, J9IndexableObject *destObject, I_32 srcIndex, I_32 destIndex, I_32 lengthInSlots)
 {
+	//TODO SATB re-enable opt
 	I_32 retValue = ARRAY_COPY_NOT_DONE;
 
+	if (_extensions->configuration->isSnapshotAtTheBeginningBarrierEnabled()) {
+		return retValue;
+	}
+
 	if(0 == lengthInSlots) {
 		retValue = ARRAY_COPY_SUCCESSFUL;
 	} else {