WIP: Reuse free memory from GMP in PGC

runtime/gc_base/GCExtensions.hpp

@@ -187,6 +187,7 @@ class MM_GCExtensions : public MM_GCExtensionsBase {
 	double maxRAMPercent; /**< Value of -XX:MaxRAMPercentage specified by the user */
 	double initialRAMPercent; /**< Value of -XX:InitialRAMPercentage specified by the user */
 
+	bool tarokEnableRecoverRegionLargestFreeMemory; /**< Enable recovering region largest free memory during post sweep of GMP, this flag is only available if tarokEnableRecoverRegionTailsAfterSweep == true */
 protected:
 private:
 protected:
@@ -317,6 +318,7 @@ class MM_GCExtensions : public MM_GCExtensionsBase {
 #endif
 		, maxRAMPercent(0.0) /* this would get overwritten by user specified value */
 		, initialRAMPercent(0.0) /* this would get overwritten by user specified value */
+		, tarokEnableRecoverRegionLargestFreeMemory(false)
 	{
 		_typeId = __FUNCTION__;
 	}

runtime/gc_modron_startup/mmparseXXgc.cpp

@@ -358,6 +358,14 @@ gcParseXXgcArguments(J9JavaVM *vm, char *optArg)
 			extensions->tarokEnableExpensiveAssertions = false;
 			continue;
 		}
+		if (try_scan(&scan_start, "tarokEnableAllocationPointerAssertion")) {
+			extensions->tarokEnableAllocationPointerAssertion = true;
+			continue;
+		}
+		if (try_scan(&scan_start, "tarokDisableAllocationPointerAssertion")) {
+			extensions->tarokEnableAllocationPointerAssertion = false;
+			continue;
+		}
 		if (try_scan(&scan_start, "tarokTgcEnableRememberedSetDuplicateDetection")) {
 			extensions->tarokTgcEnableRememberedSetDuplicateDetection = true;
 			continue;
@@ -629,6 +637,16 @@ gcParseXXgcArguments(J9JavaVM *vm, char *optArg)
 
 			continue;
 		}
+		if (try_scan(&scan_start, "tarokEnableRecoverRegionTailsAfterSweep")) {
+			extensions->tarokEnableRecoverRegionTailsAfterSweep = true;
+			continue;
+		}
+
+		if (try_scan(&scan_start, "tarokEnableRecoverRegionLargestFreeMemory")) {
+			extensions->tarokEnableRecoverRegionLargestFreeMemory = true;
+			continue;
+		}
+
 #endif /* defined (J9VM_GC_VLHGC) */
 
 		if(try_scan(&scan_start, "packetListLockSplit=")) {

runtime/gc_vlhgc/CopyForwardScheme.hpp

@@ -99,6 +99,9 @@ class MM_CopyForwardScheme : public MM_BaseNonVirtual
 		MM_HeapRegionDescriptorVLHGC *_tailCandidates; /**< A linked list of regions in this compact group which have empty tails */
 		MM_LightweightNonReentrantLock _tailCandidatesLock; /**< Lock to protect _tailCandidates */
 		UDATA _tailCandidateCount; /**< The number of regions in the _tailCandidates list */
+		MM_HeapRegionDescriptorVLHGC *_largestFreeMemoryCandidates; /**< A linked list of regions in this compact group which have largest free memory */
+		MM_LightweightNonReentrantLock _largestFreeMemoryCandidatesLock; /**< Lock to protect _largestFreeMemoryCandidates */
+		UDATA _largestFreeMemoryCandidateCount; /**< The number of regions in the _largestFreeMemoryCandidates list */
 	protected:
 	private:
 		/* Methods */
@@ -177,6 +180,11 @@ class MM_CopyForwardScheme : public MM_BaseNonVirtual
 	void verifyExternalState(MM_EnvironmentVLHGC *env);
 	friend class MM_CopyForwardVerifyScanner;
 
+	void verifyObjectInRange(MM_EnvironmentVLHGC *env, UDATA *lowAddress, UDATA *highAddress);
+	void verifyChunkSlotsAndMapSlotsInRange(MM_EnvironmentVLHGC *env, UDATA *lowAddress, UDATA *highAddress);
+	void checkConsistencyGMPMapAndPGCMap(MM_EnvironmentVLHGC *env, MM_HeapRegionDescriptorVLHGC *region, UDATA *lowAddress, UDATA *highAddress);
+	void cleanInRange(MM_EnvironmentVLHGC *env, UDATA *lowAddress, UDATA *highAddress);
+
 	/**
 	 * Called to retire a copy cache once a thread no longer wants to use it as a copy destination.
 	 * @param env[in] A GC thread
@@ -354,8 +362,10 @@ class MM_CopyForwardScheme : public MM_BaseNonVirtual
 	 * @param env[in] the current thread
 	 * @param region[in] region to set as survivor
 	 * @param survivorBase[in] survivor base address (all object below this address, if different than low region address are not part of survivor)
+	 * @param survivorLow[in]  survior low address (for only case there is another survivor chunk beside tail)
+	 * @param survivorHigh[in] survior high address (for only case there is another survivor chunk beside tail)
 	 */
-	void setRegionAsSurvivor(MM_EnvironmentVLHGC* env, MM_HeapRegionDescriptorVLHGC *region, void* survivorBase);
+	void setRegionAsSurvivor(MM_EnvironmentVLHGC* env, MM_HeapRegionDescriptorVLHGC *region, void* survivorBase, void* survivorLow = NULL, void* survivorHigh = NULL);
 
 	/**
 	 * Process the list of reference objects recorded in the specified list.
@@ -512,7 +522,7 @@ class MM_CopyForwardScheme : public MM_BaseNonVirtual
 	 * @param regionList[in] The region list to which tailRegion should be added as a a tail candidate
 	 * @param tailRegion[in] The region to add
 	 */
-	void insertTailCandidate(MM_EnvironmentVLHGC* env, MM_ReservedRegionListHeader* regionList, MM_HeapRegionDescriptorVLHGC *tailRegion);
+	void insertCandidate(MM_EnvironmentVLHGC *env, MM_HeapRegionDescriptorVLHGC *&cadidatesListHead, UDATA &cadidatesCount,  MM_HeapRegionDescriptorVLHGC *region);
 
 	/**
 	 * Remove the specified tail candidate from the tail candidate list.  The implementation assumes that the calling thread can modify 
@@ -521,8 +531,7 @@ class MM_CopyForwardScheme : public MM_BaseNonVirtual
 	 * @param regionList[in] The region list which tailRegion belongs to
 	 * @param tailRegion[in] The region to remove
 	 */
-	void removeTailCandidate(MM_EnvironmentVLHGC* env, MM_ReservedRegionListHeader* regionList, MM_HeapRegionDescriptorVLHGC *tailRegion);
-
+	void removeCandidate(MM_EnvironmentVLHGC *env, MM_HeapRegionDescriptorVLHGC *&cadidatesListHead, UDATA &cadidatesCount,  MM_HeapRegionDescriptorVLHGC *region);
 	/**
 	 * Reserve memory for an object to be copied to survivor space.
 	 * @param env[in] GC thread.
@@ -921,13 +930,23 @@ class MM_CopyForwardScheme : public MM_BaseNonVirtual
 	/**
 	 * Align the specified memory pool so that it can be used for survivor objects.
 	 * Specifically, make sure that we can't be copying objects into the area covered by a card which is 
-	 * meant to describe objects which were already in the region.
+	 * meant to describe objects which were already in the region. align region tail only.
 	 * @param env[in] the current thread
 	 * @param memoryPool[in] the memoryPool to align
 	 * @return the number of bytes lost by aligning the pool
 	 */
 	UDATA alignMemoryPool(MM_EnvironmentVLHGC *env, MM_MemoryPoolBumpPointer *memoryPool);
 
+	/**
+	 * Align the specified memory pool so that it can be used for survivor objects.
+	 * Specifically, make sure that we can't be copying objects into the area covered by a card which is
+	 * meant to describe objects which were already in the region. align both region tail and the largest free memory.
+	 * @param env[in] the current thread
+	 * @param memoryPool[in] the memoryPool to align
+	 * @return the number of bytes lost by aligning the pool
+	 */
+	UDATA alignMemoryPool4Collector(MM_EnvironmentVLHGC *env, MM_MemoryPoolBumpPointer *memoryPool);
+
 	/**
 	 * Set the specified tail candidate region to be a survivor region.
 	 * 1. Set its _survivorBase to survivorBase (essentially set survivorSet to true).
@@ -938,7 +957,8 @@ class MM_CopyForwardScheme : public MM_BaseNonVirtual
 	 * @param region[in] the tail region to convert
 	 * @param survivorBase the lowest address in the region where survivor objects can be found
 	 */
-	void convertTailCandidateToSurvivorRegion(MM_EnvironmentVLHGC* env, MM_HeapRegionDescriptorVLHGC *region, void* survivorBase);
+	void convertCandidateToSurvivorRegion(MM_EnvironmentVLHGC* env, MM_HeapRegionDescriptorVLHGC *region, void* survivorBase, void* survivorLow, void* survivorHigh);
+
 
 	/**
 	 * Scan the root set, copying-and-forwarding any objects found.

runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.cpp

@@ -1,6 +1,6 @@
 
 /*******************************************************************************
- * Copyright (c) 1991, 2019 IBM Corp. and others
+ * Copyright (c) 1991, 2020 IBM Corp. and others
  *
  * This program and the accompanying materials are made available under
  * the terms of the Eclipse Public License 2.0 which accompanies this
@@ -81,6 +81,9 @@ MM_HeapRegionDescriptorVLHGC::initialize(MM_EnvironmentBase *env, MM_HeapRegionM
 	_copyForwardData._evacuateSet = false;
 	_copyForwardData._requiresPhantomReferenceProcessing = false;
 	_copyForwardData._survivorBase = NULL;
+	_copyForwardData._survivorLow = NULL;
+	_copyForwardData._survivorHigh = NULL;
+
 	_copyForwardData._nextRegion = NULL;
 	_copyForwardData._previousRegion = NULL;
 

runtime/gc_vlhgc/HeapRegionDescriptorVLHGC.hpp

@@ -68,6 +68,8 @@ class MM_HeapRegionDescriptorVLHGC : public MM_HeapRegionDescriptor
 		bool _evacuateSet;	/**< true if the region was part of the evacuate set at the beginning of the copy-forward (flag not changed during abort) */
 		bool _requiresPhantomReferenceProcessing; /**< Set to true by main thread if this region must be processed during parallel phantom reference processing */
 		volatile void *_survivorBase;  /**< The base pointer for storage used as survivor, which will NOT match the region base if tail filling has occurred */
+		volatile void *_survivorLow;   /**< The low pointer for storage used as survivor, which will NOT be NULL if the latest free memory(not be tail) filling has occurred */
+		volatile void *_survivorHigh;  /**< The high pointer for storage used as survivor, which will NOT be NULL if the latest free memory(not be tail) filling has occurred */
 		MM_HeapRegionDescriptorVLHGC *_nextRegion;  /**< Region list link for compact group resource management during a copyforward operation */
 		MM_HeapRegionDescriptorVLHGC *_previousRegion;  /**< Region list link for compact group resource management during a copyforward operation */
 	} _copyForwardData;
@@ -269,10 +271,15 @@ class MM_HeapRegionDescriptorVLHGC : public MM_HeapRegionDescriptor
 		return (NULL != _copyForwardData._survivorBase) && (getLowAddress() != _copyForwardData._survivorBase);
 	}
 
+	MMINLINE bool isLargestFreeMemoryFilledSurvivorRegion()
+	{
+		return (NULL != _copyForwardData._survivorLow) && (NULL != _copyForwardData._survivorHigh);
+	}
+
 	/**
-	 * @return True if region is in survivor set (there is no explicit flag, but info is inferred from _survivorBase being non-null
+	 * @return True if region is in survivor set (there is no explicit flag, but info is inferred from _survivorBase being non-null(empty region or tail region) or _survivorLow being non-null(free memory is in middle of region)
 	 */
-	MMINLINE bool isSurvivorRegion() { return NULL != _copyForwardData._survivorBase; }
+	MMINLINE bool isSurvivorRegion() { return (NULL != _copyForwardData._survivorBase) || (NULL != _copyForwardData._survivorLow); }
 
 	/**
 	 * Allocate supporting resources (large enough to justify not to preallocate them for all regions at the startup) when region is being committed.

runtime/gc_vlhgc/ParallelSweepSchemeVLHGC.cpp

@@ -33,7 +33,6 @@
 #include <string.h>
 
 #include "ParallelSweepSchemeVLHGC.hpp"
-#include "SweepPoolManagerAddressOrderedList.hpp"
 
 #include "AllocateDescription.hpp"
 #include "Bits.hpp"
@@ -138,6 +137,10 @@ MM_ParallelSweepVLHGCTask::mainCleanup(MM_EnvironmentBase *envBase)
 	/* now that sweep is finished, walk the list of regions and recycle any which are completely empty */
 	MM_EnvironmentVLHGC *env = MM_EnvironmentVLHGC::getEnvironment(envBase);
 	_sweepScheme->recycleFreeRegions(env);
+	if (_sweepScheme->isNoCompactionAfterSweep()) {
+		_sweepScheme->recoverRegionAllocationPointers(env);
+		_sweepScheme->setNoCompactionAfterSweep(false);
+	}
 	_sweepScheme->clearCycleState();
 }
 
@@ -197,6 +200,7 @@ MM_ParallelSweepSchemeVLHGC::MM_ParallelSweepSchemeVLHGC(MM_EnvironmentVLHGC *en
 	, _sweepHeapSectioning(NULL)
 	, _poolSweepPoolState(NULL)
 	, _mutexSweepPoolState(NULL)
+	, _noCompactionAfterSweep(false)
 {
 	_typeId = __FUNCTION__;
 }
@@ -980,6 +984,176 @@ MM_ParallelSweepSchemeVLHGC::replenishPoolForAllocate(MM_EnvironmentBase *env, M
 }
 #endif /* J9VM_GC_CONCURRENT_SWEEP */
 
+void
+MM_ParallelSweepSchemeVLHGC::recoverRegionAllocationPointers(MM_EnvironmentVLHGC *env)
+{
+	GC_HeapRegionIteratorVLHGC regionIterator(_regionManager);
+	MM_HeapRegionDescriptorVLHGC *region = NULL;
+	while(NULL != (region = regionIterator.nextRegion())) {
+		/* Region must be marked for sweep */
+		if (!region->_sweepData._alreadySwept && region->hasValidMarkMap() && region->containsObjects()) {
+			MM_MemoryPoolBumpPointer *regionPool = (MM_MemoryPoolBumpPointer *)region->getMemoryPool();
+			MM_HeapLinkedFreeHeader *lastFreeEntry = regionPool->getLastFreeEntry();
+			MM_HeapLinkedFreeHeader *largestFreeEntry = regionPool->getLargestFreeEntryAddr();
+			if ((NULL != lastFreeEntry) && ((UDATA) lastFreeEntry + lastFreeEntry->getSize()) != (UDATA)region->getHighAddress()) {
+				lastFreeEntry = NULL;
+			}
+			if ((largestFreeEntry == lastFreeEntry) && (NULL != largestFreeEntry)) {
+				largestFreeEntry = NULL;
+				regionPool->setLargestFreeEntryAddr(NULL);
+			}
+			bool isEligibleToRestoreTail = false;
+			UDATA freeBytesFromTail = 0;
+			if (NULL != lastFreeEntry) {
+				regionPool->setAllocationPointer(env, (void *)lastFreeEntry);
+				if (_extensions->tarokEnableAllocationPointerAssertion) {
+					Assert_MM_true(verifyRegionAllocationPointer(env, region));
+				}
+				regionPool->alignAllocationPointer(CARD_SIZE);
+				freeBytesFromTail = regionPool->getAllocatableBytes();
+				if (freeBytesFromTail >= (regionPool->getMinimumFreeEntrySize() + CARD_SIZE - 1)) {
+					isEligibleToRestoreTail = true;
+					MM_CardTable *cardTable = _extensions->cardTable;
+					Card *lowCard = cardTable->heapAddrToCardAddr(env, regionPool->getAllocationPointer());
+					Card *highCard = cardTable->heapAddrToCardAddr(env, region->getHighAddress());
+					memset(lowCard, CARD_CLEAN, (UDATA)highCard - (UDATA)lowCard);
+				}
+			}
+			if (isEligibleToRestoreTail) {
+				regionPool->setLargestFreeEntry(freeBytesFromTail);
+
+			} else {
+				regionPool->setAllocationPointer(env,region->getHighAddress());
+				regionPool->setLargestFreeEntry(0);
+			}
+
+			/*  */
+			if ((_extensions->tarokEnableRecoverRegionLargestFreeMemory) && (NULL != largestFreeEntry)) {
+				void* newStartFreeEntry = NULL;
+				void* newEndFreeEntry = NULL;
+				MM_CardTable *cardTable = _extensions->cardTable;
+				Card *lowCard = NULL;
+				Card *highCard = NULL;
+
+				newStartFreeEntry = regionPool->alignWithCard((void *) largestFreeEntry, false, CARD_SIZE);
+				newEndFreeEntry = regionPool->alignWithCard((void *) ((UDATA)largestFreeEntry + largestFreeEntry->getSize()), true, CARD_SIZE);
+				if (((UDATA) newEndFreeEntry > (UDATA) newStartFreeEntry) && (((UDATA) newEndFreeEntry - (UDATA) newStartFreeEntry) >= (regionPool->getMinimumFreeEntrySize() + CARD_SIZE - 1))) {
+					lowCard = cardTable->heapAddrToCardAddr(env, newStartFreeEntry);
+					highCard = cardTable->heapAddrToCardAddr(env, newEndFreeEntry);
+					memset(lowCard, CARD_CLEAN, (UDATA)highCard - (UDATA)lowCard);
+
+					regionPool->setAllocationPointer4Collector(env, newStartFreeEntry, newEndFreeEntry);
+					regionPool->setLargestFreeEntry((UDATA)newEndFreeEntry - (UDATA)newStartFreeEntry);
+				}
+			} else {
+				regionPool->setAllocationPointer4Collector(env, NULL, NULL);
+			}
+
+//			adjustFreeList(env, region, regionPool->getMinimumFreeEntrySize());
+		}
+	}
+}
+
+void
+MM_ParallelSweepSchemeVLHGC::adjustFreeList(MM_EnvironmentBase *env, MM_HeapRegionDescriptorVLHGC *region, UDATA minimumSize4Reuse)
+{
+	MM_MemoryPoolBumpPointer *regionPool = (MM_MemoryPoolBumpPointer *)region->getMemoryPool();
+	MM_HeapLinkedFreeHeader *currentFreeEntry = (MM_HeapLinkedFreeHeader*) regionPool->getFirstFreeStartingAddr(env);
+	MM_HeapLinkedFreeHeader *previousFreeEntry = NULL;
+	MM_HeapLinkedFreeHeader *nextFreeEntry = NULL;
+	void* newStartFreeEntry = NULL;
+	void* endFreeEntry = NULL;
+	void* newEndFreeEntry = NULL;
+	MM_CardTable *cardTable = _extensions->cardTable;
+	Card *lowCard = NULL;
+	Card *highCard = NULL;
+	bool needClearCard = true;
+	bool const compressed = env->compressObjectReferences();
+
+	regionPool->setLargestFreeEntryAddr(NULL);
+	minimumSize4Reuse = OMR_MAX(minimumSize4Reuse, CARD_SIZE);
+	while (NULL != currentFreeEntry) {
+		endFreeEntry = (void *) ((UDATA)currentFreeEntry + currentFreeEntry->getSize());
+		newStartFreeEntry = regionPool->alignWithCard((void *) currentFreeEntry, false, CARD_SIZE);
+		newEndFreeEntry = regionPool->alignWithCard((void *) endFreeEntry, true, CARD_SIZE);
+		nextFreeEntry = currentFreeEntry->getNext(compressed);
+		needClearCard = true;
+
+		if (((void *) currentFreeEntry != newStartFreeEntry) || (endFreeEntry != newEndFreeEntry)) {
+			if (((UDATA) newEndFreeEntry <= (UDATA) newStartFreeEntry) || (((UDATA) newEndFreeEntry - (UDATA) newStartFreeEntry) < minimumSize4Reuse)) {
+				/* remove currentFreeEntry */
+				regionPool->removeFromFreeList((void *)currentFreeEntry, endFreeEntry, previousFreeEntry, nextFreeEntry);
+				needClearCard = false;
+			} else {
+				if ((UDATA) currentFreeEntry != (UDATA) newStartFreeEntry) {
+					regionPool->fillWithHoles((void *)currentFreeEntry, newStartFreeEntry);
+				}
+				if ((UDATA) endFreeEntry != (UDATA) newEndFreeEntry) {
+					regionPool->fillWithHoles(newEndFreeEntry, endFreeEntry);
+				}
+				regionPool->recycleHeapChunk(newStartFreeEntry, newEndFreeEntry, previousFreeEntry, nextFreeEntry);
+				previousFreeEntry = (MM_HeapLinkedFreeHeader *) newEndFreeEntry;
+			}
+		} else {
+			previousFreeEntry = currentFreeEntry;
+		}
+
+		if (needClearCard) {
+			if (regionPool->getLargestFreeEntry() == ((UDATA)endFreeEntry - (UDATA) currentFreeEntry)) {
+				regionPool->setLargestFreeEntry((UDATA)newEndFreeEntry - (UDATA) newStartFreeEntry);
+				regionPool->setLargestFreeEntryAddr(newStartFreeEntry);
+			}
+			lowCard = cardTable->heapAddrToCardAddr(env, newStartFreeEntry);
+			highCard = cardTable->heapAddrToCardAddr(env, newEndFreeEntry);
+			memset(lowCard, CARD_CLEAN, (UDATA)highCard - (UDATA)lowCard);
+		}
+		currentFreeEntry = nextFreeEntry;
+	}
+}
+
+bool
+MM_ParallelSweepSchemeVLHGC::verifyRegionAllocationPointer(MM_EnvironmentVLHGC *env, MM_HeapRegionDescriptorVLHGC *region)
+{
+	/* Region must be marked for sweep */
+	MM_MarkMap *markMap = env->_cycleState->_markMap;
+	UDATA lowIndex = markMap->getSlotIndex((J9Object *)region->getLowAddress());
+	UDATA highIndex = markMap->getSlotIndex((J9Object *)region->getHighAddress());
+	UDATA currentIndex = highIndex - 1;
+	UDATA bitIndex = 0;
+	UDATA currentSlot = 0;
+
+	while (currentIndex >= lowIndex) {
+		currentSlot = markMap->getSlot(currentIndex);
+		if (0 != currentSlot) {
+			UDATA mask = 1;
+			mask <<= (J9BITS_BITS_IN_SLOT - 1);
+			for (UDATA cnt = 0; cnt < J9BITS_BITS_IN_SLOT; cnt++, mask>>=1) {
+				if (0 != (currentSlot&mask)) {
+					bitIndex = cnt;
+					break;
+				}
+			}
+			break;
+		}
+		currentIndex -= 1;
+	}
+
+	UDATA freeBytesFromTail = ((highIndex - 1 - currentIndex) * J9BITS_BITS_IN_SLOT + bitIndex + 1) * J9MODRON_HEAP_BYTES_PER_HEAPMAP_BIT;
+	J9Object *lastObject = (J9Object *)(((uintptr_t)region->getHighAddress()) - freeBytesFromTail);
+	Assert_MM_true(NULL != lastObject);
+	Assert_MM_mustBeClass(J9GC_J9OBJECT_CLAZZ(lastObject, env));
+	UDATA lastObjectSize = _extensions->objectModel.getConsumedSizeInBytesWithHeader(lastObject);
+	if (freeBytesFromTail > lastObjectSize) {
+		freeBytesFromTail -= lastObjectSize;
+	} else {
+		freeBytesFromTail = 0;
+	}
+	void * allocPointer = (void *)(((uintptr_t)region->getHighAddress()) - freeBytesFromTail);
+
+	MM_MemoryPoolBumpPointer *regionPool = (MM_MemoryPoolBumpPointer *)region->getMemoryPool();
+	return (regionPool->getAllocationPointer() == allocPointer);
+}
+
 void
 MM_ParallelSweepSchemeVLHGC::recycleFreeRegions(MM_EnvironmentVLHGC *env)
 {