[scudo] Add fragmentation info for each memory group

[ChiaHungDuan]

This information helps with tuning the heuristic of selecting memory groups to release the unused pages.

[llvmbot]

@llvm/pr-subscribers-compiler-rt-sanitizer

Author: None (ChiaHungDuan)

Changes

This information helps with tuning the heuristic of selecting memory groups to release the unused pages.

---

Full diff: https://github.com/llvm/llvm-project/pull/107475.diff

4 Files Affected:

• (modified) compiler-rt/lib/scudo/standalone/primary32.h (+6)
• (modified) compiler-rt/lib/scudo/standalone/primary64.h (+57)
• (modified) compiler-rt/lib/scudo/standalone/release.h (+16)
• (modified) compiler-rt/lib/scudo/standalone/tests/primary_test.cpp (+1)

diff --git a/compiler-rt/lib/scudo/standalone/primary32.h b/compiler-rt/lib/scudo/standalone/primary32.h
index 87264471be2c1a..9ef5e7102e3220 100644
--- a/compiler-rt/lib/scudo/standalone/primary32.h
+++ b/compiler-rt/lib/scudo/standalone/primary32.h
@@ -332,6 +332,12 @@ template <typename Config> class SizeClassAllocator32 {
     }
   }
 
+  void getMemoryGroupFragmentationInfo(ScopedString *Str) {
+    // Each region is also a memory group because region size is the same as
+    // group size.
+    getFragmentationInfo(Str);
+  }
+
   bool setOption(Option O, sptr Value) {
     if (O == Option::ReleaseInterval) {
       const s32 Interval = Max(
diff --git a/compiler-rt/lib/scudo/standalone/primary64.h b/compiler-rt/lib/scudo/standalone/primary64.h
index ffcc22fea0c6ea..1bcfc54235b1ad 100644
--- a/compiler-rt/lib/scudo/standalone/primary64.h
+++ b/compiler-rt/lib/scudo/standalone/primary64.h
@@ -395,6 +395,18 @@ template <typename Config> class SizeClassAllocator64 {
     }
   }
 
+  void getMemoryGroupFragmentationInfo(ScopedString *Str) {
+    Str->append(
+        "Fragmentation Stats: SizeClassAllocator64: page size = %zu bytes\n",
+        getPageSizeCached());
+
+    for (uptr I = 1; I < NumClasses; I++) {
+      RegionInfo *Region = getRegionInfo(I);
+      ScopedLock L(Region->MMLock);
+      getMemoryGroupFragmentationInfoInRegion(Region, I, Str);
+    }
+  }
+
   bool setOption(Option O, sptr Value) {
     if (O == Option::ReleaseInterval) {
       const s32 Interval = Max(
@@ -1193,6 +1205,51 @@ template <typename Config> class SizeClassAllocator64 {
                 AllocatedPagesCount, InUseBytes >> 10, Integral, Fractional);
   }
 
+  void getMemoryGroupFragmentationInfoInRegion(RegionInfo *Region, uptr ClassId,
+                                               ScopedString *Str)
+      REQUIRES(Region->MMLock) {
+    const uptr BlockSize = getSizeByClassId(ClassId);
+    const uptr AllocatedUserEnd =
+        Region->MemMapInfo.AllocatedUser + Region->RegionBeg;
+
+    SinglyLinkedList<BatchGroupT> GroupsToRelease;
+    {
+      ScopedLock L(Region->FLLock);
+      GroupsToRelease = Region->FreeListInfo.BlockList;
+      Region->FreeListInfo.BlockList.clear();
+    }
+
+    constexpr uptr GroupSize = (1UL << GroupSizeLog);
+    constexpr uptr MaxNumGroups = RegionSize / GroupSize;
+
+    MemoryGroupFragmentationRecorder<GroupSize, MaxNumGroups> Recorder;
+    if (!GroupsToRelease.empty()) {
+      PageReleaseContext Context =
+          markFreeBlocks(Region, BlockSize, AllocatedUserEnd,
+                         getCompactPtrBaseByClassId(ClassId), GroupsToRelease);
+      auto SkipRegion = [](UNUSED uptr RegionIndex) { return false; };
+      releaseFreeMemoryToOS(Context, Recorder, SkipRegion);
+
+      mergeGroupsToReleaseBack(Region, GroupsToRelease);
+    }
+
+    Str->append("MemoryGroupFragmentationInfo in Region %zu (%zu)\n", ClassId,
+                BlockSize);
+
+    const uptr MaxNumGroupsInUse =
+        roundUp(Region->MemMapInfo.AllocatedUser, GroupSize) / GroupSize;
+    for (uptr I = 0; I < MaxNumGroupsInUse; ++I) {
+
+      uptr Integral;
+      uptr Fractional;
+      computePercentage(Recorder.NumPagesInOneGroup -
+                            Recorder.getNumFreePages(I),
+                        Recorder.NumPagesInOneGroup, &Integral, &Fractional);
+      Str->append("MemoryGroup #%zu (0x%zx): util: %3zu.%02zu%%\n", I,
+                  Region->RegionBeg + I * GroupSize, Integral, Fractional);
+    }
+  }
+
   NOINLINE uptr releaseToOSMaybe(RegionInfo *Region, uptr ClassId,
                                  ReleaseToOS ReleaseType = ReleaseToOS::Normal)
       REQUIRES(Region->MMLock) EXCLUDES(Region->FLLock) {
diff --git a/compiler-rt/lib/scudo/standalone/release.h b/compiler-rt/lib/scudo/standalone/release.h
index 51abdd82aa5389..6353dafde31609 100644
--- a/compiler-rt/lib/scudo/standalone/release.h
+++ b/compiler-rt/lib/scudo/standalone/release.h
@@ -95,6 +95,22 @@ class FragmentationRecorder {
   uptr ReleasedPagesCount = 0;
 };
 
+template <uptr GroupSize, uptr NumGroups>
+class MemoryGroupFragmentationRecorder {
+public:
+  const uptr NumPagesInOneGroup = GroupSize / getPageSizeCached();
+
+  void releasePageRangeToOS(uptr From, uptr To) {
+    for (uptr I = From / getPageSizeCached(); I < To / getPageSizeCached(); ++I)
+      ++FreePagesCount[I / NumPagesInOneGroup];
+  }
+
+  uptr getNumFreePages(uptr GroupId) { return FreePagesCount[GroupId]; }
+
+private:
+  uptr FreePagesCount[NumGroups] = {};
+};
+
 // A buffer pool which holds a fixed number of static buffers of `uptr` elements
 // for fast buffer allocation. If the request size is greater than
 // `StaticBufferNumElements` or if all the static buffers are in use, it'll
diff --git a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
index 1cf3bb51db0e71..0636fe73c0b591 100644
--- a/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
+++ b/compiler-rt/lib/scudo/standalone/tests/primary_test.cpp
@@ -386,6 +386,7 @@ SCUDO_TYPED_TEST(ScudoPrimaryTest, PrimaryThreaded) {
   scudo::ScopedString Str;
   Allocator->getStats(&Str);
   Allocator->getFragmentationInfo(&Str);
+  Allocator->getMemoryGroupFragmentationInfo(&Str);
   Str.output();
 }
 

[cferris1000]

Super small nit, I think you can delete this blank line.

[ChiaHungDuan]

Done

[cferris1000]

LGTM