[libc][stdlib] Add the FreelistHeap

This is the actual freelist allocator which utilizes the generic
FreeList and the Block classes. We will eventually wrap the malloc
interface around this.

This is a part of llvm#94270 to land in smaller patches.

libc/src/stdlib/CMakeLists.txt

@@ -402,6 +402,20 @@ else()
       libc.src.__support.CPP.array
       libc.src.__support.CPP.span
   )
+  add_header_library(
+    freelist_heap
+    HDRS
+      freelist_heap.h
+    DEPENDS
+      .block
+      .freelist
+      libc.src.__support.CPP.cstddef
+      libc.src.__support.CPP.array
+      libc.src.__support.CPP.optional
+      libc.src.__support.CPP.span
+      libc.src.string.memcpy
+      libc.src.string.memset
+  )
   add_entrypoint_external(
     malloc
   )

libc/src/stdlib/freelist.h

@@ -99,7 +99,7 @@ bool FreeList<NUM_BUCKETS>::add_chunk(span<cpp::byte> chunk) {
 
   aliased.bytes = chunk.data();
 
-  unsigned short chunk_ptr = find_chunk_ptr_for_size(chunk.size(), false);
+  size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), false);
 
   // Add it to the correct list.
   aliased.node->size = chunk.size();
@@ -114,7 +114,7 @@ span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk(size_t size) const {
   if (size == 0)
     return span<cpp::byte>();
 
-  unsigned short chunk_ptr = find_chunk_ptr_for_size(size, true);
+  size_t chunk_ptr = find_chunk_ptr_for_size(size, true);
 
   // Check that there's data. This catches the case where we run off the
   // end of the array
@@ -144,7 +144,7 @@ span<cpp::byte> FreeList<NUM_BUCKETS>::find_chunk(size_t size) const {
 
 template <size_t NUM_BUCKETS>
 bool FreeList<NUM_BUCKETS>::remove_chunk(span<cpp::byte> chunk) {
-  unsigned short chunk_ptr = find_chunk_ptr_for_size(chunk.size(), true);
+  size_t chunk_ptr = find_chunk_ptr_for_size(chunk.size(), true);
 
   // Walk that list, finding the chunk.
   union {

libc/src/stdlib/freelist_heap.h

@@ -0,0 +1,189 @@
+//===-- Interface for freelist_heap ---------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIBC_SRC_STDLIB_FREELIST_HEAP_H
+#define LLVM_LIBC_SRC_STDLIB_FREELIST_HEAP_H
+
+#include <stddef.h>
+
+#include "block.h"
+#include "freelist.h"
+#include "src/__support/CPP/optional.h"
+#include "src/__support/CPP/span.h"
+#include "src/string/memcpy.h"
+#include "src/string/memset.h"
+
+namespace LIBC_NAMESPACE {
+
+using cpp::optional;
+using cpp::span;
+
+static constexpr cpp::array<size_t, 6> DEFAULT_BUCKETS{16,  32,  64,
+                                                       128, 256, 512};
+
+template <size_t NUM_BUCKETS = DEFAULT_BUCKETS.size()> class FreeListHeap {
+public:
+  using BlockType = Block<>;
+
+  struct HeapStats {
+    size_t total_bytes;
+    size_t bytes_allocated;
+    size_t cumulative_allocated;
+    size_t cumulative_freed;
+    size_t total_allocate_calls;
+    size_t total_free_calls;
+  };
+  FreeListHeap(span<cpp::byte> region);
+
+  void *allocate(size_t size);
+  void free(void *ptr);
+  void *realloc(void *ptr, size_t size);
+  void *calloc(size_t num, size_t size);
+
+  const HeapStats &heap_stats() const { return heap_stats_; }
+
+private:
+  span<cpp::byte> block_to_span(BlockType *block) {
+    return span<cpp::byte>(block->usable_space(), block->inner_size());
+  }
+
+  void InvalidFreeCrash() { __builtin_trap(); }
+
+  span<cpp::byte> region_;
+  FreeList<NUM_BUCKETS> freelist_;
+  HeapStats heap_stats_;
+};
+
+template <size_t NUM_BUCKETS>
+FreeListHeap<NUM_BUCKETS>::FreeListHeap(span<cpp::byte> region)
+    : region_(region), freelist_(DEFAULT_BUCKETS), heap_stats_() {
+  auto result = BlockType::init(region);
+  BlockType *block = *result;
+
+  freelist_.add_chunk(block_to_span(block));
+
+  heap_stats_.total_bytes = region.size();
+}
+
+template <size_t NUM_BUCKETS>
+void *FreeListHeap<NUM_BUCKETS>::allocate(size_t size) {
+  // Find a chunk in the freelist. Split it if needed, then return
+  auto chunk = freelist_.find_chunk(size);
+
+  if (chunk.data() == nullptr)
+    return nullptr;
+  freelist_.remove_chunk(chunk);
+
+  BlockType *chunk_block = BlockType::from_usable_space(chunk.data());
+
+  // Split that chunk. If there's a leftover chunk, add it to the freelist
+  optional<BlockType *> result = BlockType::split(chunk_block, size);
+  if (result)
+    freelist_.add_chunk(block_to_span(*result));
+
+  chunk_block->mark_used();
+
+  heap_stats_.bytes_allocated += size;
+  heap_stats_.cumulative_allocated += size;
+  heap_stats_.total_allocate_calls += 1;
+
+  return chunk_block->usable_space();
+}
+
+template <size_t NUM_BUCKETS> void FreeListHeap<NUM_BUCKETS>::free(void *ptr) {
+  cpp::byte *bytes = static_cast<cpp::byte *>(ptr);
+
+  if (bytes < region_.data() || bytes >= region_.data() + region_.size()) {
+    InvalidFreeCrash();
+    return;
+  }
+
+  BlockType *chunk_block = BlockType::from_usable_space(bytes);
+
+  size_t size_freed = chunk_block->inner_size();
+  // Ensure that the block is in-use
+  if (!chunk_block->used()) {
+    InvalidFreeCrash();
+    return;
+  }
+  chunk_block->mark_free();
+  // Can we combine with the left or right blocks?
+  BlockType *prev = chunk_block->prev();
+  BlockType *next = nullptr;
+
+  if (!chunk_block->last())
+    next = chunk_block->next();
+
+  if (prev != nullptr && !prev->used()) {
+    // Remove from freelist and merge
+    freelist_.remove_chunk(block_to_span(prev));
+    chunk_block = chunk_block->prev();
+    BlockType::merge_next(chunk_block);
+  }
+
+  if (next != nullptr && !next->used()) {
+    freelist_.remove_chunk(block_to_span(next));
+    BlockType::merge_next(chunk_block);
+  }
+  // Add back to the freelist
+  freelist_.add_chunk(block_to_span(chunk_block));
+
+  heap_stats_.bytes_allocated -= size_freed;
+  heap_stats_.cumulative_freed += size_freed;
+  heap_stats_.total_free_calls += 1;
+}
+
+// Follows constract of the C standard realloc() function
+// If ptr is free'd, will return nullptr.
+template <size_t NUM_BUCKETS>
+void *FreeListHeap<NUM_BUCKETS>::realloc(void *ptr, size_t size) {
+  if (size == 0) {
+    free(ptr);
+    return nullptr;
+  }
+
+  // If the pointer is nullptr, allocate a new memory.
+  if (ptr == nullptr)
+    return allocate(size);
+
+  cpp::byte *bytes = static_cast<cpp::byte *>(ptr);
+
+  if (bytes < region_.data() || bytes >= region_.data() + region_.size())
+    return nullptr;
+
+  BlockType *chunk_block = BlockType::from_usable_space(bytes);
+  if (!chunk_block->used())
+    return nullptr;
+  size_t old_size = chunk_block->inner_size();
+
+  // Do nothing and return ptr if the required memory size is smaller than
+  // the current size.
+  if (old_size >= size)
+    return ptr;
+
+  void *new_ptr = allocate(size);
+  // Don't invalidate ptr if allocate(size) fails to initilize the memory.
+  if (new_ptr == nullptr)
+    return nullptr;
+  memcpy(new_ptr, ptr, old_size);
+
+  free(ptr);
+  return new_ptr;
+}
+
+template <size_t NUM_BUCKETS>
+void *FreeListHeap<NUM_BUCKETS>::calloc(size_t num, size_t size) {
+  void *ptr = allocate(num * size);
+  if (ptr != nullptr)
+    memset(ptr, 0, num * size);
+  return ptr;
+}
+
+} // namespace LIBC_NAMESPACE
+
+#endif // LLVM_LIBC_SRC_STDLIB_FREELIST_HEAP_H

libc/test/src/stdlib/CMakeLists.txt

@@ -79,6 +79,19 @@ add_libc_test(
     libc.src.__support.CPP.span
 )
 
+add_libc_test(
+  freelist_heap_test
+  SUITE
+    libc-stdlib-tests
+  SRCS
+    freelist_heap_test.cpp
+  DEPENDS
+    libc.src.__support.CPP.span
+    libc.src.stdlib.freelist_heap
+    libc.src.string.memcmp
+    libc.src.string.memcpy
+)
+
 add_fp_unittest(
   strtod_test
   SUITE