Live allocation - performances

Ensure an allocation sample is pushed even if we encounter a
collision in the bitset.

include/lib/address_bitset.hpp

@@ -21,8 +21,13 @@ class AddressBitset {
 public:
   // Publish 1 Meg as default
   constexpr static unsigned _k_default_bitset_size = 8 * 1024 * 1024;
-  explicit AddressBitset(unsigned max_addresses = 0) { init(max_addresses); }
-  void init(unsigned max_addresses);
+  explicit AddressBitset(unsigned bitset_size = 0) { init(bitset_size); }
+  AddressBitset(AddressBitset &&other) noexcept;
+  AddressBitset &operator=(AddressBitset &&other) noexcept;
+
+  AddressBitset(AddressBitset &other) = delete;
+  AddressBitset &operator=(AddressBitset &other) = delete;
+
   // returns true if the element was inserted
   bool add(uintptr_t addr);
   // returns true if the element was removed
@@ -39,15 +44,16 @@ class AddressBitset {
   // 1 Meg divided in uint64's size
   // The probability of collision is proportional to the number of elements
   // already within the bitset
-  unsigned _nb_bits = {};
+  unsigned _bitset_size = {};
   unsigned _k_nb_words = {};
   unsigned _nb_bits_mask = {};
   // We can not use an actual bitset (for atomicity reasons)
   std::unique_ptr<std::atomic<uint64_t>[]> _address_bitset;
   std::atomic<int> _nb_addresses = 0;
 
-  static unsigned int count_set_bits(Word_t w);
+  void init(unsigned bitset_size);
 
+  void move_from(AddressBitset &other) noexcept;
   // This is a kind of hash function
   // We remove the lower bits (as the alignment constraints makes them useless)
   // We fold the address

include/lib/lib_logger.hpp

@@ -24,8 +24,8 @@ template <typename Func> void log_once_helper(std::once_flag &, Func &&func) {
 // create a once flag for the line and file where this is called:
 #define LOG_ONCE(format, ...)                                                  \
   do {                                                                         \
-    static std::once_flag UNIQUE_ONCE_FLAG_##__LINE__##__FILE__;               \
-    ddprof::log_once_helper(UNIQUE_ONCE_FLAG_##__LINE__##__FILE__, [&]() {     \
+    static std::once_flag UNIQUE_ONCE_FLAG_##__COUNTER__;                      \
+    ddprof::log_once_helper(UNIQUE_ONCE_FLAG_##__COUNTER__, [&]() {            \
       fprintf(stderr, (format), ##__VA_ARGS__);                                \
     });                                                                        \
   } while (0)

src/ddprof_worker.cc

@@ -306,7 +306,8 @@ DDRes ddprof_pr_sample(DDProfContext &ctx, perf_event_sample *sample,
   // Aggregate if unwinding went well (todo : fatal error propagation)
   if (!IsDDResFatal(res)) {
     struct UnwindState *us = ctx.worker_ctx.us;
-    if (Any(EventConfMode::kLiveCallgraph & watcher->output_mode)) {
+    if (Any(EventConfMode::kLiveCallgraph & watcher->output_mode) &&
+        sample->addr) { // null address means we should not account it
       // Live callgraph mode
       // for now we hard code the live aggregation mode
       ctx.worker_ctx.live_allocation.register_allocation(

src/lib/address_bitset.cc

@@ -5,17 +5,18 @@
 #include "address_bitset.hpp"
 
 #include <algorithm>
+#include <bit>
 #include <functional>
 #include <unlikely.hpp>
 
 namespace ddprof {
 
 namespace {
-unsigned round_to_power_of_two(unsigned num) {
+unsigned round_up_to_power_of_two(unsigned num) {
   if (num == 0) {
     return num;
   }
-  // If max_addresses is already a power of two
+  // If num is already a power of two
   if ((num & (num - 1)) == 0) {
     return num;
   }
@@ -29,17 +30,43 @@ unsigned round_to_power_of_two(unsigned num) {
 }
 } // namespace
 
-void AddressBitset::init(unsigned max_addresses) {
+AddressBitset::AddressBitset(AddressBitset &&other) noexcept {
+  move_from(other);
+}
+
+AddressBitset &AddressBitset::operator=(AddressBitset &&other) noexcept {
+  if (this != &other) {
+    move_from(other);
+  }
+  return *this;
+}
+
+void AddressBitset::move_from(AddressBitset &other) noexcept {
+  _lower_bits_ignored = other._lower_bits_ignored;
+  _bitset_size = other._bitset_size;
+  _k_nb_words = other._k_nb_words;
+  _nb_bits_mask = other._nb_bits_mask;
+  _address_bitset = std::move(other._address_bitset);
+  _nb_addresses.store(other._nb_addresses.load());
+
+  // Reset the state of 'other'
+  other._bitset_size = 0;
+  other._k_nb_words = 0;
+  other._nb_bits_mask = 0;
+  other._nb_addresses = 0;
+}
+
+void AddressBitset::init(unsigned bitset_size) {
   // Due to memory alignment, on 64 bits we can assume that the first 4
   // bits can be ignored
   _lower_bits_ignored = _k_max_bits_ignored;
   if (_address_bitset) {
     _address_bitset.reset();
   }
-  _nb_bits = round_to_power_of_two(max_addresses);
-  _k_nb_words = (_nb_bits) / (_nb_bits_per_word);
-  if (_nb_bits) {
-    _nb_bits_mask = _nb_bits - 1;
+  _bitset_size = round_up_to_power_of_two(bitset_size);
+  _k_nb_words = (_bitset_size) / (_nb_bits_per_word);
+  if (_bitset_size) {
+    _nb_bits_mask = _bitset_size - 1;
     _address_bitset = std::make_unique<std::atomic<uint64_t>[]>(_k_nb_words);
   }
 }
@@ -79,20 +106,11 @@ bool AddressBitset::remove(uintptr_t addr) {
   return false;
 }
 
-unsigned int AddressBitset::count_set_bits(Word_t w) {
-  unsigned int set_bits = 0;
-  while (w) {
-    set_bits += w & 1;
-    w >>= 1;
-  }
-  return set_bits;
-}
-
 void AddressBitset::clear() {
   for (unsigned i = 0; i < _k_nb_words; ++i) {
     Word_t original_value = _address_bitset[i].exchange(0);
     // Count number of set bits in original_value
-    int num_set_bits = count_set_bits(original_value);
+    int num_set_bits = std::popcount(original_value);
     if (num_set_bits > 0) {
       _nb_addresses.fetch_sub(num_set_bits, std::memory_order_relaxed);
     }

src/lib/allocation_tracker.cc

@@ -142,7 +142,7 @@ DDRes AllocationTracker::init(uint64_t mem_profile_interval,
   }
   if (track_deallocations) {
     // 16 times as we want to probability of collision to be low enough
-    _allocated_address_set.init(liveallocation::kMaxTracked * 16);
+    _allocated_address_set = AddressBitset(liveallocation::kMaxTracked * 16);
   }
   return ddprof::ring_buffer_attach(ring_buffer, &_pevent);
 }
@@ -252,13 +252,15 @@ void AllocationTracker::track_allocation(uintptr_t addr, size_t size,
         }
       }
     } else {
-      // we won't track this allocation as we can't double count in the bitset
-      return;
+      // null the address to avoid using this for live heap profiling
+      // pushing a sample is still good to have a good representation
+      // of the allocations.
+      addr = 0;
     }
   }
   bool success = IsDDResOK(push_alloc_sample(addr, total_size, tl_state));
   free_on_consecutive_failures(success);
-  if (unlikely(!success) && _state.track_deallocations) {
+  if (unlikely(!success) && _state.track_deallocations && addr) {
     _allocated_address_set.remove(addr);
   }
 }