diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 20eca4db4..11e6e0cfe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -504,6 +504,53 @@ jobs: - name: Test run: ${{github.workspace}}/build/fuzzing/snmalloc-fuzzer + self-vendored: + strategy: + matrix: + include: + - os: windows-2022 + cxx: clang-cl + cc: clang-cl + - os: ubuntu-latest + cxx: clang++ + cc: clang + - os: ubuntu-latest + cxx: g++ + cc: gcc + - os: macos-latest + cxx: clang++ + cc: clang + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Prepare Windows + if: runner.os == 'Windows' + run: | + choco install ninja + - name: Prepare macOS + if: runner.os == 'macOS' + run: | + brew install ninja + - name: Prepare Ubuntu + if: runner.os == 'Linux' + run: | + sudo apt install ninja-build + - name: Configure CMake + run: > + cmake + -B ${{github.workspace}}/build + -DSNMALLOC_USE_SELF_VENDORED_STL=ON + -GNinja + -DCMAKE_BUILD_TYPE=RelWithDebInfo + -DCMAKE_CXX_COMPILER=${{ matrix.cxx }} + -DCMAKE_C_COMPILER=${{ matrix.cc }} + - name: Build + run: cmake --build ${{github.workspace}}/build --parallel + - name: Test + run: | + cd ${{github.workspace}}/build + ctest --parallel + all-checks: # Currently FreeBSD and NetBSD CI are not working, so we do not require them to pass. # Add fuzzing back when the memove issue is fixed. diff --git a/CMakeLists.txt b/CMakeLists.txt index a71f378f1..05e94fdd9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ option(SNMALLOC_BENCHMARK_INDIVIDUAL_MITIGATIONS "Build tests and ld_preload for option(SNMALLOC_ENABLE_DYNAMIC_LOADING "Build such that snmalloc can be dynamically loaded. This is not required for LD_PRELOAD, and will harm performance if enabled." OFF) option(SNMALLOC_ENABLE_WAIT_ON_ADDRESS "Use wait on address backoff strategy if it is available" ON) option(SNMALLOC_ENABLE_FUZZING "Enable fuzzing instrumentation tests" OFF) +option(SNMALLOC_USE_SELF_VENDORED_STL "Avoid using system STL" OFF) # Options that apply only if we're not building the header-only library cmake_dependent_option(SNMALLOC_RUST_SUPPORT "Build static library for rust" OFF "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) cmake_dependent_option(SNMALLOC_STATIC_LIBRARY "Build static libraries" ON "NOT SNMALLOC_HEADER_ONLY_LIBRARY" OFF) @@ -205,6 +206,12 @@ else() target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_WAIT_ON_ADDRESS=0) endif() +if(SNMALLOC_USE_SELF_VENDORED_STL) + target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_SELF_VENDORED_STL=1) +else() + target_compile_definitions(snmalloc INTERFACE SNMALLOC_USE_SELF_VENDORED_STL=0) +endif() + # https://learn.microsoft.com/en-us/cpp/build/reference/zc-cplusplus if(MSVC) target_compile_options(snmalloc INTERFACE "/Zc:__cplusplus") diff --git a/src/snmalloc/backend/globalconfig.h b/src/snmalloc/backend/globalconfig.h index 5d171a9b8..c3e3f7bfa 100644 --- a/src/snmalloc/backend/globalconfig.h +++ b/src/snmalloc/backend/globalconfig.h @@ -84,7 +84,7 @@ namespace snmalloc * Specifies if the Configuration has been initialised. */ SNMALLOC_REQUIRE_CONSTINIT - inline static std::atomic initialised{false}; + inline static stl::Atomic initialised{false}; /** * Used to prevent two threads attempting to initialise the configuration @@ -126,7 +126,7 @@ namespace snmalloc Authmap::init(); } - initialised.store(true, std::memory_order_release); + initialised.store(true, stl::memory_order_release); }); } @@ -146,7 +146,7 @@ namespace snmalloc // and concurrency safe. SNMALLOC_FAST_PATH static void ensure_init() { - if (SNMALLOC_LIKELY(initialised.load(std::memory_order_acquire))) + if (SNMALLOC_LIKELY(initialised.load(stl::memory_order_acquire))) return; ensure_init_slow(); diff --git a/src/snmalloc/backend_helpers/pagemap.h b/src/snmalloc/backend_helpers/pagemap.h index 2e118484f..1eaf95d0d 100644 --- a/src/snmalloc/backend_helpers/pagemap.h +++ b/src/snmalloc/backend_helpers/pagemap.h @@ -2,8 +2,8 @@ #include "../ds/ds.h" #include "../mem/mem.h" +#include "snmalloc/stl/atomic.h" -#include #include namespace snmalloc diff --git a/src/snmalloc/backend_helpers/statsrange.h b/src/snmalloc/backend_helpers/statsrange.h index 8548be9cb..d1e213777 100644 --- a/src/snmalloc/backend_helpers/statsrange.h +++ b/src/snmalloc/backend_helpers/statsrange.h @@ -2,8 +2,7 @@ #include "empty_range.h" #include "range_helpers.h" - -#include +#include "snmalloc/stl/atomic.h" namespace snmalloc { @@ -17,8 +16,8 @@ namespace snmalloc { using ContainsParent::parent; - static inline std::atomic current_usage{}; - static inline std::atomic peak_usage{}; + static inline stl::Atomic current_usage{}; + static inline stl::Atomic peak_usage{}; public: static constexpr bool Aligned = ParentRange::Aligned; diff --git a/src/snmalloc/ds/aba.h b/src/snmalloc/ds/aba.h index af75de9e0..be4ef535f 100644 --- a/src/snmalloc/ds/aba.h +++ b/src/snmalloc/ds/aba.h @@ -35,8 +35,8 @@ namespace snmalloc struct Independent { - std::atomic ptr{nullptr}; - std::atomic aba{0}; + stl::Atomic ptr{nullptr}; + stl::Atomic aba{0}; }; static_assert( @@ -49,7 +49,7 @@ namespace snmalloc private: union { - alignas(2 * sizeof(std::size_t)) std::atomic linked; + alignas(2 * sizeof(std::size_t)) stl::Atomic linked; Independent independent; }; @@ -58,8 +58,8 @@ namespace snmalloc void init(T* x) { - independent.ptr.store(x, std::memory_order_relaxed); - independent.aba.store(0, std::memory_order_relaxed); + independent.ptr.store(x, stl::memory_order_relaxed); + independent.aba.store(0, stl::memory_order_relaxed); } struct Cmp; @@ -72,8 +72,8 @@ namespace snmalloc operation_in_flight = true; # endif return Cmp{ - {independent.ptr.load(std::memory_order_relaxed), - independent.aba.load(std::memory_order_relaxed)}, + {independent.ptr.load(stl::memory_order_relaxed), + independent.aba.load(stl::memory_order_relaxed)}, this}; } @@ -109,10 +109,10 @@ namespace snmalloc # endif Linked xchg{value, old.aba + 1}; - std::atomic& addr = parent->linked; + stl::Atomic& addr = parent->linked; auto result = addr.compare_exchange_weak( - old, xchg, std::memory_order_acq_rel, std::memory_order_relaxed); + old, xchg, stl::memory_order_acq_rel, stl::memory_order_relaxed); # endif return result; } @@ -131,7 +131,7 @@ namespace snmalloc // This method is used in Verona T* peek() { - return independent.ptr.load(std::memory_order_relaxed); + return independent.ptr.load(stl::memory_order_relaxed); } }; #else @@ -141,21 +141,21 @@ namespace snmalloc template class ABA { - std::atomic ptr = nullptr; - std::atomic lock{false}; + stl::Atomic ptr = nullptr; + stl::Atomic lock{false}; public: // This method is used in Verona void init(T* x) { - ptr.store(x, std::memory_order_relaxed); + ptr.store(x, stl::memory_order_relaxed); } struct Cmp; Cmp read() { - while (lock.exchange(true, std::memory_order_acquire)) + while (lock.exchange(true, stl::memory_order_acquire)) Aal::pause(); # if !defined(NDEBUG) && !defined(SNMALLOC_DISABLE_ABA_VERIFY) @@ -185,7 +185,7 @@ namespace snmalloc ~Cmp() { - parent->lock.store(false, std::memory_order_release); + parent->lock.store(false, stl::memory_order_release); # if !defined(NDEBUG) && !defined(SNMALLOC_DISABLE_ABA_VERIFY) operation_in_flight = false; # endif @@ -195,7 +195,7 @@ namespace snmalloc // This method is used in Verona T* peek() { - return ptr.load(std::memory_order_relaxed); + return ptr.load(stl::memory_order_relaxed); } }; #endif diff --git a/src/snmalloc/ds/combininglock.h b/src/snmalloc/ds/combininglock.h index 89a4bc258..e6e796dfd 100644 --- a/src/snmalloc/ds/combininglock.h +++ b/src/snmalloc/ds/combininglock.h @@ -2,8 +2,7 @@ #include "../aal/aal.h" #include "../pal/pal.h" - -#include +#include "snmalloc/stl/atomic.h" namespace snmalloc { @@ -12,14 +11,14 @@ namespace snmalloc struct CombiningLock { // Fast path lock incase there is no contention. - std::atomic flag{false}; + stl::Atomic flag{false}; // MCS queue of work items - std::atomic last{nullptr}; + stl::Atomic last{nullptr}; void release() { - flag.store(false, std::memory_order_release); + flag.store(false, stl::memory_order_release); } }; @@ -86,10 +85,10 @@ namespace snmalloc // Status of the queue, set by the thread at the head of the queue, // When it makes the thread for this node either the head of the queue // or completes its work. - std::atomic status{LockStatus::WAITING}; + stl::Atomic status{LockStatus::WAITING}; // Used to store the queue - std::atomic next{nullptr}; + stl::Atomic next{nullptr}; // Stores the C++ lambda associated with this node in the queue. void (*f_raw)(CombiningLockNode*); @@ -98,7 +97,7 @@ namespace snmalloc void set_status(LockStatus s) { - status.store(s, std::memory_order_release); + status.store(s, stl::memory_order_release); } template @@ -111,7 +110,7 @@ namespace snmalloc else { if ( - node->status.exchange(message, std::memory_order_acq_rel) == + node->status.exchange(message, stl::memory_order_acq_rel) == LockStatus::SLEEPING) { Pal::notify_one_on_address(node->status); @@ -124,7 +123,7 @@ namespace snmalloc { if constexpr (!use_wait_on_address) { - while (status.load(std::memory_order_acquire) == LockStatus::WAITING) + while (status.load(stl::memory_order_acquire) == LockStatus::WAITING) Aal::pause(); } else @@ -132,14 +131,14 @@ namespace snmalloc int remaining = 100; while (remaining > 0) { - if (status.load(std::memory_order_acquire) != LockStatus::WAITING) + if (status.load(stl::memory_order_acquire) != LockStatus::WAITING) return; Aal::pause(); remaining--; } LockStatus expected = LockStatus::WAITING; if (status.compare_exchange_strong( - expected, LockStatus::SLEEPING, std::memory_order_acq_rel)) + expected, LockStatus::SLEEPING, stl::memory_order_acq_rel)) { Pal::wait_on_address(status, LockStatus::SLEEPING); } @@ -150,18 +149,18 @@ namespace snmalloc { // There is contention for the lock, we need to add our work to the // queue of pending work - auto prev = lock.last.exchange(this, std::memory_order_acq_rel); + auto prev = lock.last.exchange(this, stl::memory_order_acq_rel); if (prev != nullptr) { // If we aren't the head, link into predecessor - prev->next.store(this, std::memory_order_release); + prev->next.store(this, stl::memory_order_release); // Wait to for predecessor to complete wait(); // Determine if another thread completed our work. - if (status.load(std::memory_order_acquire) == LockStatus::DONE) + if (status.load(stl::memory_order_acquire) == LockStatus::DONE) return; } else @@ -170,9 +169,9 @@ namespace snmalloc // lock. As we are in the queue future requests shouldn't try to // acquire the fast path lock, but stale views of the queue being empty // could still be concurrent with this thread. - while (lock.flag.exchange(true, std::memory_order_acquire)) + while (lock.flag.exchange(true, stl::memory_order_acquire)) { - while (lock.flag.load(std::memory_order_relaxed)) + while (lock.flag.load(stl::memory_order_relaxed)) { Aal::pause(); } @@ -190,14 +189,14 @@ namespace snmalloc while (true) { // Start pulling in the next element of the queue - auto n = curr->next.load(std::memory_order_acquire); + auto n = curr->next.load(stl::memory_order_acquire); Aal::prefetch(n); // Perform work for head of the queue curr->f_raw(curr); // Determine if there are more elements. - n = curr->next.load(std::memory_order_acquire); + n = curr->next.load(stl::memory_order_acquire); if (n == nullptr) break; // Signal this work was completed and move on to @@ -212,8 +211,8 @@ namespace snmalloc if (lock.last.compare_exchange_strong( curr_c, nullptr, - std::memory_order_release, - std::memory_order_relaxed)) + stl::memory_order_release, + stl::memory_order_relaxed)) { // Queue was successfully closed. // Notify last element the work was completed. @@ -224,10 +223,10 @@ namespace snmalloc // Failed to close the queue wait for next thread to be // added. - while (curr->next.load(std::memory_order_relaxed) == nullptr) + while (curr->next.load(stl::memory_order_relaxed) == nullptr) Aal::pause(); - auto n = curr->next.load(std::memory_order_acquire); + auto n = curr->next.load(stl::memory_order_acquire); // As we had to wait, give the job to the next thread // to carry on performing the work. @@ -272,12 +271,12 @@ namespace snmalloc inline void with(CombiningLock& lock, F&& f) { // Test if no one is waiting - if (SNMALLOC_LIKELY(lock.last.load(std::memory_order_relaxed) == nullptr)) + if (SNMALLOC_LIKELY(lock.last.load(stl::memory_order_relaxed) == nullptr)) { // No one was waiting so low contention. Attempt to acquire the flag // lock. if (SNMALLOC_LIKELY( - lock.flag.exchange(true, std::memory_order_acquire) == false)) + lock.flag.exchange(true, stl::memory_order_acquire) == false)) { // We grabbed the lock. // Execute the thunk. diff --git a/src/snmalloc/ds/flaglock.h b/src/snmalloc/ds/flaglock.h index 546350485..8676ce996 100644 --- a/src/snmalloc/ds/flaglock.h +++ b/src/snmalloc/ds/flaglock.h @@ -2,14 +2,13 @@ #include "../aal/aal.h" #include "../pal/pal.h" - -#include +#include "snmalloc/stl/atomic.h" namespace snmalloc { /** * @brief The DebugFlagWord struct - * Wrapper for std::atomic_flag so that we can examine + * Wrapper for stl::AtomicBool so that we can examine * the re-entrancy problem at debug mode. */ struct DebugFlagWord @@ -20,7 +19,7 @@ namespace snmalloc * @brief flag * The underlying atomic field. */ - std::atomic_bool flag{false}; + stl::AtomicBool flag{false}; constexpr DebugFlagWord() = default; @@ -62,7 +61,7 @@ namespace snmalloc * @brief owner * We use the Pal to provide the ThreadIdentity. */ - std::atomic owner = ThreadIdentity(); + stl::Atomic owner = ThreadIdentity(); /** * @brief get_thread_identity @@ -82,7 +81,7 @@ namespace snmalloc */ struct ReleaseFlagWord { - std::atomic_bool flag{false}; + stl::AtomicBool flag{false}; constexpr ReleaseFlagWord() = default; @@ -112,7 +111,7 @@ namespace snmalloc public: FlagLock(FlagWord& lock) : lock(lock) { - while (lock.flag.exchange(true, std::memory_order_acquire)) + while (lock.flag.exchange(true, stl::memory_order_acquire)) { // assert_not_owned_by_current_thread is only called when the first // acquiring is failed; which means the lock is already held somewhere @@ -120,7 +119,7 @@ namespace snmalloc lock.assert_not_owned_by_current_thread(); // This loop is better for spin-waiting because it won't issue // expensive write operation (xchg for example). - while (lock.flag.load(std::memory_order_relaxed)) + while (lock.flag.load(stl::memory_order_relaxed)) { Aal::pause(); } @@ -131,7 +130,7 @@ namespace snmalloc ~FlagLock() { lock.clear_owner(); - lock.flag.store(false, std::memory_order_release); + lock.flag.store(false, stl::memory_order_release); } }; diff --git a/src/snmalloc/ds/mpmcstack.h b/src/snmalloc/ds/mpmcstack.h index e6a3b1d9f..5024b196c 100644 --- a/src/snmalloc/ds/mpmcstack.h +++ b/src/snmalloc/ds/mpmcstack.h @@ -16,17 +16,17 @@ namespace snmalloc #ifdef SNMALLOC_THREAD_SANITIZER_ENABLED __attribute__((no_sanitize("thread"))) static T* - racy_read(std::atomic& ptr) + racy_read(stl::Atomic& ptr) { // reinterpret_cast is required as TSAN still instruments - // std::atomic operations, even if you disable TSAN on + // stl::Atomic operations, even if you disable TSAN on // the function. return *reinterpret_cast(&ptr); } #else - static T* racy_read(std::atomic& ptr) + static T* racy_read(stl::Atomic& ptr) { - return ptr.load(std::memory_order_relaxed); + return ptr.load(stl::memory_order_relaxed); } #endif @@ -36,8 +36,8 @@ namespace snmalloc void push(T* item) { static_assert( - std::is_same>::value, - "T->next must be an std::atomic"); + std::is_same>::value, + "T->next must be an stl::Atomic"); return push(item, item); } @@ -50,7 +50,7 @@ namespace snmalloc do { auto top = cmp.ptr(); - last->next.store(top, std::memory_order_release); + last->next.store(top, stl::memory_order_release); } while (!cmp.store_conditional(first)); } diff --git a/src/snmalloc/ds/singleton.h b/src/snmalloc/ds/singleton.h index 174128e77..c8edae223 100644 --- a/src/snmalloc/ds/singleton.h +++ b/src/snmalloc/ds/singleton.h @@ -2,8 +2,8 @@ #include "../ds_core/ds_core.h" #include "flaglock.h" +#include "snmalloc/stl/atomic.h" -#include #include namespace snmalloc @@ -17,7 +17,7 @@ namespace snmalloc class Singleton { inline static FlagWord flag; - inline static std::atomic initialised{false}; + inline static stl::Atomic initialised{false}; inline static Object obj; public: @@ -31,13 +31,13 @@ namespace snmalloc // If defined should be initially false; SNMALLOC_ASSERT(first == nullptr || *first == false); - if (SNMALLOC_UNLIKELY(!initialised.load(std::memory_order_acquire))) + if (SNMALLOC_UNLIKELY(!initialised.load(stl::memory_order_acquire))) { with(flag, [&]() { if (!initialised) { init(&obj); - initialised.store(true, std::memory_order_release); + initialised.store(true, stl::memory_order_release); if (first != nullptr) *first = true; } diff --git a/src/snmalloc/ds_core/bits.h b/src/snmalloc/ds_core/bits.h index b192c8275..8ead75487 100644 --- a/src/snmalloc/ds_core/bits.h +++ b/src/snmalloc/ds_core/bits.h @@ -6,8 +6,8 @@ // #define USE_LZCNT #include "defines.h" +#include "snmalloc/stl/atomic.h" -#include #include #include #include diff --git a/src/snmalloc/ds_core/ptrwrap.h b/src/snmalloc/ds_core/ptrwrap.h index ae8fef9e2..01ad4639a 100644 --- a/src/snmalloc/ds_core/ptrwrap.h +++ b/src/snmalloc/ds_core/ptrwrap.h @@ -2,8 +2,7 @@ #include "concept.h" #include "defines.h" - -#include +#include "snmalloc/stl/atomic.h" namespace snmalloc { @@ -41,7 +40,7 @@ namespace snmalloc using Pointer = T*; template - using AtomicPointer = std::atomic; + using AtomicPointer = stl::Atomic; /** * Summaries of StrictProvenance metadata. We abstract away the particular @@ -450,17 +449,17 @@ namespace snmalloc /** * - * Wrap a std::atomic with bounds annotation and speak in terms of + * Wrap a stl::Atomic with bounds annotation and speak in terms of * bounds-annotated pointers at the interface. * * Note the membranous sleight of hand being pulled here: this class puts - * annotations around an un-annotated std::atomic, to appease C++, yet + * annotations around an un-annotated stl::Atomic, to appease C++, yet * will expose or consume only CapPtr with the same bounds annotation. */ template class AtomicCapPtr { - std::atomic unsafe_capptr; + stl::Atomic unsafe_capptr; public: /** @@ -487,7 +486,7 @@ namespace snmalloc return CapPtr(this->unsafe_capptr); } - // Our copy-assignment operator follows std::atomic and returns a copy of + // Our copy-assignment operator follows stl::Atomic and returns a copy of // the RHS. Clang finds this surprising; we suppress the warning. // NOLINTNEXTLINE(misc-unconventional-assign-operator) SNMALLOC_FAST_PATH CapPtr operator=(CapPtr p) noexcept @@ -497,21 +496,21 @@ namespace snmalloc } SNMALLOC_FAST_PATH CapPtr - load(std::memory_order order = std::memory_order_seq_cst) noexcept + load(stl::MemoryOrder order = stl::memory_order_seq_cst) noexcept { return CapPtr::unsafe_from(this->unsafe_capptr.load(order)); } SNMALLOC_FAST_PATH void store( CapPtr desired, - std::memory_order order = std::memory_order_seq_cst) noexcept + stl::MemoryOrder order = stl::memory_order_seq_cst) noexcept { this->unsafe_capptr.store(desired.unsafe_ptr(), order); } SNMALLOC_FAST_PATH CapPtr exchange( CapPtr desired, - std::memory_order order = std::memory_order_seq_cst) noexcept + stl::MemoryOrder order = stl::memory_order_seq_cst) noexcept { return CapPtr::unsafe_from( this->unsafe_capptr.exchange(desired.unsafe_ptr(), order)); diff --git a/src/snmalloc/mem/freelist.h b/src/snmalloc/mem/freelist.h index f49004d93..37cdfa365 100644 --- a/src/snmalloc/mem/freelist.h +++ b/src/snmalloc/mem/freelist.h @@ -197,7 +197,7 @@ namespace snmalloc { auto n_wild = Object::decode_next( address_cast(&this->next_object), - this->atomic_next_object.load(std::memory_order_acquire), + this->atomic_next_object.load(stl::memory_order_acquire), key, key_tweak); auto n_tame = domesticate(n_wild); @@ -472,7 +472,7 @@ namespace snmalloc // so requires release semantics. curr->atomic_next_object.store( encode_next(address_cast(&curr->next_object), next, key, key_tweak), - std::memory_order_release); + stl::memory_order_release); } template< @@ -491,7 +491,7 @@ namespace snmalloc BQueuePtr(nullptr), key, key_tweak), - std::memory_order_relaxed); + stl::memory_order_relaxed); } }; diff --git a/src/snmalloc/mem/freelist_queue.h b/src/snmalloc/mem/freelist_queue.h index fb38f7c88..0688a75a5 100644 --- a/src/snmalloc/mem/freelist_queue.h +++ b/src/snmalloc/mem/freelist_queue.h @@ -2,8 +2,7 @@ #include "../ds/ds.h" #include "freelist.h" - -#include +#include "snmalloc/stl/atomic.h" namespace snmalloc { @@ -61,14 +60,14 @@ namespace snmalloc freelist::HeadPtr stub_ptr = freelist::HeadPtr::unsafe_from(&stub); freelist::Object::atomic_store_null(stub_ptr, Key, Key_tweak); front.store(freelist::QueuePtr::unsafe_from(&stub)); - back.store(nullptr, std::memory_order_relaxed); + back.store(nullptr, stl::memory_order_relaxed); invariant(); } freelist::QueuePtr destroy() { freelist::QueuePtr fnt = front.load(); - back.store(nullptr, std::memory_order_relaxed); + back.store(nullptr, stl::memory_order_relaxed); if (address_cast(front.load()) == address_cast(&stub)) return nullptr; return fnt; @@ -116,7 +115,7 @@ namespace snmalloc // * Needs to be acquire, so linking into the list does not race with // the other threads nullptr init of the next field. freelist::QueuePtr prev = - back.exchange(capptr_rewild(last), std::memory_order_acq_rel); + back.exchange(capptr_rewild(last), stl::memory_order_acq_rel); if (SNMALLOC_LIKELY(prev != nullptr)) { @@ -149,7 +148,7 @@ namespace snmalloc SNMALLOC_ASSERT(front.load() != nullptr); // Use back to bound, so we don't handle new entries. - auto b = back.load(std::memory_order_relaxed); + auto b = back.load(stl::memory_order_relaxed); freelist::HeadPtr curr = domesticate_head(front.load()); while (address_cast(curr) != address_cast(b)) diff --git a/src/snmalloc/mem/pooled.h b/src/snmalloc/mem/pooled.h index 4e7c76884..489d72f9d 100644 --- a/src/snmalloc/mem/pooled.h +++ b/src/snmalloc/mem/pooled.h @@ -15,14 +15,14 @@ namespace snmalloc template class PoolState; + // clang-format off #ifdef __cpp_concepts template concept Constructable = requires() { - { - C::make() - } -> ConceptSame>; - }; + { C::make() } -> ConceptSame>; + }; #endif // __cpp_concepts + /** * Required to be implemented by all types that are pooled. @@ -46,7 +46,7 @@ namespace snmalloc capptr::Alloc next{nullptr}; /// Used by the pool to keep the list of all entries ever created. capptr::Alloc list_next; - std::atomic in_use{false}; + stl::Atomic in_use{false}; public: void set_in_use() diff --git a/src/snmalloc/mem/remotecache.h b/src/snmalloc/mem/remotecache.h index 585fb9146..2780da777 100644 --- a/src/snmalloc/mem/remotecache.h +++ b/src/snmalloc/mem/remotecache.h @@ -6,9 +6,9 @@ #include "metadata.h" #include "remoteallocator.h" #include "sizeclasstable.h" +#include "snmalloc/stl/atomic.h" #include -#include namespace snmalloc { diff --git a/src/snmalloc/pal/pal_apple.h b/src/snmalloc/pal/pal_apple.h index f6a7f1a2d..f96a8c0e4 100644 --- a/src/snmalloc/pal/pal_apple.h +++ b/src/snmalloc/pal/pal_apple.h @@ -321,10 +321,10 @@ namespace snmalloc # endif template - static void wait_on_address(std::atomic& addr, T expected) + static void wait_on_address(stl::Atomic& addr, T expected) { [[maybe_unused]] int errno_backup = errno; - while (addr.load(std::memory_order_relaxed) == expected) + while (addr.load(stl::memory_order_relaxed) == expected) { # ifdef SNMALLOC_APPLE_HAS_OS_SYNC_WAIT_ON_ADDRESS if ( @@ -349,7 +349,7 @@ namespace snmalloc } template - static void notify_one_on_address(std::atomic& addr) + static void notify_one_on_address(stl::Atomic& addr) { # ifdef SNMALLOC_APPLE_HAS_OS_SYNC_WAIT_ON_ADDRESS os_sync_wake_by_address_any(&addr, sizeof(T), 0); @@ -366,7 +366,7 @@ namespace snmalloc } template - static void notify_all_on_address(std::atomic& addr) + static void notify_all_on_address(stl::Atomic& addr) { # ifdef SNMALLOC_APPLE_HAS_OS_SYNC_WAIT_ON_ADDRESS os_sync_wake_by_address_all(&addr, sizeof(T), 0); diff --git a/src/snmalloc/pal/pal_consts.h b/src/snmalloc/pal/pal_consts.h index c4c4c25a2..293ad3318 100644 --- a/src/snmalloc/pal/pal_consts.h +++ b/src/snmalloc/pal/pal_consts.h @@ -1,8 +1,7 @@ #pragma once #include "../ds_core/ds_core.h" - -#include +#include "snmalloc/stl/atomic.h" namespace snmalloc { diff --git a/src/snmalloc/pal/pal_ds.h b/src/snmalloc/pal/pal_ds.h index 008d1f2c2..f9f92defc 100644 --- a/src/snmalloc/pal/pal_ds.h +++ b/src/snmalloc/pal/pal_ds.h @@ -1,8 +1,7 @@ #pragma once #include "../ds_core/ds_core.h" - -#include +#include "snmalloc/stl/atomic.h" namespace snmalloc { @@ -12,10 +11,10 @@ namespace snmalloc /** * List of callbacks to notify */ - std::atomic elements{nullptr}; + stl::Atomic elements{nullptr}; static_assert( - std::is_same>::value, + std::is_same>::value, "Required pal_next type."); public: @@ -57,7 +56,7 @@ namespace snmalloc */ struct PalNotificationObject { - std::atomic pal_next = nullptr; + stl::Atomic pal_next = nullptr; void (*pal_notify)(PalNotificationObject* self); @@ -103,7 +102,7 @@ namespace snmalloc template friend class PalList; - std::atomic pal_next; + stl::Atomic pal_next; void (*pal_notify)(PalTimerObject* self); @@ -140,10 +139,10 @@ namespace snmalloc void check(uint64_t time_ms) { - static std::atomic_bool lock{false}; + static stl::AtomicBool lock{false}; // Deduplicate calls into here, and make single threaded. - if (lock.exchange(true, std::memory_order_acquire)) + if (lock.exchange(true, stl::memory_order_acquire)) return; timers.apply_all([time_ms](PalTimerObject* curr) { @@ -155,7 +154,7 @@ namespace snmalloc } }); - lock.store(false, std::memory_order_release); + lock.store(false, stl::memory_order_release); } }; } // namespace snmalloc diff --git a/src/snmalloc/pal/pal_freebsd.h b/src/snmalloc/pal/pal_freebsd.h index d967dc1b5..6c1e0740b 100644 --- a/src/snmalloc/pal/pal_freebsd.h +++ b/src/snmalloc/pal/pal_freebsd.h @@ -137,13 +137,13 @@ namespace snmalloc using WaitingWord = unsigned int; template - static void wait_on_address(std::atomic& addr, T expected) + static void wait_on_address(stl::Atomic& addr, T expected) { static_assert( sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord), "T must be the same size and alignment as WaitingWord"); int backup = errno; - while (addr.load(std::memory_order_relaxed) == expected) + while (addr.load(stl::memory_order_relaxed) == expected) { int ret = _umtx_op( &addr, @@ -159,7 +159,7 @@ namespace snmalloc } template - static void notify_one_on_address(std::atomic& addr) + static void notify_one_on_address(stl::Atomic& addr) { static_assert( sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord), @@ -168,7 +168,7 @@ namespace snmalloc } template - static void notify_all_on_address(std::atomic& addr) + static void notify_all_on_address(stl::Atomic& addr) { static_assert( sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord), diff --git a/src/snmalloc/pal/pal_linux.h b/src/snmalloc/pal/pal_linux.h index e1774fbd3..f98da7033 100644 --- a/src/snmalloc/pal/pal_linux.h +++ b/src/snmalloc/pal/pal_linux.h @@ -177,11 +177,11 @@ namespace snmalloc // give a try to SYS_getrandom # ifdef SYS_getrandom - static std::atomic_bool syscall_not_working = false; + static stl::AtomicBool syscall_not_working = false; // Relaxed ordering should be fine here. This function will be called // during early initialisation, which will examine the availability in a // protected routine. - if (false == syscall_not_working.load(std::memory_order_relaxed)) + if (false == syscall_not_working.load(stl::memory_order_relaxed)) { auto current = std::begin(buffer); auto target = std::end(buffer); @@ -224,7 +224,7 @@ namespace snmalloc // in this routine, the only possible situations should be ENOSYS // or EPERM (forbidden by seccomp, for example). SNMALLOC_ASSERT(errno == ENOSYS || errno == EPERM); - syscall_not_working.store(true, std::memory_order_relaxed); + syscall_not_working.store(true, stl::memory_order_relaxed); } else { @@ -246,13 +246,13 @@ namespace snmalloc using WaitingWord = int; template - static void wait_on_address(std::atomic& addr, T expected) + static void wait_on_address(stl::Atomic& addr, T expected) { int backup = errno; static_assert( sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord), "T must be the same size and alignment as WaitingWord"); - while (addr.load(std::memory_order_relaxed) == expected) + while (addr.load(stl::memory_order_relaxed) == expected) { long ret = syscall( SYS_futex, &addr, FUTEX_WAIT_PRIVATE, expected, nullptr, nullptr, 0); @@ -264,7 +264,7 @@ namespace snmalloc } template - static void notify_one_on_address(std::atomic& addr) + static void notify_one_on_address(stl::Atomic& addr) { static_assert( sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord), @@ -273,7 +273,7 @@ namespace snmalloc } template - static void notify_all_on_address(std::atomic& addr) + static void notify_all_on_address(stl::Atomic& addr) { static_assert( sizeof(T) == sizeof(WaitingWord) && alignof(T) == alignof(WaitingWord), diff --git a/src/snmalloc/pal/pal_tid_default.h b/src/snmalloc/pal/pal_tid_default.h index 678af98b0..c854159fa 100644 --- a/src/snmalloc/pal/pal_tid_default.h +++ b/src/snmalloc/pal/pal_tid_default.h @@ -1,6 +1,6 @@ #pragma once -#include +#include "snmalloc/stl/atomic.h" namespace snmalloc { @@ -18,7 +18,7 @@ namespace snmalloc static inline ThreadIdentity get_tid() noexcept { static thread_local size_t tid{0}; - static std::atomic tid_source{0}; + static stl::Atomic tid_source{0}; if (tid == 0) { @@ -27,4 +27,4 @@ namespace snmalloc return tid; } }; -} // namespace snmalloc \ No newline at end of file +} // namespace snmalloc diff --git a/src/snmalloc/pal/pal_windows.h b/src/snmalloc/pal/pal_windows.h index d025b1beb..20492706a 100644 --- a/src/snmalloc/pal/pal_windows.h +++ b/src/snmalloc/pal/pal_windows.h @@ -35,7 +35,7 @@ namespace snmalloc * A flag indicating that we have tried to register for low-memory * notifications. */ - static inline std::atomic registered_for_notifications; + static inline stl::Atomic registered_for_notifications; static inline HANDLE lowMemoryObject; /** @@ -240,9 +240,9 @@ namespace snmalloc using WaitingWord = char; template - static void wait_on_address(std::atomic& addr, T expected) + static void wait_on_address(stl::Atomic& addr, T expected) { - while (addr.load(std::memory_order_relaxed) == expected) + while (addr.load(stl::memory_order_relaxed) == expected) { if (::WaitOnAddress(&addr, &expected, sizeof(T), INFINITE)) break; @@ -250,13 +250,13 @@ namespace snmalloc } template - static void notify_one_on_address(std::atomic& addr) + static void notify_one_on_address(stl::Atomic& addr) { ::WakeByAddressSingle(&addr); } template - static void notify_all_on_address(std::atomic& addr) + static void notify_all_on_address(stl::Atomic& addr) { ::WakeByAddressAll(&addr); } diff --git a/src/snmalloc/stl/README.md b/src/snmalloc/stl/README.md new file mode 100644 index 000000000..1df92f613 --- /dev/null +++ b/src/snmalloc/stl/README.md @@ -0,0 +1,4 @@ +# Standard Library Implementation + +To support build environment without C++ STL, snmalloc can optionally use self-vendored STL functionalities. +To use self-vendored implementations, one need to set `SNMALLOC_USE_SELF_VENDORED_STL` to `ON` when configuring cmake. diff --git a/src/snmalloc/stl/atomic.h b/src/snmalloc/stl/atomic.h new file mode 100644 index 000000000..a28def439 --- /dev/null +++ b/src/snmalloc/stl/atomic.h @@ -0,0 +1,9 @@ +#pragma once + +#include "snmalloc/stl/common.h" + +#if SNMALLOC_USE_SELF_VENDORED_STL +# include "snmalloc/stl/gnu/atomic.h" +#else +# include "snmalloc/stl/cxx/atomic.h" +#endif diff --git a/src/snmalloc/stl/common.h b/src/snmalloc/stl/common.h new file mode 100644 index 000000000..9b47e323e --- /dev/null +++ b/src/snmalloc/stl/common.h @@ -0,0 +1,15 @@ +#pragma once + +#include "snmalloc/ds_core/defines.h" + +// Default on using the system STL. +#ifndef SNMALLOC_USE_SELF_VENDORED_STL +# define SNMALLOC_USE_SELF_VENDORED_STL 0 +#endif + +// Check that the vendored STL is only used with GNU/Clang extensions. +#if SNMALLOC_USE_SELF_VENDORED_STL +# if !defined(__GNUC__) && !defined(__clang__) +# error "cannot use vendored STL without GNU/Clang extensions" +# endif +#endif diff --git a/src/snmalloc/stl/cxx/README.md b/src/snmalloc/stl/cxx/README.md new file mode 100644 index 000000000..d8d5a2b41 --- /dev/null +++ b/src/snmalloc/stl/cxx/README.md @@ -0,0 +1,3 @@ +# CXX Standard Library + +This directory provides an interface to use default STL. diff --git a/src/snmalloc/stl/cxx/atomic.h b/src/snmalloc/stl/cxx/atomic.h new file mode 100644 index 000000000..a85d3feb0 --- /dev/null +++ b/src/snmalloc/stl/cxx/atomic.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +namespace snmalloc +{ + namespace stl + { + template + using Atomic = std::atomic; + + constexpr auto memory_order_relaxed = std::memory_order_relaxed; + constexpr auto memory_order_consume = std::memory_order_consume; + constexpr auto memory_order_acquire = std::memory_order_acquire; + constexpr auto memory_order_release = std::memory_order_release; + constexpr auto memory_order_acq_rel = std::memory_order_acq_rel; + constexpr auto memory_order_seq_cst = std::memory_order_seq_cst; + + using AtomicBool = std::atomic; + using MemoryOrder = std::memory_order; + } // namespace stl +} // namespace snmalloc diff --git a/src/snmalloc/stl/gnu/README.md b/src/snmalloc/stl/gnu/README.md new file mode 100644 index 000000000..7c513c1f0 --- /dev/null +++ b/src/snmalloc/stl/gnu/README.md @@ -0,0 +1,3 @@ +# Self-Vendored STL Using GNU Language Extensions + +This directory contains implementations of STL functionalities using GNU extensions. Such extensions are also available with clang. diff --git a/src/snmalloc/stl/gnu/atomic.h b/src/snmalloc/stl/gnu/atomic.h new file mode 100644 index 000000000..a8d9f42aa --- /dev/null +++ b/src/snmalloc/stl/gnu/atomic.h @@ -0,0 +1,232 @@ +#pragma once + +#include +#include // TODO: switch the vendored headers when we have them. + +namespace snmalloc +{ + namespace stl + { + + enum class MemoryOrder : int + { + RELAXED = __ATOMIC_RELAXED, + CONSUME = __ATOMIC_CONSUME, + ACQUIRE = __ATOMIC_ACQUIRE, + RELEASE = __ATOMIC_RELEASE, + ACQ_REL = __ATOMIC_ACQ_REL, + SEQ_CST = __ATOMIC_SEQ_CST + }; + + constexpr MemoryOrder memory_order_relaxed = MemoryOrder::RELAXED; + constexpr MemoryOrder memory_order_consume = MemoryOrder::CONSUME; + constexpr MemoryOrder memory_order_acquire = MemoryOrder::ACQUIRE; + constexpr MemoryOrder memory_order_release = MemoryOrder::RELEASE; + constexpr MemoryOrder memory_order_acq_rel = MemoryOrder::ACQ_REL; + constexpr MemoryOrder memory_order_seq_cst = MemoryOrder::SEQ_CST; + + template + class Atomic + { + static_assert( + std::is_trivially_copyable_v && std::is_copy_constructible_v && + std::is_move_constructible_v && std::is_copy_assignable_v && + std::is_move_assignable_v, + "Atomic requires T to be trivially copyable, copy " + "constructible, move constructible, copy assignable, " + "and move assignable."); + + static_assert( + std::has_unique_object_representations_v, + "vendored Atomic only supports types with unique object " + "representations"); + + // type conversion helper to avoid long c++ style casts + SNMALLOC_FAST_PATH static int order(MemoryOrder mem_ord) + { + return static_cast(mem_ord); + } + + SNMALLOC_FAST_PATH static int infer_failure_order(MemoryOrder mem_ord) + { + if (mem_ord == MemoryOrder::RELEASE) + return order(MemoryOrder::RELAXED); + if (mem_ord == MemoryOrder::ACQ_REL) + return order(MemoryOrder::ACQUIRE); + return order(mem_ord); + } + + SNMALLOC_FAST_PATH static T* addressof(T& ref) + { + return __builtin_addressof(ref); + } + + // From libc++: + // require types that are 1, 2, 4, 8, or 16 bytes in length to be aligned + // to at least their size to be potentially + // used lock-free + static constexpr size_t MIN_ALIGNMENT = + (sizeof(T) & (sizeof(T) - 1)) || (sizeof(T) > 16) ? 0 : sizeof(T); + + static constexpr size_t REQUIRED_ALIGNMENT = + alignof(T) > MIN_ALIGNMENT ? alignof(T) : MIN_ALIGNMENT; + + alignas(REQUIRED_ALIGNMENT) T val; + + public: + SNMALLOC_FAST_PATH constexpr Atomic() = default; + + SNMALLOC_FAST_PATH constexpr Atomic(T v) : val(v) {} + + SNMALLOC_FAST_PATH Atomic(const Atomic&) = delete; + SNMALLOC_FAST_PATH Atomic& operator=(const Atomic&) = delete; + + // Atomic load. + SNMALLOC_FAST_PATH operator T() + { + return load(); + } + + SNMALLOC_FAST_PATH T load(MemoryOrder mem_ord = MemoryOrder::SEQ_CST) + { + T res; + __atomic_load(addressof(val), addressof(res), order(mem_ord)); + return res; + } + + // Atomic store. + // NOLINTNEXTLINE(misc-unconventional-assign-operator) + SNMALLOC_FAST_PATH + T operator=(T rhs) + { + store(rhs); + return rhs; + } + + SNMALLOC_FAST_PATH void + store(T rhs, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) + { + __atomic_store(addressof(val), addressof(rhs), order(mem_ord)); + } + + // Atomic compare exchange + SNMALLOC_FAST_PATH bool compare_exchange_strong( + T& expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) + { + return __atomic_compare_exchange( + addressof(val), + addressof(expected), + addressof(desired), + false, + order(mem_ord), + infer_failure_order(mem_ord)); + } + + // Atomic compare exchange (separate success and failure memory orders) + SNMALLOC_FAST_PATH bool compare_exchange_strong( + T& expected, + T desired, + MemoryOrder success_order, + MemoryOrder failure_order) + { + return __atomic_compare_exchange( + addressof(val), + addressof(expected), + addressof(desired), + false, + order(success_order), + order(failure_order)); + } + + // Atomic compare exchange (weak version) + SNMALLOC_FAST_PATH bool compare_exchange_weak( + T& expected, T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) + { + return __atomic_compare_exchange( + addressof(val), + addressof(expected), + addressof(desired), + true, + order(mem_ord), + infer_failure_order(mem_ord)); + } + + // Atomic compare exchange (weak version with separate success and failure + // memory orders) + SNMALLOC_FAST_PATH bool compare_exchange_weak( + T& expected, + T desired, + MemoryOrder success_order, + MemoryOrder failure_order) + { + return __atomic_compare_exchange( + addressof(val), + addressof(expected), + addressof(desired), + true, + order(success_order), + order(failure_order)); + } + + SNMALLOC_FAST_PATH T + exchange(T desired, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) + { + T ret; + __atomic_exchange( + addressof(val), addressof(desired), addressof(ret), order(mem_ord)); + return ret; + } + + SNMALLOC_FAST_PATH T + fetch_add(T increment, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) + { + static_assert(std::is_integral_v, "T must be an integral type."); + return __atomic_fetch_add(addressof(val), increment, order(mem_ord)); + } + + SNMALLOC_FAST_PATH T + fetch_or(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) + { + static_assert(std::is_integral_v, "T must be an integral type."); + return __atomic_fetch_or(addressof(val), mask, order(mem_ord)); + } + + SNMALLOC_FAST_PATH T + fetch_and(T mask, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) + { + static_assert(std::is_integral_v, "T must be an integral type."); + return __atomic_fetch_and(addressof(val), mask, order(mem_ord)); + } + + SNMALLOC_FAST_PATH T + fetch_sub(T decrement, MemoryOrder mem_ord = MemoryOrder::SEQ_CST) + { + static_assert(std::is_integral_v, "T must be an integral type."); + return __atomic_fetch_sub(addressof(val), decrement, order(mem_ord)); + } + + SNMALLOC_FAST_PATH T operator++() + { + static_assert(std::is_integral_v, "T must be an integral type."); + return __atomic_add_fetch( + addressof(val), 1, order(MemoryOrder::SEQ_CST)); + } + + SNMALLOC_FAST_PATH const T operator++(int) + { + static_assert(std::is_integral_v, "T must be an integral type."); + return __atomic_fetch_add( + addressof(val), 1, order(MemoryOrder::SEQ_CST)); + } + + SNMALLOC_FAST_PATH T operator-=(T decrement) + { + static_assert(std::is_integral_v, "T must be an integral type."); + return __atomic_sub_fetch( + addressof(val), decrement, order(MemoryOrder::SEQ_CST)); + } + }; + + using AtomicBool = Atomic; + } // namespace stl +} // namespace snmalloc diff --git a/src/test/func/client_meta/client_meta.cc b/src/test/func/client_meta/client_meta.cc index 0359666bb..dd576d304 100644 --- a/src/test/func/client_meta/client_meta.cc +++ b/src/test/func/client_meta/client_meta.cc @@ -5,6 +5,7 @@ #include "test/setup.h" +#include #include #include #include diff --git a/src/test/func/miracle_ptr/miracle_ptr.cc b/src/test/func/miracle_ptr/miracle_ptr.cc index c4e4783bb..8577dfb5e 100644 --- a/src/test/func/miracle_ptr/miracle_ptr.cc +++ b/src/test/func/miracle_ptr/miracle_ptr.cc @@ -15,6 +15,7 @@ int main() # include "test/setup.h" +# include # include # include # include @@ -201,4 +202,4 @@ int main() # endif return 0; } -#endif \ No newline at end of file +#endif diff --git a/src/test/perf/low_memory/low-memory.cc b/src/test/perf/low_memory/low-memory.cc index fa2997fdf..703dad4c6 100644 --- a/src/test/perf/low_memory/low-memory.cc +++ b/src/test/perf/low_memory/low-memory.cc @@ -1,3 +1,4 @@ +#include #include #include #include diff --git a/src/test/perf/msgpass/msgpass.cc b/src/test/perf/msgpass/msgpass.cc index 7e639a02b..3f43e13dd 100644 --- a/src/test/perf/msgpass/msgpass.cc +++ b/src/test/perf/msgpass/msgpass.cc @@ -275,7 +275,7 @@ int main(int argc, char** argv) /* Spawn proxies */ for (size_t i = param.N_CONSUMER; i < param.N_QUEUE; i++) { - queue_threads[i] = std::thread(proxy, ¶m, i); + queue_threads[i] = std::thread(::proxy, ¶m, i); } /* Spawn producers */