From 3355f8e9163b8463bb0de6253e04fb3b6c49c70f Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Tue, 17 Jul 2018 09:15:39 -0600 Subject: [PATCH] opal/atomic: always use C11 atomics if available This commit disables the use of both the builtin and hand-written atomics if proper C11 atomic support is detected. This is the first step towards requiring the availability of C11 atomics for the C compiler used to build Open MPI. Signed-off-by: Nathan Hjelm --- opal/class/opal_object.c | 2 +- opal/include/opal/sys/atomic.h | 8 + opal/include/opal/sys/atomic_stdc.h | 235 +++++++++++++++++++++++ opal/include/opal_stdatomic.h | 28 +++ opal/include/opal_stdint.h | 2 + opal/mca/btl/vader/btl_vader_component.c | 5 +- opal/threads/mutex_unix.h | 8 +- orte/runtime/orte_locks.c | 8 +- 8 files changed, 286 insertions(+), 10 deletions(-) create mode 100644 opal/include/opal/sys/atomic_stdc.h diff --git a/opal/class/opal_object.c b/opal/class/opal_object.c index cd09d647f66..64fef5712fc 100644 --- a/opal/class/opal_object.c +++ b/opal/class/opal_object.c @@ -55,7 +55,7 @@ int opal_class_init_epoch = 1; /* * Local variables */ -static opal_atomic_lock_t class_lock = { { OPAL_ATOMIC_LOCK_UNLOCKED } }; +static opal_atomic_lock_t class_lock = OPAL_ATOMIC_LOCK_INIT; static void** classes = NULL; static int num_classes = 0; static int max_classes = 0; diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h index b9967f2117b..c5d0951f357 100644 --- a/opal/include/opal/sys/atomic.h +++ b/opal/include/opal/sys/atomic.h @@ -58,6 +58,12 @@ #include "opal/sys/architecture.h" #include "opal_stdatomic.h" +#if OPAL_C_HAVE__ATOMIC + +#include "atomic_stdc.h" + +#else /* !OPAL_C_HAVE__ATOMIC */ + /* do some quick #define cleanup in cases where we are doing testing... */ #ifdef OPAL_DISABLE_INLINE_ASM @@ -643,6 +649,8 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, vo */ #include "opal/sys/atomic_impl.h" +#endif /* !OPAL_C_HAVE__ATOMIC */ + END_C_DECLS #endif /* OPAL_SYS_ATOMIC_H */ diff --git a/opal/include/opal/sys/atomic_stdc.h b/opal/include/opal/sys/atomic_stdc.h new file mode 100644 index 00000000000..9815e43b81c --- /dev/null +++ b/opal/include/opal/sys/atomic_stdc.h @@ -0,0 +1,235 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/* This file provides shims between the opal atomics interface and the C11 atomics interface. It + * is intended as the first step in moving to using C11 atomics across the entire codebase. Once + * all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then + * this shim will go away and the codebase will be updated to use C11's atomic support + * directly. + * This shim contains some functions already present in atomic_impl.h because we do not include + * atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate + * definitions to be worthwhile. */ + +#if !defined(OPAL_ATOMIC_STDC_H) +#define OPAL_ATOMIC_STDC_H + +#include +#include +#include "opal/include/opal_stdint.h" + +#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 +#define OPAL_HAVE_ATOMIC_SWAP_32 1 + +#define OPAL_HAVE_ATOMIC_MATH_32 1 +#define OPAL_HAVE_ATOMIC_ADD_32 1 +#define OPAL_HAVE_ATOMIC_AND_32 1 +#define OPAL_HAVE_ATOMIC_OR_32 1 +#define OPAL_HAVE_ATOMIC_XOR_32 1 +#define OPAL_HAVE_ATOMIC_SUB_32 1 + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 +#define OPAL_HAVE_ATOMIC_SWAP_64 1 + +#define OPAL_HAVE_ATOMIC_MATH_64 1 +#define OPAL_HAVE_ATOMIC_ADD_64 1 +#define OPAL_HAVE_ATOMIC_AND_64 1 +#define OPAL_HAVE_ATOMIC_OR_64 1 +#define OPAL_HAVE_ATOMIC_XOR_64 1 +#define OPAL_HAVE_ATOMIC_SUB_64 1 + +#define OPAL_HAVE_ATOMIC_LLSC_32 0 +#define OPAL_HAVE_ATOMIC_LLSC_64 0 +#define OPAL_HAVE_ATOMIC_LLSC_PTR 0 + +#define OPAL_HAVE_ATOMIC_MIN_32 1 +#define OPAL_HAVE_ATOMIC_MAX_32 1 + +#define OPAL_HAVE_ATOMIC_MIN_64 1 +#define OPAL_HAVE_ATOMIC_MAX_64 1 + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 + +#define OPAL_HAVE_ATOMIC_SPINLOCKS 1 + +static inline void opal_atomic_mb (void) +{ + atomic_thread_fence (memory_order_seq_cst); +} + +static inline void opal_atomic_wmb (void) +{ + atomic_thread_fence (memory_order_release); +} + +static inline void opal_atomic_rmb (void) +{ + atomic_thread_fence (memory_order_acquire); +} + +#define opal_atomic_compare_exchange_strong_32 atomic_compare_exchange_strong +#define opal_atomic_compare_exchange_strong_64 atomic_compare_exchange_strong +#define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong +#define opal_atomic_compare_exchange_strong_ptr atomic_compare_exchange_strong +#define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) + +#define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) + +#define opal_atomic_compare_exchange_strong atomic_compare_exchange_strong +#define opal_atomic_compare_exchange_strong_acq(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_rel(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed) + +#define opal_atomic_swap_32 atomic_exchange +#define opal_atomic_swap_64 atomic_exchange +#define opal_atomic_swap_ptr atomic_exchange + +#define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator) \ + static inline type opal_atomic_fetch_ ## op ##_## bits (opal_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op (addr, value); \ + } \ + \ + static inline type opal_atomic_## op ## _fetch_ ## bits (opal_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op (addr, value) operator value; \ + } + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &) + +#define opal_atomic_add (void) atomic_fetch_add + +static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = opal_atomic_fetch_min_32 (addr, value); + return old <= value ? old : value; +} + +static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = opal_atomic_fetch_max_32 (addr, value); + return old >= value ? old : value; +} + +static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = opal_atomic_fetch_min_64 (addr, value); + return old <= value ? old : value; +} + +static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = opal_atomic_fetch_max_64 (addr, value); + return old >= value ? old : value; +} + +#define OPAL_ATOMIC_LOCK_UNLOCKED false +#define OPAL_ATOMIC_LOCK_LOCKED true + +#define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT + +typedef atomic_flag opal_atomic_lock_t; + +/* + * Lock initialization function. It set the lock to UNLOCKED. + */ +static inline void opal_atomic_lock_init (opal_atomic_lock_t *lock, bool value) +{ + atomic_flag_clear (lock); +} + + +static inline int opal_atomic_trylock (opal_atomic_lock_t *lock) +{ + return (int) atomic_flag_test_and_set (lock); +} + + +static inline void opal_atomic_lock(opal_atomic_lock_t *lock) +{ + while (opal_atomic_trylock (lock)) { + } +} + + +static inline void opal_atomic_unlock (opal_atomic_lock_t *lock) +{ + atomic_flag_clear (lock); +} + +#endif /* !defined(OPAL_ATOMIC_STDC_H) */ diff --git a/opal/include/opal_stdatomic.h b/opal/include/opal_stdatomic.h index d80cfb34292..816a77bfcd6 100644 --- a/opal/include/opal_stdatomic.h +++ b/opal/include/opal_stdatomic.h @@ -14,6 +14,9 @@ #include "opal_stdint.h" +#if !OPAL_C_HAVE__ATOMIC + +typedef volatile int opal_atomic_int_t; typedef volatile long opal_atomic_long_t; typedef volatile int32_t opal_atomic_int32_t; @@ -22,6 +25,7 @@ typedef volatile int64_t opal_atomic_int64_t; typedef volatile uint64_t opal_atomic_uint64_t; typedef volatile size_t opal_atomic_size_t; +typedef volatile ssize_t opal_atomic_ssize_t; typedef volatile intptr_t opal_atomic_intptr_t; typedef volatile uintptr_t opal_atomic_uintptr_t; @@ -31,5 +35,29 @@ typedef volatile opal_int128_t opal_atomic_int128_t; #endif +#else /* OPAL_HAVE_C__ATOMIC */ + +#include + +typedef atomic_int opal_atomic_int_t; +typedef atomic_long opal_atomic_long_t; + +typedef _Atomic int32_t opal_atomic_int32_t; +typedef _Atomic uint32_t opal_atomic_uint32_t; +typedef _Atomic int64_t opal_atomic_int64_t; +typedef _Atomic uint64_t opal_atomic_uint64_t; + +typedef _Atomic size_t opal_atomic_size_t; +typedef _Atomic ssize_t opal_atomic_ssize_t; +typedef _Atomic intptr_t opal_atomic_intptr_t; +typedef _Atomic uintptr_t opal_atomic_uintptr_t; + +#if HAVE_OPAL_INT128_T + +typedef _Atomic opal_int128_t opal_atomic_int128_t; + +#endif + +#endif /* OPAL_HAVE_C__ATOMIC */ #endif /* !defined(OPAL_STDATOMIC_H) */ diff --git a/opal/include/opal_stdint.h b/opal/include/opal_stdint.h index 4089cb55a99..efbd72c645d 100644 --- a/opal/include/opal_stdint.h +++ b/opal/include/opal_stdint.h @@ -28,6 +28,8 @@ #ifndef OPAL_STDINT_H #define OPAL_STDINT_H 1 +#include "opal_config.h" + /* * Include what we can and define what is missing. */ diff --git a/opal/mca/btl/vader/btl_vader_component.c b/opal/mca/btl/vader/btl_vader_component.c index 0e0c450d877..3549d8401f4 100644 --- a/opal/mca/btl/vader/btl_vader_component.c +++ b/opal/mca/btl/vader/btl_vader_component.c @@ -365,6 +365,7 @@ static int mca_btl_base_vader_modex_send (void) return rc; } +#if OPAL_BTL_VADER_HAVE_XPMEM || OPAL_BTL_VADER_HAVE_CMA || OPAL_BTL_VADER_HAVE_KNEM static void mca_btl_vader_select_next_single_copy_mechanism (void) { for (int i = 0 ; single_copy_mechanisms[i].value != MCA_BTL_VADER_NONE ; ++i) { @@ -374,10 +375,13 @@ static void mca_btl_vader_select_next_single_copy_mechanism (void) } } } +#endif static void mca_btl_vader_check_single_copy (void) { +#if OPAL_BTL_VADER_HAVE_XPMEM || OPAL_BTL_VADER_HAVE_CMA || OPAL_BTL_VADER_HAVE_KNEM int initial_mechanism = mca_btl_vader_component.single_copy_mechanism; +#endif #if OPAL_BTL_VADER_HAVE_XPMEM if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) { @@ -520,7 +524,6 @@ static mca_btl_base_module_t **mca_btl_vader_component_init (int *num_btls, mca_btl_vader_check_single_copy (); if (MCA_BTL_VADER_XPMEM != mca_btl_vader_component.single_copy_mechanism) { - const char *base_dir = opal_process_info.proc_session_dir; char *sm_file; rc = asprintf(&sm_file, "%s" OPAL_PATH_SEP "vader_segment.%s.%x.%d", mca_btl_vader_component.backing_directory, diff --git a/opal/threads/mutex_unix.h b/opal/threads/mutex_unix.h index a61623826ca..1cafdedd4e1 100644 --- a/opal/threads/mutex_unix.h +++ b/opal/threads/mutex_unix.h @@ -76,14 +76,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #else #define OPAL_MUTEX_STATIC_INIT \ { \ .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #endif @@ -97,14 +97,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #else #define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ { \ .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #endif diff --git a/orte/runtime/orte_locks.c b/orte/runtime/orte_locks.c index 9cf31a4889a..386bc472b8f 100644 --- a/orte/runtime/orte_locks.c +++ b/orte/runtime/orte_locks.c @@ -25,12 +25,12 @@ #include "orte/runtime/orte_locks.h" /* for everyone */ -opal_atomic_lock_t orte_finalize_lock = {{0}}; +opal_atomic_lock_t orte_finalize_lock = OPAL_ATOMIC_LOCK_INIT; /* for HNPs */ -opal_atomic_lock_t orte_abort_inprogress_lock = {{0}}; -opal_atomic_lock_t orte_jobs_complete_lock = {{0}}; -opal_atomic_lock_t orte_quit_lock = {{0}}; +opal_atomic_lock_t orte_abort_inprogress_lock = OPAL_ATOMIC_LOCK_INIT; +opal_atomic_lock_t orte_jobs_complete_lock = OPAL_ATOMIC_LOCK_INIT; +opal_atomic_lock_t orte_quit_lock = OPAL_ATOMIC_LOCK_INIT; int orte_locks_init(void) {