diff options
| -rw-r--r-- | erts/aclocal.m4 | 70 | ||||
| -rw-r--r-- | erts/include/internal/ethread_header_config.h.in | 20 | ||||
| -rw-r--r-- | erts/include/internal/gcc/ethr_membar.h | 46 | ||||
| -rw-r--r-- | erts/include/internal/gcc/ethread.h | 30 |
4 files changed, 151 insertions, 15 deletions
diff --git a/erts/aclocal.m4 b/erts/aclocal.m4 index d237f7ae08..cf22456518 100644 --- a/erts/aclocal.m4 +++ b/erts/aclocal.m4 @@ -1374,28 +1374,55 @@ AC_DEFUN(ETHR_CHK_GCC_ATOMIC_OPS, ETHR_CHK_GCC_ATOMIC_OP__(__atomic_compare_exchange_n) ethr_have_gcc_native_atomics=no - ethr_arm_dbm_instr_val=0 + ethr_arm_dbm_sy_instr_val=0 + ethr_arm_dbm_st_instr_val=0 + ethr_arm_dbm_ld_instr_val=0 case "$GCC-$host_cpu" in yes-arm*) - AC_CACHE_CHECK([for ARM DMB instruction], ethr_cv_arm_dbm_instr, + AC_CACHE_CHECK([for ARM 'dmb sy' instruction], ethr_cv_arm_dbm_sy_instr, [ - ethr_cv_arm_dbm_instr=no + ethr_cv_arm_dbm_sy_instr=no AC_TRY_LINK([], [ __asm__ __volatile__("dmb sy" : : : "memory"); - __asm__ __volatile__("dmb st" : : : "memory"); ], - [ethr_cv_arm_dbm_instr=yes]) + [ethr_cv_arm_dbm_sy_instr=yes]) ]) - if test $ethr_cv_arm_dbm_instr = yes; then + if test $ethr_cv_arm_dbm_sy_instr = yes; then ethr_arm_dbm_instr_val=1 test $ethr_cv_64bit___atomic_compare_exchange_n = yes && ethr_have_gcc_native_atomics=yes + fi + AC_CACHE_CHECK([for ARM 'dmb st' instruction], ethr_cv_arm_dbm_st_instr, + [ + ethr_cv_arm_dbm_st_instr=no + AC_TRY_LINK([], + [ + __asm__ __volatile__("dmb st" : : : "memory"); + ], + [ethr_cv_arm_dbm_st_instr=yes]) + ]) + if test $ethr_cv_arm_dbm_sy_instr = yes; then + ethr_arm_dbm_st_instr_val=1 + fi + AC_CACHE_CHECK([for ARM 'dmb ld' instruction], ethr_cv_arm_dbm_ld_instr, + [ + ethr_cv_arm_dbm_ld_instr=no + AC_TRY_LINK([], + [ + __asm__ __volatile__("dmb ld" : : : "memory"); + ], + [ethr_cv_arm_dbm_ld_instr=yes]) + ]) + if test $ethr_cv_arm_dbm_ld_instr = yes; then + ethr_arm_dbm_ld_instr_val=1 fi;; *) ;; esac - AC_DEFINE_UNQUOTED([ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION], [$ethr_arm_dbm_instr_val], [Define as a boolean indicating whether you have a gcc compatible compiler capable of generating the ARM DMB instruction, and are compiling for an ARM processor with ARM DMB instruction support, or not]) + AC_DEFINE_UNQUOTED([ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION], [$ethr_arm_dbm_instr_val], [Define as a boolean indicating whether you have a gcc compatible compiler capable of generating the ARM 'dmb sy' instruction, and are compiling for an ARM processor with ARM DMB instruction support, or not]) + AC_DEFINE_UNQUOTED([ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION], [$ethr_arm_dbm_st_instr_val], [Define as a boolean indicating whether you have a gcc compatible compiler capable of generating the ARM 'dmb st' instruction, and are compiling for an ARM processor with ARM DMB instruction support, or not]) + AC_DEFINE_UNQUOTED([ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION], [$ethr_arm_dbm_ld_instr_val], [Define as a boolean indicating whether you have a gcc compatible compiler capable of generating the ARM 'dmb ld' instruction, and are compiling for an ARM processor with ARM DMB instruction support, or not]) test $ethr_cv_32bit___sync_val_compare_and_swap = yes && ethr_have_gcc_native_atomics=yes test $ethr_cv_64bit___sync_val_compare_and_swap = yes && @@ -1512,6 +1539,33 @@ AC_ARG_WITH(with_sparc_memory_order, AS_HELP_STRING([--with-sparc-memory-order=TSO|PSO|RMO], [specify sparc memory order (defaults to RMO)])) +AC_ARG_ENABLE(ppc-lwsync-instruction, +AS_HELP_STRING([--enable-ppc-lwsync-instruction], [enable use of powerpc lwsync instruction]) +AS_HELP_STRING([--disable-ppc-lwsync-instruction], [disable use of powerpc lwsync instruction]), +[ case "$enableval" in + no) enable_lwsync=no ;; + *) enable_lwsync=yes ;; + esac ], +[ + AC_CHECK_SIZEOF(void *) + case $host_cpu-$ac_cv_sizeof_void_p in + macppc-8|powerpc-8|ppc-8|powerpc64-8|ppc64-8|powerpc64le-8|ppc64le-8|"Power Macintosh"-8) + enable_lwsync=yes;; + *) + enable_lwsync=undefined;; + esac ]) + +case $enable_lwsync in + no) + AC_DEFINE(ETHR_PPC_HAVE_NO_LWSYNC, [1], [Define if you do not have the powerpc lwsync instruction]) + ;; + yes) + AC_DEFINE(ETHR_PPC_HAVE_LWSYNC, [1], [Define if you have the powerpc lwsync instruction]) + ;; + *) + ;; +esac + LM_CHECK_THR_LIB ERL_INTERNAL_LIBS @@ -2823,6 +2877,8 @@ AC_DEFUN([LM_HARDWARE_ARCH], [ ppc) ARCH=ppc;; ppc64) ARCH=ppc64;; ppc64le) ARCH=ppc64le;; + powerpc64) ARCH=ppc64;; + powerpc64le) ARCH=ppc64le;; "Power Macintosh") ARCH=ppc;; arm64) ARCH=arm64;; armv5b) ARCH=arm;; diff --git a/erts/include/internal/ethread_header_config.h.in b/erts/include/internal/ethread_header_config.h.in index 6309f10439..300068b952 100644 --- a/erts/include/internal/ethread_header_config.h.in +++ b/erts/include/internal/ethread_header_config.h.in @@ -76,6 +76,12 @@ /* Define if x86/x86_64 out of order instructions should be synchronized */ #undef ETHR_X86_OUT_OF_ORDER +/* Define if you have the powerpc lwsync instruction */ +#undef ETHR_PPC_HAVE_LWSYNC + +/* Define if you do not have the powerpc lwsync instruction */ +#undef ETHR_PPC_HAVE_NO_LWSYNC + /* Define if only run in Sparc TSO mode */ #undef ETHR_SPARC_TSO @@ -86,10 +92,20 @@ #undef ETHR_SPARC_RMO /* Define as a boolean indicating whether you have a gcc compatible compiler - capable of generating the ARM DMB instruction, and are compiling for an ARM - processor with ARM DMB instruction support, or not */ + capable of generating the ARM 'dmb sy' instruction, and are compiling for + an ARM processor with ARM DMB instruction support, or not */ #undef ETHR_HAVE_GCC_ASM_ARM_DMB_INSTRUCTION +/* Define as a boolean indicating whether you have a gcc compatible compiler + capable of generating the ARM 'dmb ld' instruction, and are compiling for + an ARM processor with ARM DMB instruction support, or not */ +#undef ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION + +/* Define as a boolean indicating whether you have a gcc compatible compiler + capable of generating the ARM 'dmb st' instruction, and are compiling for + an ARM processor with ARM DMB instruction support, or not */ +#undef ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION + /* Define as a bitmask corresponding to the word sizes that __sync_synchronize() can handle on your system */ #undef ETHR_HAVE___sync_synchronize diff --git a/erts/include/internal/gcc/ethr_membar.h b/erts/include/internal/gcc/ethr_membar.h index 643b243683..4e1eb1117e 100644 --- a/erts/include/internal/gcc/ethr_membar.h +++ b/erts/include/internal/gcc/ethr_membar.h @@ -149,14 +149,51 @@ ethr_full_fence__(void) __asm__ __volatile__("dmb sy" : : : "memory"); } +#if ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION static __inline__ __attribute__((__always_inline__)) void ethr_store_fence__(void) { + /* StoreStore */ __asm__ __volatile__("dmb st" : : : "memory"); } +#endif + +#if ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION +static __inline__ __attribute__((__always_inline__)) void +ethr_load_fence__(void) +{ + /* LoadLoad and LoadStore */ + __asm__ __volatile__("dmb ld" : : : "memory"); +} +#endif -#define ETHR_MEMBAR(B) \ - ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, ethr_store_fence__(), ethr_full_fence__()) +#if ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION && ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION +/* sy, st & ld */ +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_store_fence__(), \ + ETHR_CHOOSE_EXPR((B) & (ETHR_StoreStore \ + | ETHR_StoreLoad), \ + ethr_full_fence__(), \ + ethr_load_fence__())) +#elif ETHR_HAVE_GCC_ASM_ARM_DMB_ST_INSTRUCTION +/* sy & st */ +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) == ETHR_StoreStore, \ + ethr_store_fence__(), \ + ethr_full_fence__()) +#elif ETHR_HAVE_GCC_ASM_ARM_DMB_LD_INSTRUCTION +/* sy & ld */ +#define ETHR_MEMBAR(B) \ + ETHR_CHOOSE_EXPR((B) & (ETHR_StoreStore \ + | ETHR_StoreLoad), \ + ethr_full_fence__(), \ + ethr_load_fence__()) +#else +/* sy */ +#define ETHR_MEMBAR(B) \ + ethr_full_fence__() +#endif #elif ETHR_HAVE___sync_synchronize @@ -205,9 +242,12 @@ ethr_full_fence__(void) /* * Define ETHR_READ_DEPEND_MEMORY_BARRIER for all architechtures * not known to order data dependent loads + * + * This is a bit too conservative, but better safe than sorry... + * Add more archs as needed... */ -#if !defined(__ia64__) && !defined(__arm__) +#if !defined(__ia64__) && !defined(__arm__) && !defined(__arm64__) # define ETHR_READ_DEPEND_MEMORY_BARRIER ETHR_MEMBAR(ETHR_LoadLoad) #endif diff --git a/erts/include/internal/gcc/ethread.h b/erts/include/internal/gcc/ethread.h index 12b41f8704..300a8c6922 100644 --- a/erts/include/internal/gcc/ethread.h +++ b/erts/include/internal/gcc/ethread.h @@ -44,6 +44,10 @@ #undef ETHR_GCC_RELB_VERSIONS__ #undef ETHR_GCC_RELB_MOD_VERSIONS__ #undef ETHR_GCC_MB_MOD_VERSIONS__ +#undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ + +#define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ \ + ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS /* * True GNU GCCs before version 4.8 do not emit a memory barrier @@ -52,18 +56,38 @@ */ #undef ETHR___atomic_load_ACQUIRE_barrier_bug #if ETHR_GCC_COMPILER != ETHR_GCC_COMPILER_TRUE + +#if ETHR_GCC_COMPILER == ETHR_GCC_COMPILER_CLANG \ + && defined(__apple_build_version__) \ + && __clang_major__ >= 12 +/* Apples clang verified not to have this bug */ +# define ETHR___atomic_load_ACQUIRE_barrier_bug 0 +/* Also trust builtin barriers */ +# undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ +# define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 1 +# else /* - * A gcc compatible compiler. We have no information + * Another gcc compatible compiler. We have no information * about the existence of this bug, but we assume * that it is not impossible that it could have * been "inherited". Therefore, until we are certain * that the bug does not exist, we assume that it * does. */ -# define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__ +# define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__ +# endif + #elif !ETHR_AT_LEAST_GCC_VSN__(4, 8, 0) /* True gcc of version < 4.8, i.e., bug exist... */ # define ETHR___atomic_load_ACQUIRE_barrier_bug ETHR_GCC_VERSIONS_MASK__ +#elif ETHR_AT_LEAST_GCC_VSN__(9, 3, 0) \ + && (defined(__powerpc__) || defined(__ppc__) || defined(__powerpc64__)) \ + && ETHR_SIZEOF_PTR == 8 +/* Verified not to have this bug */ +# define ETHR___atomic_load_ACQUIRE_barrier_bug 0 +/* Also trust builtin barriers */ +# undef ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ +# define ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ 1 #else /* True gcc of version >= 4.8 */ /* * Sizes less than or equal to word size have been fixed, @@ -87,7 +111,7 @@ #define ETHR_GCC_RELAXED_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ #define ETHR_GCC_RELAXED_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ -#if ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS +#if ETHR_TRUST_GCC_ATOMIC_BUILTINS_MEMORY_BARRIERS__ # define ETHR_GCC_ACQB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ # define ETHR_GCC_ACQB_MOD_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ # define ETHR_GCC_RELB_VERSIONS__ ETHR_GCC_VERSIONS_MASK__ |