From 83fca6816765592cca864a81060cb0f58d3e401c Mon Sep 17 00:00:00 2001 From: Ivan Maidanski Date: Wed, 6 Apr 2016 11:40:45 +0300 Subject: Avoid atomic_compare_exchange_n if no __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n * src/atomic_ops/sysdeps/gcc/aarch64.h (AO_GCC_FORCE_HAVE_CAS, AO_GCC_HAVE_double_SYNC_CAS): Define macro before include generic.h if __clang__ (workaround). * src/atomic_ops/sysdeps/gcc/generic-small.h: Regenerate. * src/atomic_ops/sysdeps/gcc/generic-small.template (AO_XSIZE_fetch_compare_and_swap, AO_XSIZE_compare_and_swap): Do not define unless AO_GCC_HAVE_XSIZE_SYNC_CAS. * src/atomic_ops/sysdeps/gcc/generic.h (AO_GCC_HAVE_char_SYNC_CAS, AO_GCC_HAVE_short_SYNC_CAS, AO_GCC_HAVE_int_SYNC_CAS, AO_GCC_HAVE_SYNC_CAS): New internal macro (based on __GCC_HAVE_SYNC_COMPARE_AND_SWAP_ or AO_GCC_FORCE_HAVE_CAS presence). * src/atomic_ops/sysdeps/gcc/generic.h (AO_GCC_HAVE_double_SYNC_CAS): New internal macro if AO_HAVE_DOUBLE_PTR_STORAGE. * src/atomic_ops/sysdeps/gcc/generic.h (AO_double_compare_and_swap): Check AO_GCC_HAVE_double_SYNC_CAS instead of AO_HAVE_DOUBLE_PTR_STORAGE. --- src/atomic_ops/sysdeps/gcc/aarch64.h | 6 + src/atomic_ops/sysdeps/gcc/generic-small.h | 160 ++++++++++++---------- src/atomic_ops/sysdeps/gcc/generic-small.template | 40 +++--- src/atomic_ops/sysdeps/gcc/generic.h | 34 ++++- 4 files changed, 144 insertions(+), 96 deletions(-) diff --git a/src/atomic_ops/sysdeps/gcc/aarch64.h b/src/atomic_ops/sysdeps/gcc/aarch64.h index d3508a6..9a6ef5a 100644 --- a/src/atomic_ops/sysdeps/gcc/aarch64.h +++ b/src/atomic_ops/sysdeps/gcc/aarch64.h @@ -207,4 +207,10 @@ #endif /* !AO_PREFER_BUILTIN_ATOMICS */ +#if defined(__clang__) + /* As of clang-3.6/arm64, __GCC_HAVE_SYNC_COMPARE_AND_SWAP_n are missing. */ +# define AO_GCC_FORCE_HAVE_CAS +# define AO_GCC_HAVE_double_SYNC_CAS +#endif + #include "generic.h" diff --git a/src/atomic_ops/sysdeps/gcc/generic-small.h b/src/atomic_ops/sysdeps/gcc/generic-small.h index 72f4a5b..f358f97 100644 --- a/src/atomic_ops/sysdeps/gcc/generic-small.h +++ b/src/atomic_ops/sysdeps/gcc/generic-small.h @@ -47,27 +47,29 @@ AO_char_store_release(volatile unsigned/**/char *addr, unsigned/**/char value) } #define AO_HAVE_char_store_release -AO_INLINE unsigned/**/char -AO_char_fetch_compare_and_swap(volatile unsigned/**/char *addr, - unsigned/**/char old_val, unsigned/**/char new_val) -{ - return __sync_val_compare_and_swap(addr, old_val, new_val - /* empty protection list */); -} -#define AO_HAVE_char_fetch_compare_and_swap - -/* TODO: Add CAS _acquire/release/full primitives. */ - -#ifndef AO_GENERALIZE_ASM_BOOL_CAS - AO_INLINE int - AO_char_compare_and_swap(volatile unsigned/**/char *addr, - unsigned/**/char old_val, unsigned/**/char new_val) +#ifdef AO_GCC_HAVE_char_SYNC_CAS + AO_INLINE unsigned/**/char + AO_char_fetch_compare_and_swap(volatile unsigned/**/char *addr, + unsigned/**/char old_val, unsigned/**/char new_val) { - return __sync_bool_compare_and_swap(addr, old_val, new_val - /* empty protection list */); + return __sync_val_compare_and_swap(addr, old_val, new_val + /* empty protection list */); } -# define AO_HAVE_char_compare_and_swap -#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +# define AO_HAVE_char_fetch_compare_and_swap + + /* TODO: Add CAS _acquire/release/full primitives. */ + +# ifndef AO_GENERALIZE_ASM_BOOL_CAS + AO_INLINE int + AO_char_compare_and_swap(volatile unsigned/**/char *addr, + unsigned/**/char old_val, unsigned/**/char new_val) + { + return __sync_bool_compare_and_swap(addr, old_val, new_val + /* empty protection list */); + } +# define AO_HAVE_char_compare_and_swap +# endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +#endif /* AO_GCC_HAVE_char_SYNC_CAS */ /* * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. @@ -117,27 +119,29 @@ AO_short_store_release(volatile unsigned/**/short *addr, unsigned/**/short value } #define AO_HAVE_short_store_release -AO_INLINE unsigned/**/short -AO_short_fetch_compare_and_swap(volatile unsigned/**/short *addr, - unsigned/**/short old_val, unsigned/**/short new_val) -{ - return __sync_val_compare_and_swap(addr, old_val, new_val - /* empty protection list */); -} -#define AO_HAVE_short_fetch_compare_and_swap - -/* TODO: Add CAS _acquire/release/full primitives. */ - -#ifndef AO_GENERALIZE_ASM_BOOL_CAS - AO_INLINE int - AO_short_compare_and_swap(volatile unsigned/**/short *addr, - unsigned/**/short old_val, unsigned/**/short new_val) +#ifdef AO_GCC_HAVE_short_SYNC_CAS + AO_INLINE unsigned/**/short + AO_short_fetch_compare_and_swap(volatile unsigned/**/short *addr, + unsigned/**/short old_val, unsigned/**/short new_val) { - return __sync_bool_compare_and_swap(addr, old_val, new_val - /* empty protection list */); + return __sync_val_compare_and_swap(addr, old_val, new_val + /* empty protection list */); } -# define AO_HAVE_short_compare_and_swap -#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +# define AO_HAVE_short_fetch_compare_and_swap + + /* TODO: Add CAS _acquire/release/full primitives. */ + +# ifndef AO_GENERALIZE_ASM_BOOL_CAS + AO_INLINE int + AO_short_compare_and_swap(volatile unsigned/**/short *addr, + unsigned/**/short old_val, unsigned/**/short new_val) + { + return __sync_bool_compare_and_swap(addr, old_val, new_val + /* empty protection list */); + } +# define AO_HAVE_short_compare_and_swap +# endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +#endif /* AO_GCC_HAVE_short_SYNC_CAS */ /* * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. @@ -187,27 +191,29 @@ AO_int_store_release(volatile unsigned *addr, unsigned value) } #define AO_HAVE_int_store_release -AO_INLINE unsigned -AO_int_fetch_compare_and_swap(volatile unsigned *addr, - unsigned old_val, unsigned new_val) -{ - return __sync_val_compare_and_swap(addr, old_val, new_val - /* empty protection list */); -} -#define AO_HAVE_int_fetch_compare_and_swap - -/* TODO: Add CAS _acquire/release/full primitives. */ - -#ifndef AO_GENERALIZE_ASM_BOOL_CAS - AO_INLINE int - AO_int_compare_and_swap(volatile unsigned *addr, - unsigned old_val, unsigned new_val) +#ifdef AO_GCC_HAVE_int_SYNC_CAS + AO_INLINE unsigned + AO_int_fetch_compare_and_swap(volatile unsigned *addr, + unsigned old_val, unsigned new_val) { - return __sync_bool_compare_and_swap(addr, old_val, new_val - /* empty protection list */); + return __sync_val_compare_and_swap(addr, old_val, new_val + /* empty protection list */); } -# define AO_HAVE_int_compare_and_swap -#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +# define AO_HAVE_int_fetch_compare_and_swap + + /* TODO: Add CAS _acquire/release/full primitives. */ + +# ifndef AO_GENERALIZE_ASM_BOOL_CAS + AO_INLINE int + AO_int_compare_and_swap(volatile unsigned *addr, + unsigned old_val, unsigned new_val) + { + return __sync_bool_compare_and_swap(addr, old_val, new_val + /* empty protection list */); + } +# define AO_HAVE_int_compare_and_swap +# endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +#endif /* AO_GCC_HAVE_int_SYNC_CAS */ /* * Copyright (c) 1991-1994 by Xerox Corporation. All rights reserved. * Copyright (c) 1996-1999 by Silicon Graphics. All rights reserved. @@ -257,24 +263,26 @@ AO_store_release(volatile AO_t *addr, AO_t value) } #define AO_HAVE_store_release -AO_INLINE AO_t -AO_fetch_compare_and_swap(volatile AO_t *addr, - AO_t old_val, AO_t new_val) -{ - return __sync_val_compare_and_swap(addr, old_val, new_val - /* empty protection list */); -} -#define AO_HAVE_fetch_compare_and_swap - -/* TODO: Add CAS _acquire/release/full primitives. */ - -#ifndef AO_GENERALIZE_ASM_BOOL_CAS - AO_INLINE int - AO_compare_and_swap(volatile AO_t *addr, - AO_t old_val, AO_t new_val) +#ifdef AO_GCC_HAVE_SYNC_CAS + AO_INLINE AO_t + AO_fetch_compare_and_swap(volatile AO_t *addr, + AO_t old_val, AO_t new_val) { - return __sync_bool_compare_and_swap(addr, old_val, new_val - /* empty protection list */); + return __sync_val_compare_and_swap(addr, old_val, new_val + /* empty protection list */); } -# define AO_HAVE_compare_and_swap -#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +# define AO_HAVE_fetch_compare_and_swap + + /* TODO: Add CAS _acquire/release/full primitives. */ + +# ifndef AO_GENERALIZE_ASM_BOOL_CAS + AO_INLINE int + AO_compare_and_swap(volatile AO_t *addr, + AO_t old_val, AO_t new_val) + { + return __sync_bool_compare_and_swap(addr, old_val, new_val + /* empty protection list */); + } +# define AO_HAVE_compare_and_swap +# endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +#endif /* AO_GCC_HAVE_SYNC_CAS */ diff --git a/src/atomic_ops/sysdeps/gcc/generic-small.template b/src/atomic_ops/sysdeps/gcc/generic-small.template index 9685acf..e016488 100644 --- a/src/atomic_ops/sysdeps/gcc/generic-small.template +++ b/src/atomic_ops/sysdeps/gcc/generic-small.template @@ -47,24 +47,26 @@ AO_XSIZE_store_release(volatile XCTYPE *addr, XCTYPE value) } #define AO_HAVE_XSIZE_store_release -AO_INLINE XCTYPE -AO_XSIZE_fetch_compare_and_swap(volatile XCTYPE *addr, - XCTYPE old_val, XCTYPE new_val) -{ - return __sync_val_compare_and_swap(addr, old_val, new_val - /* empty protection list */); -} -#define AO_HAVE_XSIZE_fetch_compare_and_swap - -/* TODO: Add CAS _acquire/release/full primitives. */ - -#ifndef AO_GENERALIZE_ASM_BOOL_CAS - AO_INLINE int - AO_XSIZE_compare_and_swap(volatile XCTYPE *addr, - XCTYPE old_val, XCTYPE new_val) +#ifdef AO_GCC_HAVE_XSIZE_SYNC_CAS + AO_INLINE XCTYPE + AO_XSIZE_fetch_compare_and_swap(volatile XCTYPE *addr, + XCTYPE old_val, XCTYPE new_val) { - return __sync_bool_compare_and_swap(addr, old_val, new_val - /* empty protection list */); + return __sync_val_compare_and_swap(addr, old_val, new_val + /* empty protection list */); } -# define AO_HAVE_XSIZE_compare_and_swap -#endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +# define AO_HAVE_XSIZE_fetch_compare_and_swap + + /* TODO: Add CAS _acquire/release/full primitives. */ + +# ifndef AO_GENERALIZE_ASM_BOOL_CAS + AO_INLINE int + AO_XSIZE_compare_and_swap(volatile XCTYPE *addr, + XCTYPE old_val, XCTYPE new_val) + { + return __sync_bool_compare_and_swap(addr, old_val, new_val + /* empty protection list */); + } +# define AO_HAVE_XSIZE_compare_and_swap +# endif /* !AO_GENERALIZE_ASM_BOOL_CAS */ +#endif /* AO_GCC_HAVE_XSIZE_SYNC_CAS */ diff --git a/src/atomic_ops/sysdeps/gcc/generic.h b/src/atomic_ops/sysdeps/gcc/generic.h index 1df2914..7bb4625 100644 --- a/src/atomic_ops/sysdeps/gcc/generic.h +++ b/src/atomic_ops/sysdeps/gcc/generic.h @@ -19,6 +19,29 @@ /* For the details, see GNU Manual, chapter 6.52 (Built-in functions */ /* for memory model aware atomic operations). */ +#if defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_1) \ + || defined(AO_GCC_FORCE_HAVE_CAS) +# define AO_GCC_HAVE_char_SYNC_CAS +#endif + +#if (__SIZEOF_SHORT__ == 2 && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_2)) \ + || defined(AO_GCC_FORCE_HAVE_CAS) +# define AO_GCC_HAVE_short_SYNC_CAS +#endif + +#if (__SIZEOF_INT__ == 4 && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)) \ + || (__SIZEOF_INT__ == 8 && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8)) \ + || defined(AO_GCC_FORCE_HAVE_CAS) +# define AO_GCC_HAVE_int_SYNC_CAS +#endif + +#if (__SIZEOF_SIZE_T__ == 4 && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_4)) \ + || (__SIZEOF_SIZE_T__ == 8 \ + && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8)) \ + || defined(AO_GCC_FORCE_HAVE_CAS) +# define AO_GCC_HAVE_SYNC_CAS +#endif + #ifdef AO_UNIPROCESSOR /* If only a single processor (core) is used, AO_UNIPROCESSOR could */ /* be defined by the client to avoid unnecessary memory barrier. */ @@ -134,6 +157,15 @@ # define AO_HAVE_double_store_release # endif +# if (__SIZEOF_SIZE_T__ == 4 && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_8)) \ + || (__SIZEOF_SIZE_T__ == 8 /* half of AO_double_t */ \ + && defined(__GCC_HAVE_SYNC_COMPARE_AND_SWAP_16)) +# define AO_GCC_HAVE_double_SYNC_CAS +# endif + +#endif /* AO_HAVE_DOUBLE_PTR_STORAGE */ + +#ifdef AO_GCC_HAVE_double_SYNC_CAS # ifndef AO_HAVE_double_compare_and_swap AO_INLINE int AO_double_compare_and_swap(volatile AO_double_t *addr, @@ -150,4 +182,4 @@ # endif /* TODO: Add double CAS _acquire/release/full primitives. */ -#endif /* AO_HAVE_DOUBLE_PTR_STORAGE */ +#endif /* AO_GCC_HAVE_double_SYNC_CAS */ -- cgit v1.2.1