Implement CAS_acquire/release/full using __atomic_compare_exchange_n (gcc)

* src/atomic_ops/sysdeps/gcc/generic-small.h: Regenerate.
* src/atomic_ops/sysdeps/gcc/generic-small.template
(AO_XSIZE_fetch_compare_and_swap, AO_XSIZE_compare_and_swap): Use
__atomic_compare_exchange_n(RELAXED) instead of
__sync_val/bool_compare_and_swap.
* src/atomic_ops/sysdeps/gcc/generic.h (AO_double_compare_and_swap):
Likewise.
* src/atomic_ops/sysdeps/gcc/generic-small.template
[AO_GCC_HAVE_XSIZE_SYNC_CAS] (AO_XSIZE_fetch_compare_and_swap_acquire,
AO_XSIZE_fetch_compare_and_swap_release,
AO_XSIZE_fetch_compare_and_swap_full): New inline function; remove
TODO item.
* src/atomic_ops/sysdeps/gcc/generic-small.template
[AO_GCC_HAVE_XSIZE_SYNC_CAS && !AO_GENERALIZE_ASM_BOOL_CAS]
(AO_XSIZE_compare_and_swap_acquire, AO_XSIZE_compare_and_swap_release,
AO_XSIZE_compare_and_swap_full): Likewise.
* src/atomic_ops/sysdeps/gcc/generic.h [AO_GCC_HAVE_double_SYNC_CAS]
(AO_double_compare_and_swap_acquire,
AO_double_compare_and_swap_release, AO_double_compare_and_swap_full):
Likewise.
* src/atomic_ops/sysdeps/gcc/generic.h (AO_compare_and_swap_acquire):
Reformat code.

3 files changed, 404 insertions, 31 deletions

diff --git a/src/atomic_ops/sysdeps/gcc/generic-small.h b/src/atomic_ops/sysdeps/gcc/generic-small.h
index f358f97..ad34568 100644
--- a/src/atomic_ops/sysdeps/gcc/generic-small.h
+++ b/src/atomic_ops/sysdeps/gcc/generic-small.h
@@ -48,27 +48,94 @@ AO_char_store_release(volatile unsigned/**/char *addr, unsigned/**/char value)
 #define AO_HAVE_char_store_release
 
 #ifdef AO_GCC_HAVE_char_SYNC_CAS
+
   AO_INLINE unsigned/**/char
   AO_char_fetch_compare_and_swap(volatile unsigned/**/char *addr,
                                   unsigned/**/char old_val, unsigned/**/char new_val)
   {
-    return __sync_val_compare_and_swap(addr, old_val, new_val
-                                       /* empty protection list */);
+    (void)__atomic_compare_exchange_n(addr,
+                                      &old_val /* p_expected */,
+                                      new_val /* desired */,
+                                      0 /* is_weak: false */,
+                                      __ATOMIC_RELAXED /* success */,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
   }
 # define AO_HAVE_char_fetch_compare_and_swap
 
-  /* TODO: Add CAS _acquire/release/full primitives.    */
+  AO_INLINE unsigned/**/char
+  AO_char_fetch_compare_and_swap_acquire(volatile unsigned/**/char *addr,
+                                          unsigned/**/char old_val, unsigned/**/char new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    return old_val;
+  }
+# define AO_HAVE_char_fetch_compare_and_swap_acquire
+
+  AO_INLINE unsigned/**/char
+  AO_char_fetch_compare_and_swap_release(volatile unsigned/**/char *addr,
+                                          unsigned/**/char old_val, unsigned/**/char new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_RELEASE,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_char_fetch_compare_and_swap_release
+
+  AO_INLINE unsigned/**/char
+  AO_char_fetch_compare_and_swap_full(volatile unsigned/**/char *addr,
+                                       unsigned/**/char old_val, unsigned/**/char new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQ_REL,
+                                      __ATOMIC_ACQUIRE /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_char_fetch_compare_and_swap_full
 
 # ifndef AO_GENERALIZE_ASM_BOOL_CAS
     AO_INLINE int
     AO_char_compare_and_swap(volatile unsigned/**/char *addr,
                               unsigned/**/char old_val, unsigned/**/char new_val)
     {
-      return __sync_bool_compare_and_swap(addr, old_val, new_val
-                                          /* empty protection list */);
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_RELAXED, __ATOMIC_RELAXED);
     }
 #   define AO_HAVE_char_compare_and_swap
+
+    AO_INLINE int
+    AO_char_compare_and_swap_acquire(volatile unsigned/**/char *addr,
+                                      unsigned/**/char old_val, unsigned/**/char new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    }
+#   define AO_HAVE_char_compare_and_swap_acquire
+
+    AO_INLINE int
+    AO_char_compare_and_swap_release(volatile unsigned/**/char *addr,
+                                      unsigned/**/char old_val, unsigned/**/char new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_RELEASE,
+                                              __ATOMIC_RELAXED /* failure */);
+    }
+#   define AO_HAVE_char_compare_and_swap_release
+
+    AO_INLINE int
+    AO_char_compare_and_swap_full(volatile unsigned/**/char *addr,
+                                   unsigned/**/char old_val, unsigned/**/char new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_ACQ_REL,
+                                              __ATOMIC_ACQUIRE /* failure */);
+    }
+#   define AO_HAVE_char_compare_and_swap_full
+
 # endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
+
 #endif /* AO_GCC_HAVE_char_SYNC_CAS */
 /*
  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
@@ -120,27 +187,94 @@ AO_short_store_release(volatile unsigned/**/short *addr, unsigned/**/short value
 #define AO_HAVE_short_store_release
 
 #ifdef AO_GCC_HAVE_short_SYNC_CAS
+
   AO_INLINE unsigned/**/short
   AO_short_fetch_compare_and_swap(volatile unsigned/**/short *addr,
                                   unsigned/**/short old_val, unsigned/**/short new_val)
   {
-    return __sync_val_compare_and_swap(addr, old_val, new_val
-                                       /* empty protection list */);
+    (void)__atomic_compare_exchange_n(addr,
+                                      &old_val /* p_expected */,
+                                      new_val /* desired */,
+                                      0 /* is_weak: false */,
+                                      __ATOMIC_RELAXED /* success */,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
   }
 # define AO_HAVE_short_fetch_compare_and_swap
 
-  /* TODO: Add CAS _acquire/release/full primitives.    */
+  AO_INLINE unsigned/**/short
+  AO_short_fetch_compare_and_swap_acquire(volatile unsigned/**/short *addr,
+                                          unsigned/**/short old_val, unsigned/**/short new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    return old_val;
+  }
+# define AO_HAVE_short_fetch_compare_and_swap_acquire
+
+  AO_INLINE unsigned/**/short
+  AO_short_fetch_compare_and_swap_release(volatile unsigned/**/short *addr,
+                                          unsigned/**/short old_val, unsigned/**/short new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_RELEASE,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_short_fetch_compare_and_swap_release
+
+  AO_INLINE unsigned/**/short
+  AO_short_fetch_compare_and_swap_full(volatile unsigned/**/short *addr,
+                                       unsigned/**/short old_val, unsigned/**/short new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQ_REL,
+                                      __ATOMIC_ACQUIRE /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_short_fetch_compare_and_swap_full
 
 # ifndef AO_GENERALIZE_ASM_BOOL_CAS
     AO_INLINE int
     AO_short_compare_and_swap(volatile unsigned/**/short *addr,
                               unsigned/**/short old_val, unsigned/**/short new_val)
     {
-      return __sync_bool_compare_and_swap(addr, old_val, new_val
-                                          /* empty protection list */);
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_RELAXED, __ATOMIC_RELAXED);
     }
 #   define AO_HAVE_short_compare_and_swap
+
+    AO_INLINE int
+    AO_short_compare_and_swap_acquire(volatile unsigned/**/short *addr,
+                                      unsigned/**/short old_val, unsigned/**/short new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    }
+#   define AO_HAVE_short_compare_and_swap_acquire
+
+    AO_INLINE int
+    AO_short_compare_and_swap_release(volatile unsigned/**/short *addr,
+                                      unsigned/**/short old_val, unsigned/**/short new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_RELEASE,
+                                              __ATOMIC_RELAXED /* failure */);
+    }
+#   define AO_HAVE_short_compare_and_swap_release
+
+    AO_INLINE int
+    AO_short_compare_and_swap_full(volatile unsigned/**/short *addr,
+                                   unsigned/**/short old_val, unsigned/**/short new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_ACQ_REL,
+                                              __ATOMIC_ACQUIRE /* failure */);
+    }
+#   define AO_HAVE_short_compare_and_swap_full
+
 # endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
+
 #endif /* AO_GCC_HAVE_short_SYNC_CAS */
 /*
  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
@@ -192,27 +326,94 @@ AO_int_store_release(volatile unsigned *addr, unsigned value)
 #define AO_HAVE_int_store_release
 
 #ifdef AO_GCC_HAVE_int_SYNC_CAS
+
   AO_INLINE unsigned
   AO_int_fetch_compare_and_swap(volatile unsigned *addr,
                                   unsigned old_val, unsigned new_val)
   {
-    return __sync_val_compare_and_swap(addr, old_val, new_val
-                                       /* empty protection list */);
+    (void)__atomic_compare_exchange_n(addr,
+                                      &old_val /* p_expected */,
+                                      new_val /* desired */,
+                                      0 /* is_weak: false */,
+                                      __ATOMIC_RELAXED /* success */,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
   }
 # define AO_HAVE_int_fetch_compare_and_swap
 
-  /* TODO: Add CAS _acquire/release/full primitives.    */
+  AO_INLINE unsigned
+  AO_int_fetch_compare_and_swap_acquire(volatile unsigned *addr,
+                                          unsigned old_val, unsigned new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    return old_val;
+  }
+# define AO_HAVE_int_fetch_compare_and_swap_acquire
+
+  AO_INLINE unsigned
+  AO_int_fetch_compare_and_swap_release(volatile unsigned *addr,
+                                          unsigned old_val, unsigned new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_RELEASE,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_int_fetch_compare_and_swap_release
+
+  AO_INLINE unsigned
+  AO_int_fetch_compare_and_swap_full(volatile unsigned *addr,
+                                       unsigned old_val, unsigned new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQ_REL,
+                                      __ATOMIC_ACQUIRE /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_int_fetch_compare_and_swap_full
 
 # ifndef AO_GENERALIZE_ASM_BOOL_CAS
     AO_INLINE int
     AO_int_compare_and_swap(volatile unsigned *addr,
                               unsigned old_val, unsigned new_val)
     {
-      return __sync_bool_compare_and_swap(addr, old_val, new_val
-                                          /* empty protection list */);
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_RELAXED, __ATOMIC_RELAXED);
     }
 #   define AO_HAVE_int_compare_and_swap
+
+    AO_INLINE int
+    AO_int_compare_and_swap_acquire(volatile unsigned *addr,
+                                      unsigned old_val, unsigned new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    }
+#   define AO_HAVE_int_compare_and_swap_acquire
+
+    AO_INLINE int
+    AO_int_compare_and_swap_release(volatile unsigned *addr,
+                                      unsigned old_val, unsigned new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_RELEASE,
+                                              __ATOMIC_RELAXED /* failure */);
+    }
+#   define AO_HAVE_int_compare_and_swap_release
+
+    AO_INLINE int
+    AO_int_compare_and_swap_full(volatile unsigned *addr,
+                                   unsigned old_val, unsigned new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_ACQ_REL,
+                                              __ATOMIC_ACQUIRE /* failure */);
+    }
+#   define AO_HAVE_int_compare_and_swap_full
+
 # endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
+
 #endif /* AO_GCC_HAVE_int_SYNC_CAS */
 /*
  * Copyright (c) 1991-1994 by Xerox Corporation.  All rights reserved.
@@ -264,25 +465,92 @@ AO_store_release(volatile AO_t *addr, AO_t value)
 #define AO_HAVE_store_release
 
 #ifdef AO_GCC_HAVE_SYNC_CAS
+
   AO_INLINE AO_t
   AO_fetch_compare_and_swap(volatile AO_t *addr,
                                   AO_t old_val, AO_t new_val)
   {
-    return __sync_val_compare_and_swap(addr, old_val, new_val
-                                       /* empty protection list */);
+    (void)__atomic_compare_exchange_n(addr,
+                                      &old_val /* p_expected */,
+                                      new_val /* desired */,
+                                      0 /* is_weak: false */,
+                                      __ATOMIC_RELAXED /* success */,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
   }
 # define AO_HAVE_fetch_compare_and_swap
 
-  /* TODO: Add CAS _acquire/release/full primitives.    */
+  AO_INLINE AO_t
+  AO_fetch_compare_and_swap_acquire(volatile AO_t *addr,
+                                          AO_t old_val, AO_t new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    return old_val;
+  }
+# define AO_HAVE_fetch_compare_and_swap_acquire
+
+  AO_INLINE AO_t
+  AO_fetch_compare_and_swap_release(volatile AO_t *addr,
+                                          AO_t old_val, AO_t new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_RELEASE,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_fetch_compare_and_swap_release
+
+  AO_INLINE AO_t
+  AO_fetch_compare_and_swap_full(volatile AO_t *addr,
+                                       AO_t old_val, AO_t new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQ_REL,
+                                      __ATOMIC_ACQUIRE /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_fetch_compare_and_swap_full
 
 # ifndef AO_GENERALIZE_ASM_BOOL_CAS
     AO_INLINE int
     AO_compare_and_swap(volatile AO_t *addr,
                               AO_t old_val, AO_t new_val)
     {
-      return __sync_bool_compare_and_swap(addr, old_val, new_val
-                                          /* empty protection list */);
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_RELAXED, __ATOMIC_RELAXED);
     }
 #   define AO_HAVE_compare_and_swap
+
+    AO_INLINE int
+    AO_compare_and_swap_acquire(volatile AO_t *addr,
+                                      AO_t old_val, AO_t new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    }
+#   define AO_HAVE_compare_and_swap_acquire
+
+    AO_INLINE int
+    AO_compare_and_swap_release(volatile AO_t *addr,
+                                      AO_t old_val, AO_t new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_RELEASE,
+                                              __ATOMIC_RELAXED /* failure */);
+    }
+#   define AO_HAVE_compare_and_swap_release
+
+    AO_INLINE int
+    AO_compare_and_swap_full(volatile AO_t *addr,
+                                   AO_t old_val, AO_t new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_ACQ_REL,
+                                              __ATOMIC_ACQUIRE /* failure */);
+    }
+#   define AO_HAVE_compare_and_swap_full
+
 # endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
+
 #endif /* AO_GCC_HAVE_SYNC_CAS */
diff --git a/src/atomic_ops/sysdeps/gcc/generic-small.template b/src/atomic_ops/sysdeps/gcc/generic-small.template
index e016488..dd17d25 100644
--- a/src/atomic_ops/sysdeps/gcc/generic-small.template
+++ b/src/atomic_ops/sysdeps/gcc/generic-small.template
@@ -48,25 +48,92 @@ AO_XSIZE_store_release(volatile XCTYPE *addr, XCTYPE value)
 #define AO_HAVE_XSIZE_store_release
 
 #ifdef AO_GCC_HAVE_XSIZE_SYNC_CAS
+
   AO_INLINE XCTYPE
   AO_XSIZE_fetch_compare_and_swap(volatile XCTYPE *addr,
                                   XCTYPE old_val, XCTYPE new_val)
   {
-    return __sync_val_compare_and_swap(addr, old_val, new_val
-                                       /* empty protection list */);
+    (void)__atomic_compare_exchange_n(addr,
+                                      &old_val /* p_expected */,
+                                      new_val /* desired */,
+                                      0 /* is_weak: false */,
+                                      __ATOMIC_RELAXED /* success */,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
   }
 # define AO_HAVE_XSIZE_fetch_compare_and_swap
 
-  /* TODO: Add CAS _acquire/release/full primitives.    */
+  AO_INLINE XCTYPE
+  AO_XSIZE_fetch_compare_and_swap_acquire(volatile XCTYPE *addr,
+                                          XCTYPE old_val, XCTYPE new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    return old_val;
+  }
+# define AO_HAVE_XSIZE_fetch_compare_and_swap_acquire
+
+  AO_INLINE XCTYPE
+  AO_XSIZE_fetch_compare_and_swap_release(volatile XCTYPE *addr,
+                                          XCTYPE old_val, XCTYPE new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_RELEASE,
+                                      __ATOMIC_RELAXED /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_XSIZE_fetch_compare_and_swap_release
+
+  AO_INLINE XCTYPE
+  AO_XSIZE_fetch_compare_and_swap_full(volatile XCTYPE *addr,
+                                       XCTYPE old_val, XCTYPE new_val)
+  {
+    (void)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                      __ATOMIC_ACQ_REL,
+                                      __ATOMIC_ACQUIRE /* failure */);
+    return old_val;
+  }
+# define AO_HAVE_XSIZE_fetch_compare_and_swap_full
 
 # ifndef AO_GENERALIZE_ASM_BOOL_CAS
     AO_INLINE int
     AO_XSIZE_compare_and_swap(volatile XCTYPE *addr,
                               XCTYPE old_val, XCTYPE new_val)
     {
-      return __sync_bool_compare_and_swap(addr, old_val, new_val
-                                          /* empty protection list */);
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_RELAXED, __ATOMIC_RELAXED);
     }
 #   define AO_HAVE_XSIZE_compare_and_swap
+
+    AO_INLINE int
+    AO_XSIZE_compare_and_swap_acquire(volatile XCTYPE *addr,
+                                      XCTYPE old_val, XCTYPE new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                        __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    }
+#   define AO_HAVE_XSIZE_compare_and_swap_acquire
+
+    AO_INLINE int
+    AO_XSIZE_compare_and_swap_release(volatile XCTYPE *addr,
+                                      XCTYPE old_val, XCTYPE new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_RELEASE,
+                                              __ATOMIC_RELAXED /* failure */);
+    }
+#   define AO_HAVE_XSIZE_compare_and_swap_release
+
+    AO_INLINE int
+    AO_XSIZE_compare_and_swap_full(volatile XCTYPE *addr,
+                                   XCTYPE old_val, XCTYPE new_val)
+    {
+      return (int)__atomic_compare_exchange_n(addr, &old_val, new_val, 0,
+                                              __ATOMIC_ACQ_REL,
+                                              __ATOMIC_ACQUIRE /* failure */);
+    }
+#   define AO_HAVE_XSIZE_compare_and_swap_full
+
 # endif /* !AO_GENERALIZE_ASM_BOOL_CAS */
+
 #endif /* AO_GCC_HAVE_XSIZE_SYNC_CAS */
diff --git a/src/atomic_ops/sysdeps/gcc/generic.h b/src/atomic_ops/sysdeps/gcc/generic.h
index 7bb4625..ffe993d 100644
--- a/src/atomic_ops/sysdeps/gcc/generic.h
+++ b/src/atomic_ops/sysdeps/gcc/generic.h
@@ -172,14 +172,52 @@
                                AO_double_t old_val, AO_double_t new_val)
     {
       return (int)__atomic_compare_exchange_n(&addr->AO_whole,
-                                  &old_val.AO_whole /* p_expected */,
-                                  new_val.AO_whole /* desired */,
-                                  0 /* is_weak: false */,
-                                  __ATOMIC_RELAXED /* success */,
-                                  __ATOMIC_RELAXED /* failure */);
+                                &old_val.AO_whole /* p_expected */,
+                                new_val.AO_whole /* desired */,
+                                0 /* is_weak: false */,
+                                __ATOMIC_RELAXED /* success */,
+                                __ATOMIC_RELAXED /* failure */);
     }
 #   define AO_HAVE_double_compare_and_swap
 # endif
 
-  /* TODO: Add double CAS _acquire/release/full primitives. */
+# ifndef AO_HAVE_double_compare_and_swap_acquire
+    AO_INLINE int
+    AO_double_compare_and_swap_acquire(volatile AO_double_t *addr,
+                                       AO_double_t old_val,
+                                       AO_double_t new_val)
+    {
+      return (int)__atomic_compare_exchange_n(&addr->AO_whole,
+                                &old_val.AO_whole, new_val.AO_whole, 0,
+                                __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE);
+    }
+#   define AO_HAVE_double_compare_and_swap_acquire
+# endif
+
+# ifndef AO_HAVE_double_compare_and_swap_release
+    AO_INLINE int
+    AO_double_compare_and_swap_release(volatile AO_double_t *addr,
+                                       AO_double_t old_val,
+                                       AO_double_t new_val)
+    {
+      return (int)__atomic_compare_exchange_n(&addr->AO_whole,
+                                &old_val.AO_whole, new_val.AO_whole, 0,
+                                __ATOMIC_RELEASE,
+                                __ATOMIC_RELAXED /* failure */);
+    }
+#   define AO_HAVE_double_compare_and_swap_release
+# endif
+
+# ifndef AO_HAVE_double_compare_and_swap_full
+    AO_INLINE int
+    AO_double_compare_and_swap_full(volatile AO_double_t *addr,
+                                    AO_double_t old_val, AO_double_t new_val)
+    {
+      return (int)__atomic_compare_exchange_n(&addr->AO_whole,
+                                &old_val.AO_whole, new_val.AO_whole, 0,
+                                __ATOMIC_ACQ_REL,
+                                __ATOMIC_ACQUIRE /* failure */);
+    }
+#   define AO_HAVE_double_compare_and_swap_full
+# endif
 #endif /* AO_GCC_HAVE_double_SYNC_CAS */