Support double-wide CAS on UWP/arm64 (MS VC)

Also, do not include msftc/x86_64.h if target is arm64 (i.e., new
internal header is created - msftc/arm64.h).

* src/Makefile.am (nobase_private_HEADERS): Add msftc/aarch64.h.
* src/atomic_ops.h [(_MSC_VER || __DMC__ || __BORLANDC__
|| __WATCOMC__ && __NT__) && _M_ARM64]: Include msftc/aarch64.h
instead of msftc/x86_64.h.
* src/atomic_ops/sysdeps/msftc/arm64.h: New file.
* src/atomic_ops/sysdeps/msftc/x86_64.h: Include ordered_except_wr.h
unconditionally (again).
* src/atomic_ops/sysdeps/standard_ao_double_t.h [_M_ARM64 && _MSC_VER]
(double_ptr_storage): Define; add comment.

5 files changed, 121 insertions, 4 deletions

diff --git a/src/Makefile.am b/src/Makefile.am
index ae5f24f..61a092c 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -108,6 +108,7 @@ nobase_private_HEADERS = atomic_ops/ao_version.h \
           atomic_ops/sysdeps/loadstore/double_atomic_load_store.h \
         \
           atomic_ops/sysdeps/msftc/arm.h \
+          atomic_ops/sysdeps/msftc/arm64.h \
           atomic_ops/sysdeps/msftc/common32_defs.h \
           atomic_ops/sysdeps/msftc/x86.h \
           atomic_ops/sysdeps/msftc/x86_64.h \
diff --git a/src/atomic_ops.h b/src/atomic_ops.h
index 92d1f4a..9e39b75 100644
--- a/src/atomic_ops.h
+++ b/src/atomic_ops.h
@@ -395,8 +395,10 @@
 
 #if defined(_MSC_VER) || defined(__DMC__) || defined(__BORLANDC__) \
         || (defined(__WATCOMC__) && defined(__NT__))
-# if defined(_AMD64_) || defined(_M_X64) || defined(_M_ARM64)
+# if defined(_AMD64_) || defined(_M_X64)
 #   include "atomic_ops/sysdeps/msftc/x86_64.h"
+# elif defined(_M_ARM64)
+#   include "atomic_ops/sysdeps/msftc/arm64.h"
 # elif defined(_M_IX86) || defined(x86)
 #   include "atomic_ops/sysdeps/msftc/x86.h"
 # elif defined(_M_ARM) || defined(ARM) || defined(_ARM_)
diff --git a/src/atomic_ops/sysdeps/msftc/arm64.h b/src/atomic_ops/sysdeps/msftc/arm64.h
new file mode 100644
index 0000000..89db08a
--- /dev/null
+++ b/src/atomic_ops/sysdeps/msftc/arm64.h
@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2003 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2021 Ivan Maidanski
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a copy
+ * of this software and associated documentation files (the "Software"), to deal
+ * in the Software without restriction, including without limitation the rights
+ * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the Software is
+ * furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+ * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#include "../all_aligned_atomic_load_store.h"
+#include "../test_and_set_t_is_ao_t.h"
+
+#ifndef AO_ASSUME_WINDOWS98
+# define AO_ASSUME_WINDOWS98
+#endif
+#ifndef AO_USE_INTERLOCKED_INTRINSICS
+# define AO_USE_INTERLOCKED_INTRINSICS
+#endif
+#include "common32_defs.h"
+
+#indef AO_NO_DOUBLE_CAS
+
+# include "../standard_ao_double_t.h"
+
+# pragma intrinsic (_InterlockedCompareExchange128)
+# pragma intrinsic (_InterlockedCompareExchange128_acq)
+# pragma intrinsic (_InterlockedCompareExchange128_nf)
+# pragma intrinsic (_InterlockedCompareExchange128_rel)
+
+  AO_INLINE int
+  AO_compare_double_and_swap_double(volatile AO_double_t *addr,
+                                    AO_t old_val1, AO_t old_val2,
+                                    AO_t new_val1, AO_t new_val2)
+  {
+    __int64 comparandResult[2];
+
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    comparandResult[0] = old_val1; /* low */
+    comparandResult[1] = old_val2; /* high */
+    return _InterlockedCompareExchange128_nf((volatile __int64 *)addr,
+                                             new_val2 /* high */,
+                                             new_val1 /* low */,
+                                             comparandResult);
+  }
+# define AO_HAVE_compare_double_and_swap_double
+
+  AO_INLINE int
+  AO_compare_double_and_swap_double_acquire(volatile AO_double_t *addr,
+                                            AO_t old_val1, AO_t old_val2,
+                                            AO_t new_val1, AO_t new_val2)
+  {
+    __int64 comparandResult[2];
+
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    comparandResult[0] = old_val1; /* low */
+    comparandResult[1] = old_val2; /* high */
+    return _InterlockedCompareExchange128_acq((volatile __int64 *)addr,
+                                              new_val2 /* high */,
+                                              new_val1 /* low */,
+                                              comparandResult);
+  }
+# define AO_HAVE_compare_double_and_swap_double_acquire
+
+  AO_INLINE int
+  AO_compare_double_and_swap_double_release(volatile AO_double_t *addr,
+                                            AO_t old_val1, AO_t old_val2,
+                                            AO_t new_val1, AO_t new_val2)
+  {
+    __int64 comparandResult[2];
+
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    comparandResult[0] = old_val1; /* low */
+    comparandResult[1] = old_val2; /* high */
+    return _InterlockedCompareExchange128_rel((volatile __int64 *)addr,
+                                              new_val2 /* high */,
+                                              new_val1 /* low */,
+                                              comparandResult);
+  }
+# define AO_HAVE_compare_double_and_swap_double_release
+
+  AO_INLINE int
+  AO_compare_double_and_swap_double_full(volatile AO_double_t *addr,
+                                         AO_t old_val1, AO_t old_val2,
+                                         AO_t new_val1, AO_t new_val2)
+  {
+    __int64 comparandResult[2];
+
+    AO_ASSERT_ADDR_ALIGNED(addr);
+    comparandResult[0] = old_val1; /* low */
+    comparandResult[1] = old_val2; /* high */
+    return _InterlockedCompareExchange128((volatile __int64 *)addr,
+                                          new_val2 /* high */,
+                                          new_val1 /* low */,
+                                          comparandResult);
+  }
+# define AO_HAVE_compare_double_and_swap_double_full
+
+#endif /* !AO_NO_DOUBLE_CAS */
diff --git a/src/atomic_ops/sysdeps/msftc/x86_64.h b/src/atomic_ops/sysdeps/msftc/x86_64.h
index ef13824..d1038b3 100644
--- a/src/atomic_ops/sysdeps/msftc/x86_64.h
+++ b/src/atomic_ops/sysdeps/msftc/x86_64.h
@@ -23,7 +23,6 @@
 
 #include "../all_aligned_atomic_load_store.h"
 
-#if !defined(_M_ARM64)
 /* Real X86 implementations appear                                      */
 /* to enforce ordering between memory operations, EXCEPT that a later   */
 /* read can pass earlier writes, presumably due to the visible          */
@@ -31,7 +30,6 @@
 /* We ignore the fact that the official specs                           */
 /* seem to be much weaker (and arguably too weak to be usable).         */
 #include "../ordered_except_wr.h"
-#endif
 
 #ifdef AO_ASM_X64_AVAILABLE
 # include "../test_and_set_t_is_char.h"
@@ -74,7 +72,7 @@
     }
   }
 # define AO_HAVE_short_fetch_and_add_full
-#endif /* _MSC_VER < 1800  */
+#endif /* _MSC_VER < 1800 */
 
   /* As far as we can tell, the lfence and sfence instructions are not  */
   /* currently needed or useful for cached memory accesses.             */
diff --git a/src/atomic_ops/sysdeps/standard_ao_double_t.h b/src/atomic_ops/sysdeps/standard_ao_double_t.h
index 630c376..7b69ae4 100644
--- a/src/atomic_ops/sysdeps/standard_ao_double_t.h
+++ b/src/atomic_ops/sysdeps/standard_ao_double_t.h
@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2004-2011 Hewlett-Packard Development Company, L.P.
+ * Copyright (c) 2012-2021 Ivan Maidanski
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -39,6 +40,9 @@
 # else /* pragma diagnostic is not supported */
     typedef unsigned __int128 double_ptr_storage;
 # endif
+#elif defined(_M_ARM64) && defined(_MSC_VER)
+  /* __int128 does not seem to be available.    */
+  typedef __declspec(align(16)) unsigned __int64 double_ptr_storage[2];
 #elif ((defined(__x86_64__) && AO_GNUC_PREREQ(4, 0)) || defined(_WIN64)) \
       && !defined(__ILP32__)
   /* x86-64 (except for x32): __m128 serves as a placeholder which also */