bytesex: more endianness detection hackself

A few more tricks for sussing out endinanness, and add an ultimate
fallback option.

Signed-off-by: H. Peter Anvin <hpa@zytor.com>

2 files changed, 54 insertions, 11 deletions

diff --git a/include/bytesex.h b/include/bytesex.h
index 376bb6d6..ee20adff 100644
--- a/include/bytesex.h
+++ b/include/bytesex.h
@@ -223,8 +223,42 @@ static inline uint64_t cpu_to_le64(uint64_t v)
 
 #else /* not WORDS_LITTLEENDIAN or WORDS_BIGENDIAN */
 
-#error "Update byteord.h to include arbitrary byte orders"
+static inline uint16_t cpu_to_le16(uint16_t v)
+{
+    union u16 {
+        uint16_t v;
+        uint8_t c[2];
+    } x;
+    uint8_t *cp = &x.c;
+
+    WRITESHORT(cp, v);
+    return x.v;
+}
+
+static inline uint32_t cpu_to_le32(uint32_t v)
+{
+    union u32 {
+        uint32_t v;
+        uint8_t c[4];
+    } x;
+    uint8_t *cp = &x.c;
+
+    WRITELONG(cp, v);
+    return x.v;
+}
+
+static inline uint64_t cpu_to_le64(uint64_t v)
+{
+    union u64 {
+        uint64_t v;
+        uint8_t c[8];
+    } x;
+    uint8_t *cp = &x.c;
+
+    WRITEDLONG(cp, v);
+    return x.v;
+}
 
 #endif
 
-#endif /* NASM_BYTEORD_H */
+#endif /* NASM_BYTESEX_H */
diff --git a/include/compiler.h b/include/compiler.h
index 47d8595b..3bdd9ee2 100644
--- a/include/compiler.h
+++ b/include/compiler.h
@@ -96,11 +96,20 @@
 #endif
 
 /*
- * If we have BYTE_ORDER defined, trust it over what autoconf came up
- * with, especially since autoconf obviously can't figure things out
- * for a universal compiler.
+ * If we have BYTE_ORDER defined, or the compiler provides
+ * __BIG_ENDIAN__ or __LITTLE_ENDIAN__, trust it over what autoconf
+ * came up with, especially since autoconf obviously can't figure
+ * things out for a universal compiler.
  */
-#if defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN)
+#if defined(__BIG_ENDIAN__) && !defined(__LITTLE_ENDIAN__)
+# undef WORDS_LITTLEENDIAN
+# undef WORDS_BIGENDIAN
+# define WORDS_BIGENDIAN 1
+#elif defined(__LITTLE_ENDIAN__) && !defined(__BIG_ENDIAN__)
+# undef WORDS_LITTLEENDIAN
+# undef WORDS_BIGENDIAN
+# define WORDS_LITTLEENDIAN 1
+#elif defined(BYTE_ORDER) && defined(LITTLE_ENDIAN) && defined(BIG_ENDIAN)
 # undef WORDS_LITTLEENDIAN
 # undef WORDS_BIGENDIAN
 # if BYTE_ORDER == LITTLE_ENDIAN
@@ -112,15 +121,15 @@
 
 /*
  * Define this to 1 for faster performance if this is a littleendian
- * platform which can do unaligned memory references.  It is safe
- * to leave it defined to 0 even if that is true.
+ * platform *and* it can do arbitrary unaligned memory references.  It
+ * is safe to leave it defined to 0 even if that is true.
  */
 #if defined(__386__) || defined(__i386__) || defined(__x86_64__) \
     || defined(_M_IX86) || defined(_M_X64)
 # define X86_MEMORY 1
-# ifndef WORDS_LITTLEENDIAN
-#  define WORDS_LITTLEENDIAN 1
-# endif
+# undef WORDS_BIGENDIAN
+# undef WORDS_LITTLEENDIAN
+# define WORDS_LITTLEENDIAN 1
 #else
 # define X86_MEMORY 0
 #endif