[Sanitizers] Solaris largefile fixes

While testing the Solaris libsanitizer port on GCC mainline, I found that
I'd messed up
the largefile checks in various ways, some of which showed as compile failures
(wrong structure sizes and member offsets), others at runtime, some of those only
on sparc as a big-endian target.

This patch fixes all of them:

- OFF_T is now correctly defined for 32-bit largefile and traditional
  environments, and 64-bit.

- The definition of __sanitizer_dirent now checks the correct conditionals.

- sanitizer_procmaps_solaris.cc undefines _FILE_OFFSET_BITS: before
  Solaris 11.4 <procfs.h> doesn't even compile with largefile support
  enabled, but the use at hand doesn't need it anyway while g++ 9 will
  define _FILE_OFFSET_BITS=64 out of the box.

- With full largefile support enabled, one needs to use e.g. mmap64
  instead of mmap; this is hidden behind macros.

With this patch I could bootstrap gcc mainline on both sparc-sun-solaris2.11 and
i386-pc-solaris2.11.  In addition, I've successfully built llvm on
i386-pc-solaris2.11.

Differential Revision: https://reviews.llvm.org/D54101


git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@346153 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 32 insertions, 19 deletions

diff --git a/lib/sanitizer_common/sanitizer_internal_defs.h b/lib/sanitizer_common/sanitizer_internal_defs.h
index 34022430d..397bc4cf3 100644
--- a/lib/sanitizer_common/sanitizer_internal_defs.h
+++ b/lib/sanitizer_common/sanitizer_internal_defs.h
@@ -172,6 +172,7 @@ typedef int pid_t;
 
 #if SANITIZER_FREEBSD || SANITIZER_NETBSD || \
     SANITIZER_OPENBSD || SANITIZER_MAC || \
+    (SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)) || \
     (SANITIZER_LINUX && defined(__x86_64__))
 typedef u64 OFF_T;
 #else
diff --git a/lib/sanitizer_common/sanitizer_platform_limits_solaris.h b/lib/sanitizer_common/sanitizer_platform_limits_solaris.h
index c0aa4cc1b..08bd24cd2 100644
--- a/lib/sanitizer_common/sanitizer_platform_limits_solaris.h
+++ b/lib/sanitizer_common/sanitizer_platform_limits_solaris.h
@@ -210,8 +210,7 @@ struct __sanitizer_cmsghdr {
   int cmsg_type;
 };
 
-#if SANITIZER_SOLARIS32 && 0
-// FIXME: need to deal with large file and non-large file cases
+#if SANITIZER_SOLARIS && (defined(_LP64) || _FILE_OFFSET_BITS == 64)
 struct __sanitizer_dirent {
   unsigned long long d_ino;
   long long d_off;
diff --git a/lib/sanitizer_common/sanitizer_procmaps_solaris.cc b/lib/sanitizer_common/sanitizer_procmaps_solaris.cc
index bfe83170f..9e5e37e6b 100644
--- a/lib/sanitizer_common/sanitizer_procmaps_solaris.cc
+++ b/lib/sanitizer_common/sanitizer_procmaps_solaris.cc
@@ -13,6 +13,8 @@
 #include "sanitizer_common.h"
 #include "sanitizer_procmaps.h"
 
+// Before Solaris 11.4, <procfs.h> doesn't work in a largefile environment.
+#undef _FILE_OFFSET_BITS
 #include <procfs.h>
 #include <limits.h>
 
diff --git a/lib/sanitizer_common/sanitizer_solaris.cc b/lib/sanitizer_common/sanitizer_solaris.cc
index a5db22994..9d0c3d93d 100644
--- a/lib/sanitizer_common/sanitizer_solaris.cc
+++ b/lib/sanitizer_common/sanitizer_solaris.cc
@@ -48,10 +48,21 @@ namespace __sanitizer {
   DECLARE__REAL(ret_type, func, __VA_ARGS__); \
   ret_type internal_ ## func(__VA_ARGS__)
 
+#if !defined(_LP64) && _FILE_OFFSET_BITS == 64
+#define _REAL64(func) _ ## func ## 64
+#else
+#define _REAL64(func) _REAL(func)
+#endif
+#define DECLARE__REAL64(ret_type, func, ...) \
+  extern "C" ret_type _REAL64(func)(__VA_ARGS__)
+#define DECLARE__REAL_AND_INTERNAL64(ret_type, func, ...) \
+  DECLARE__REAL64(ret_type, func, __VA_ARGS__); \
+  ret_type internal_ ## func(__VA_ARGS__)
+
 // ---------------------- sanitizer_libc.h
-DECLARE__REAL_AND_INTERNAL(uptr, mmap, void *addr, uptr /*size_t*/ length,
-                           int prot, int flags, int fd, OFF_T offset) {
-  return (uptr)_REAL(mmap)(addr, length, prot, flags, fd, offset);
+DECLARE__REAL_AND_INTERNAL64(uptr, mmap, void *addr, uptr /*size_t*/ length,
+                             int prot, int flags, int fd, OFF_T offset) {
+  return (uptr)_REAL64(mmap)(addr, length, prot, flags, fd, offset);
 }
 
 DECLARE__REAL_AND_INTERNAL(uptr, munmap, void *addr, uptr length) {
@@ -66,14 +77,14 @@ DECLARE__REAL_AND_INTERNAL(uptr, close, fd_t fd) {
   return _REAL(close)(fd);
 }
 
-extern "C" int _REAL(open)(const char *, int, ...);
+extern "C" int _REAL64(open)(const char *, int, ...);
 
 uptr internal_open(const char *filename, int flags) {
-  return _REAL(open)(filename, flags);
+  return _REAL64(open)(filename, flags);
 }
 
 uptr internal_open(const char *filename, int flags, u32 mode) {
-  return _REAL(open)(filename, flags, mode);
+  return _REAL64(open)(filename, flags, mode);
 }
 
 uptr OpenFile(const char *filename, bool write) {
@@ -94,16 +105,16 @@ DECLARE__REAL_AND_INTERNAL(uptr, ftruncate, fd_t fd, uptr size) {
   return ftruncate(fd, size);
 }
 
-DECLARE__REAL_AND_INTERNAL(uptr, stat, const char *path, void *buf) {
-  return _REAL(stat)(path, (struct stat *)buf);
+DECLARE__REAL_AND_INTERNAL64(uptr, stat, const char *path, void *buf) {
+  return _REAL64(stat)(path, (struct stat *)buf);
 }
 
-DECLARE__REAL_AND_INTERNAL(uptr, lstat, const char *path, void *buf) {
-  return _REAL(lstat)(path, (struct stat *)buf);
+DECLARE__REAL_AND_INTERNAL64(uptr, lstat, const char *path, void *buf) {
+  return _REAL64(lstat)(path, (struct stat *)buf);
 }
 
-DECLARE__REAL_AND_INTERNAL(uptr, fstat, fd_t fd, void *buf) {
-  return _REAL(fstat)(fd, (struct stat *)buf);
+DECLARE__REAL_AND_INTERNAL64(uptr, fstat, fd_t fd, void *buf) {
+  return _REAL64(fstat)(fd, (struct stat *)buf);
 }
 
 uptr internal_filesize(fd_t fd) {
@@ -153,13 +164,13 @@ DECLARE__REAL_AND_INTERNAL(uptr, getpid, void) {
 }
 
 // FIXME: This might be wrong: _getdents doesn't take a struct linux_dirent *.
-DECLARE__REAL_AND_INTERNAL(uptr, getdents, fd_t fd, struct linux_dirent *dirp,
-                           unsigned int count) {
-  return _REAL(getdents)(fd, dirp, count);
+DECLARE__REAL_AND_INTERNAL64(uptr, getdents, fd_t fd, struct linux_dirent *dirp,
+                             unsigned int count) {
+  return _REAL64(getdents)(fd, dirp, count);
 }
 
-DECLARE__REAL_AND_INTERNAL(uptr, lseek, fd_t fd, OFF_T offset, int whence) {
-  return _REAL(lseek)(fd, offset, whence);
+DECLARE__REAL_AND_INTERNAL64(uptr, lseek, fd_t fd, OFF_T offset, int whence) {
+  return _REAL64(lseek)(fd, offset, whence);
 }
 
 // FIXME: This might be wrong: _sigfillset doesn't take a