[Sanitizers] ASan and LSan allocator set errno on failure.

Summary:
Set proper errno code on alloction failures and change some
implementations to satisfy their man-specified requirements:

LSan: valloc and memalign
ASan: pvalloc, memalign and posix_memalign

Changing both allocators in one patch since LSan depends on ASan allocator in some configurations.

Reviewers: vitalybuka

Subscribers: kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D35440

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@308064 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 57 insertions, 18 deletions

diff --git a/lib/asan/asan_allocator.cc b/lib/asan/asan_allocator.cc
index de6613f56..0c7c7465f 100644
--- a/lib/asan/asan_allocator.cc
+++ b/lib/asan/asan_allocator.cc
@@ -22,6 +22,7 @@
 #include "asan_stack.h"
 #include "asan_thread.h"
 #include "sanitizer_common/sanitizer_allocator_interface.h"
+#include "sanitizer_common/sanitizer_errno.h"
 #include "sanitizer_common/sanitizer_flags.h"
 #include "sanitizer_common/sanitizer_internal_defs.h"
 #include "sanitizer_common/sanitizer_list.h"
@@ -799,11 +800,6 @@ void PrintInternalAllocatorStats() {
   instance.PrintStats();
 }
 
-void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
-                    AllocType alloc_type) {
-  return instance.Allocate(size, alignment, stack, alloc_type, true);
-}
-
 void asan_free(void *ptr, BufferedStackTrace *stack, AllocType alloc_type) {
   instance.Deallocate(ptr, 0, stack, alloc_type);
 }
@@ -813,17 +809,23 @@ void asan_sized_free(void *ptr, uptr size, BufferedStackTrace *stack,
   instance.Deallocate(ptr, size, stack, alloc_type);
 }
 
+inline void *check_ptr(void *ptr) {
+  if (UNLIKELY(!ptr))
+    errno = errno_ENOMEM;
+  return ptr;
+}
+
 void *asan_malloc(uptr size, BufferedStackTrace *stack) {
-  return instance.Allocate(size, 8, stack, FROM_MALLOC, true);
+  return check_ptr(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
 }
 
 void *asan_calloc(uptr nmemb, uptr size, BufferedStackTrace *stack) {
-  return instance.Calloc(nmemb, size, stack);
+  return check_ptr(instance.Calloc(nmemb, size, stack));
 }
 
 void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
   if (!p)
-    return instance.Allocate(size, 8, stack, FROM_MALLOC, true);
+    return check_ptr(instance.Allocate(size, 8, stack, FROM_MALLOC, true));
   if (size == 0) {
     if (flags()->allocator_frees_and_returns_null_on_realloc_zero) {
       instance.Deallocate(p, 0, stack, FROM_MALLOC);
@@ -832,26 +834,41 @@ void *asan_realloc(void *p, uptr size, BufferedStackTrace *stack) {
     // Allocate a size of 1 if we shouldn't free() on Realloc to 0
     size = 1;
   }
-  return instance.Reallocate(p, size, stack);
+  return check_ptr(instance.Reallocate(p, size, stack));
 }
 
 void *asan_valloc(uptr size, BufferedStackTrace *stack) {
-  return instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true);
+  return check_ptr(
+      instance.Allocate(size, GetPageSizeCached(), stack, FROM_MALLOC, true));
 }
 
 void *asan_pvalloc(uptr size, BufferedStackTrace *stack) {
   uptr PageSize = GetPageSizeCached();
-  size = RoundUpTo(size, PageSize);
-  if (size == 0) {
-    // pvalloc(0) should allocate one page.
-    size = PageSize;
+  // pvalloc(0) should allocate one page.
+  size = size ? RoundUpTo(size, PageSize) : PageSize;
+  return check_ptr(instance.Allocate(size, PageSize, stack, FROM_MALLOC, true));
+}
+
+void *asan_memalign(uptr alignment, uptr size, BufferedStackTrace *stack,
+                    AllocType alloc_type) {
+  if (UNLIKELY(!IsPowerOfTwo(alignment))) {
+    errno = errno_EINVAL;
+    return AsanAllocator::FailureHandler::OnBadRequest();
   }
-  return instance.Allocate(size, PageSize, stack, FROM_MALLOC, true);
+  return check_ptr(
+      instance.Allocate(size, alignment, stack, alloc_type, true));
 }
 
 int asan_posix_memalign(void **memptr, uptr alignment, uptr size,
                         BufferedStackTrace *stack) {
+  if (UNLIKELY(!IsPowerOfTwo(alignment) ||
+               (alignment % sizeof(void *)) != 0)) {  // NOLINT
+    AsanAllocator::FailureHandler::OnBadRequest();
+    return errno_EINVAL;
+  }
   void *ptr = instance.Allocate(size, alignment, stack, FROM_MALLOC, true);
+  if (!ptr)
+    return errno_ENOMEM;
   CHECK(IsAligned((uptr)ptr, alignment));
   *memptr = ptr;
   return 0;
diff --git a/lib/asan/tests/asan_test.cc b/lib/asan/tests/asan_test.cc
index d0128e34d..78c3a0e4e 100644
--- a/lib/asan/tests/asan_test.cc
+++ b/lib/asan/tests/asan_test.cc
@@ -12,6 +12,8 @@
 //===----------------------------------------------------------------------===//
 #include "asan_test_utils.h"
 
+#include <errno.h>
+
 NOINLINE void *malloc_fff(size_t size) {
   void *res = malloc/**/(size); break_optimization(0); return res;}
 NOINLINE void *malloc_eee(size_t size) {
@@ -74,10 +76,19 @@ TEST(AddressSanitizer, VariousMallocsTest) {
   delete c;
 
 #if SANITIZER_TEST_HAS_POSIX_MEMALIGN
-  int *pm;
-  int pm_res = posix_memalign((void**)&pm, kPageSize, kPageSize);
+  void *pm = 0;
+  // Valid allocation.
+  int pm_res = posix_memalign(&pm, kPageSize, kPageSize);
   EXPECT_EQ(0, pm_res);
+  EXPECT_NE(nullptr, pm);
   free(pm);
+
+  // Alignment is not a power of 2.
+  EXPECT_DEATH(posix_memalign(&pm, 3, kPageSize),
+               "allocator is terminating the process instead of returning 0");
+  // Alignment is a power of 2, but not a multiple of size(void *).
+  EXPECT_DEATH(posix_memalign(&pm, 2, kPageSize),
+               "allocator is terminating the process instead of returning 0");
 #endif  // SANITIZER_TEST_HAS_POSIX_MEMALIGN
 
 #if SANITIZER_TEST_HAS_MEMALIGN
@@ -85,6 +96,9 @@ TEST(AddressSanitizer, VariousMallocsTest) {
   EXPECT_EQ(0U, (uintptr_t)ma % kPageSize);
   ma[123] = 0;
   free(ma);
+
+  EXPECT_DEATH(memalign(3, kPageSize),
+               "allocator is terminating the process instead of returning 0");
 #endif  // SANITIZER_TEST_HAS_MEMALIGN
 }
 
diff --git a/test/asan/TestCases/allocator_returns_null.cc b/test/asan/TestCases/allocator_returns_null.cc
index 90e25b55e..309814451 100644
--- a/test/asan/TestCases/allocator_returns_null.cc
+++ b/test/asan/TestCases/allocator_returns_null.cc
@@ -37,9 +37,10 @@
 // RUN:   | FileCheck %s --check-prefix=CHECK-nnNULL
 
 #include <assert.h>
-#include <string.h>
+#include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <string.h>
 #include <limits>
 #include <new>
 
@@ -84,6 +85,8 @@ int main(int argc, char **argv) {
     assert(0);
   }
 
+  fprintf(stderr, "errno: %d\n", errno);
+
   // The NULL pointer is printed differently on different systems, while (long)0
   // is always the same.
   fprintf(stderr, "x: %lx\n", (long)x);
@@ -108,14 +111,19 @@ int main(int argc, char **argv) {
 // CHECK-nnCRASH: AddressSanitizer's allocator is terminating the process
 
 // CHECK-mNULL: malloc:
+// CHECK-mNULL: errno: 12
 // CHECK-mNULL: x: 0
 // CHECK-cNULL: calloc:
+// CHECK-cNULL: errno: 12
 // CHECK-cNULL: x: 0
 // CHECK-coNULL: calloc-overflow:
+// CHECK-coNULL: errno: 12
 // CHECK-coNULL: x: 0
 // CHECK-rNULL: realloc:
+// CHECK-rNULL: errno: 12
 // CHECK-rNULL: x: 0
 // CHECK-mrNULL: realloc-after-malloc:
+// CHECK-mrNULL: errno: 12
 // CHECK-mrNULL: x: 0
 // CHECK-nnNULL: new-nothrow:
 // CHECK-nnNULL: x: 0