[hwasan] Madvise away unused shadow.

Summary:
Whenever a large shadow region is tagged to zero, madvise(DONT_NEED)
as much of it as possible.

This reduces shadow RSS on Android by 45% or so, and total memory use
by 2-4%, probably even more on long running multithreaded programs.
CPU time seems to be in the noise.

Reviewers: kcc, pcc

Subscribers: srhines, kubamracek, llvm-commits

Differential Revision: https://reviews.llvm.org/D56757

git-svn-id: https://llvm.org/svn/llvm-project/compiler-rt/trunk@351620 91177308-0d34-0410-b5e6-96231b3b80d8

4 files changed, 100 insertions, 1 deletions

diff --git a/lib/hwasan/hwasan.cc b/lib/hwasan/hwasan.cc
index e2bfea5e4..ff2538316 100644
--- a/lib/hwasan/hwasan.cc
+++ b/lib/hwasan/hwasan.cc
@@ -88,6 +88,8 @@ static void InitializeFlags() {
     cf.check_printf = false;
     cf.intercept_tls_get_addr = true;
     cf.exitcode = 99;
+    // 8 shadow pages ~512kB, small enough to cover common stack sizes.
+    cf.clear_shadow_mmap_threshold = 4096 * (SANITIZER_ANDROID ? 2 : 8);
     // Sigtrap is used in error reporting.
     cf.handle_sigtrap = kHandleSignalExclusive;
 
diff --git a/lib/hwasan/hwasan_poisoning.cc b/lib/hwasan/hwasan_poisoning.cc
index 9c8e16b12..6fb7d15db 100644
--- a/lib/hwasan/hwasan_poisoning.cc
+++ b/lib/hwasan/hwasan_poisoning.cc
@@ -16,6 +16,7 @@
 #include "hwasan_mapping.h"
 #include "interception/interception.h"
 #include "sanitizer_common/sanitizer_common.h"
+#include "sanitizer_common/sanitizer_linux.h"
 
 namespace __hwasan {
 
@@ -24,7 +25,22 @@ uptr TagMemoryAligned(uptr p, uptr size, tag_t tag) {
   CHECK(IsAligned(size, kShadowAlignment));
   uptr shadow_start = MemToShadow(p);
   uptr shadow_size = MemToShadowSize(size);
-  internal_memset((void *)shadow_start, tag, shadow_size);
+
+  uptr page_size = GetPageSizeCached();
+  uptr page_start = RoundUpTo(shadow_start, page_size);
+  uptr page_end = RoundDownTo(shadow_start + shadow_size, page_size);
+  uptr threshold = common_flags()->clear_shadow_mmap_threshold;
+  if (SANITIZER_LINUX &&
+      UNLIKELY(page_end >= page_start + threshold && tag == 0)) {
+    internal_memset((void *)shadow_start, tag, page_start - shadow_start);
+    internal_memset((void *)page_end, tag,
+                    shadow_start + shadow_size - page_end);
+    // For an anonymous private mapping MADV_DONTNEED will return a zero page on
+    // Linux.
+    ReleaseMemoryPagesToOSAndZeroFill(page_start, page_end);
+  } else {
+    internal_memset((void *)shadow_start, tag, shadow_size);
+  }
   return AddTagToPointer(p, tag);
 }
 
diff --git a/lib/sanitizer_common/sanitizer_linux.h b/lib/sanitizer_common/sanitizer_linux.h
index c309e33f8..522eb092a 100644
--- a/lib/sanitizer_common/sanitizer_linux.h
+++ b/lib/sanitizer_common/sanitizer_linux.h
@@ -106,6 +106,17 @@ bool LibraryNameIs(const char *full_name, const char *base_name);
 // Call cb for each region mapped by map.
 void ForEachMappedRegion(link_map *map, void (*cb)(const void *, uptr));
 
+// Releases memory pages entirely within the [beg, end] address range.
+// The pages no longer count toward RSS; reads are guaranteed to return 0.
+// Requires (but does not verify!) that pages are MAP_PRIVATE.
+INLINE void ReleaseMemoryPagesToOSAndZeroFill(uptr beg, uptr end) {
+  // man madvise on Linux promises zero-fill for anonymous private pages.
+  // Testing shows the same behaviour for private (but not anonymous) mappings
+  // of shm_open() files, as long as the underlying file is untouched.
+  CHECK(SANITIZER_LINUX);
+  ReleaseMemoryPagesToOS(beg, end);
+}
+
 #if SANITIZER_ANDROID
 
 #if defined(__aarch64__)
diff --git a/test/hwasan/TestCases/Linux/release-shadow.c b/test/hwasan/TestCases/Linux/release-shadow.c
new file mode 100644
index 000000000..9aae35063
--- /dev/null
+++ b/test/hwasan/TestCases/Linux/release-shadow.c
@@ -0,0 +1,70 @@
+// Test that tagging a large region to 0 reduces RSS.
+// RUN: %clang_hwasan -mllvm -hwasan-instrument-stack=0 %s -o %t && %run %t 2>&1
+
+#include <assert.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include <sanitizer/hwasan_interface.h>
+
+const unsigned char kTag = 42;
+const size_t kNumShadowPages = 256;
+const size_t kNumPages = 16 * kNumShadowPages;
+const size_t kPageSize = 4096;
+const size_t kMapSize = kNumPages * kPageSize;
+
+void sync_rss() {
+  char *page = (char *)mmap(0, kPageSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+  // Linux kernel updates RSS counters after a set number of page faults.
+  for (int i = 0; i < 1000; ++i) {
+    page[0] = 42;
+    madvise(page, kPageSize, MADV_DONTNEED);
+  }
+  munmap(page, kPageSize);
+}
+
+size_t current_rss() {
+  sync_rss();
+  int statm_fd = open("/proc/self/statm", O_RDONLY);
+  assert(statm_fd >= 0);
+
+  char buf[100];
+  assert(read(statm_fd, &buf, sizeof(buf)) > 0);
+  size_t size, rss;
+  assert(sscanf(buf, "%zu %zu", &size, &rss) == 2);
+
+  close(statm_fd);
+  return rss;
+}
+
+void test_rss_difference(void *p) {
+  __hwasan_tag_memory(p, kTag, kMapSize);
+  size_t rss_before = current_rss();
+  __hwasan_tag_memory(p, 0, kMapSize);
+  size_t rss_after = current_rss();
+  fprintf(stderr, "%zu -> %zu\n", rss_before, rss_after);
+  assert(rss_before > rss_after);
+  size_t diff = rss_before - rss_after;
+  fprintf(stderr, "diff %zu\n", diff);
+  // Check that the difference is at least close to kNumShadowPages.
+  assert(diff > kNumShadowPages / 4 * 3);
+}
+
+int main() {
+  fprintf(stderr, "starting rss %zu\n", current_rss());
+  fprintf(stderr, "shadow pages: %zu\n", kNumShadowPages);
+
+  void *p = mmap(0, kMapSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, 0, 0);
+  fprintf(stderr, "p = %p\n", p);
+
+  test_rss_difference(p);
+  test_rss_difference(p);
+  test_rss_difference(p);
+
+  return 0;
+}