issue-448: New environment setting that allows you to set an explicit heap limit

TCMALLOC_HEAP_LIMIT_MB - The maximum amount of heap memory that tcmalloc can use.
TCMALLOC_DISABLE_MEMORY_RELEASE - emulate platform with no MADV_DONTNEED support (generally for testing purposes).


git-svn-id: http://gperftools.googlecode.com/svn/trunk@178 6b5cf1ce-ec42-a296-1ba9-69fdba395a50

7 files changed, 244 insertions, 39 deletions

diff --git a/src/page_heap.cc b/src/page_heap.cc
index 18bdb94..8f0e967 100644
--- a/src/page_heap.cc
+++ b/src/page_heap.cc
@@ -50,6 +50,14 @@ DEFINE_double(tcmalloc_release_rate,
               "to return memory slower.  Reasonable rates are in the "
               "range [0,10]");
 
+DEFINE_int64(tcmalloc_heap_limit_mb,
+              EnvToInt("TCMALLOC_HEAP_LIMIT_MB", 0),
+              "Limit total size of the process heap to the "
+              "specified number of MiB. "
+              "When we approach the limit the memory is released "
+              "to the system more aggressively (more minor page faults). "
+              "Zero means to allocate as long as system allows.");
+
 namespace tcmalloc {
 
 PageHeap::PageHeap()
@@ -82,8 +90,18 @@ Span* PageHeap::SearchFreeAndLargeLists(Length n) {
     // Alternatively, maybe there's a usable returned span.
     ll = &free_[s].returned;
     if (!DLL_IsEmpty(ll)) {
-      ASSERT(ll->next->location == Span::ON_RETURNED_FREELIST);
-      return Carve(ll->next, n);
+      // We did not call EnsureLimit before, to avoid releasing the span
+      // that will be taken immediately back.
+      // Calling EnsureLimit here is not very expensive, as it fails only if
+      // there is no more normal spans (and it fails efficiently)
+      // or SystemRelease does not work (there is probably no returned spans).
+      if (EnsureLimit(n)) {
+        // ll may have became empty due to coalescing
+        if (!DLL_IsEmpty(ll)) {
+          ASSERT(ll->next->location == Span::ON_RETURNED_FREELIST);
+          return Carve(ll->next, n);
+        }
+      }
     }
   }
   // No luck in free lists, our last chance is in a larger class.
@@ -125,6 +143,8 @@ Span* PageHeap::AllocLarge(Length n) {
     }
   }
 
+  Span *bestNormal = best;
+
   // Search through released list in case it has a better fit
   for (Span* span = large_.returned.next;
        span != &large_.returned;
@@ -139,7 +159,27 @@ Span* PageHeap::AllocLarge(Length n) {
     }
   }
 
-  return best == NULL ? NULL : Carve(best, n);
+  if (best == bestNormal) {
+    return best == NULL ? NULL : Carve(best, n);
+  }
+
+  // best comes from returned list.
+
+  if (EnsureLimit(n, false)) {
+    return Carve(best, n);
+  }
+
+  if (EnsureLimit(n, true)) {
+    // best could have been destroyed by coalescing.
+    // bestNormal is not a best-fit, and it could be destroyed as well.
+    // We retry, the limit is already ensured:
+    return AllocLarge(n);
+  }
+
+  // If bestNormal existed, EnsureLimit would succeeded:
+  ASSERT(bestNormal == NULL);
+  // We are not allowed to take best from returned list.
+  return NULL;
 }
 
 Span* PageHeap::Split(Span* span, Length n) {
@@ -294,28 +334,26 @@ void PageHeap::IncrementalScavenge(Length n) {
 Length PageHeap::ReleaseLastNormalSpan(SpanList* slist) {
   Span* s = slist->normal.prev;
   ASSERT(s->location == Span::ON_NORMAL_FREELIST);
-  RemoveFromFreeList(s);
-  const Length n = s->length;
-  TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),
-                         static_cast<size_t>(s->length << kPageShift));
-  s->location = Span::ON_RETURNED_FREELIST;
-  MergeIntoFreeList(s);  // Coalesces if possible.
-  return n;
+
+  if (TCMalloc_SystemRelease(reinterpret_cast<void*>(s->start << kPageShift),
+                         static_cast<size_t>(s->length << kPageShift))) {
+    RemoveFromFreeList(s);
+    const Length n = s->length;
+    s->location = Span::ON_RETURNED_FREELIST;
+    MergeIntoFreeList(s);  // Coalesces if possible.
+    return n;
+  }
+
+  return 0;
 }
 
 Length PageHeap::ReleaseAtLeastNPages(Length num_pages) {
   Length released_pages = 0;
-  Length prev_released_pages = -1;
 
   // Round robin through the lists of free spans, releasing the last
-  // span in each list.  Stop after releasing at least num_pages.
-  while (released_pages < num_pages) {
-    if (released_pages == prev_released_pages) {
-      // Last iteration of while loop made no progress.
-      break;
-    }
-    prev_released_pages = released_pages;
-
+  // span in each list.  Stop after releasing at least num_pages
+  // or when there is nothing more to release.
+  while (released_pages < num_pages && stats_.free_bytes > 0) {
     for (int i = 0; i < kMaxPages+1 && released_pages < num_pages;
          i++, release_index_++) {
       if (release_index_ > kMaxPages) release_index_ = 0;
@@ -323,6 +361,8 @@ Length PageHeap::ReleaseAtLeastNPages(Length num_pages) {
           &large_ : &free_[release_index_];
       if (!DLL_IsEmpty(&slist->normal)) {
         Length released_len = ReleaseLastNormalSpan(slist);
+        // Some systems do not support release
+        if (released_len == 0) return released_pages;
         released_pages += released_len;
       }
     }
@@ -330,6 +370,30 @@ Length PageHeap::ReleaseAtLeastNPages(Length num_pages) {
   return released_pages;
 }
 
+bool PageHeap::EnsureLimit(Length n, bool withRelease)
+{
+  Length limit = (FLAGS_tcmalloc_heap_limit_mb*1024*1024) >> kPageShift;
+  if (limit == 0) return true; //there is no limit
+
+  // We do not use stats_.system_bytes because it does not take
+  // MetaDataAllocs into account.
+  Length takenPages = TCMalloc_SystemTaken >> kPageShift;
+  //XXX takenPages may be slightly bigger than limit for two reasons:
+  //* MetaDataAllocs ignore the limit (it is not easy to handle
+  //  out of memory there)
+  //* sys_alloc may round allocation up to huge page size,
+  //  although smaller limit was ensured
+
+  ASSERT(takenPages >= stats_.unmapped_bytes >> kPageShift);
+  takenPages -= stats_.unmapped_bytes >> kPageShift;
+
+  if (takenPages + n > limit && withRelease) {
+    takenPages -= ReleaseAtLeastNPages(takenPages + n - limit);
+  }
+
+  return takenPages + n <= limit;
+}
+
 void PageHeap::RegisterSizeClass(Span* span, size_t sc) {
   // Associate span object with all interior pages as well
   ASSERT(span->location == Span::IN_USE);
@@ -407,12 +471,17 @@ bool PageHeap::GrowHeap(Length n) {
   if (n > kMaxValidPages) return false;
   Length ask = (n>kMinSystemAlloc) ? n : static_cast<Length>(kMinSystemAlloc);
   size_t actual_size;
-  void* ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);
+  void* ptr = NULL;
+  if (EnsureLimit(ask)) {
+      ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);
+  }
   if (ptr == NULL) {
     if (n < ask) {
       // Try growing just "n" pages
       ask = n;
-      ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);
+      if (EnsureLimit(ask)) {
+        ptr = TCMalloc_SystemAlloc(ask << kPageShift, &actual_size, kPageSize);
+      }
     }
     if (ptr == NULL) return false;
   }
diff --git a/src/page_heap.h b/src/page_heap.h
index 3718801..e06e203 100644
--- a/src/page_heap.h
+++ b/src/page_heap.h
@@ -274,9 +274,13 @@ class PERFTOOLS_DLL_DECL PageHeap {
   void IncrementalScavenge(Length n);
 
   // Release the last span on the normal portion of this list.
-  // Return the length of that span.
+  // Return the length of that span or zero if release failed.
   Length ReleaseLastNormalSpan(SpanList* slist);
 
+  // Checks if we are allowed to take more memory from the system.
+  // If limit is reached and allowRelease is true, tries to release
+  // some unused spans.
+  bool EnsureLimit(Length n, bool allowRelease = true);
 
   // Number of pages to deallocate before doing more scavenging
   int64_t scavenge_counter_;
diff --git a/src/system-alloc.cc b/src/system-alloc.cc
index abfe472..e380d16 100644
--- a/src/system-alloc.cc
+++ b/src/system-alloc.cc
@@ -122,6 +122,9 @@ static size_t pagesize = 0;
 // The current system allocator
 SysAllocator* sys_alloc = NULL;
 
+// Number of bytes taken from system.
+size_t TCMalloc_SystemTaken = 0;
+
 // Configuration parameters.
 DEFINE_int32(malloc_devmem_start,
              EnvToInt("TCMALLOC_DEVMEM_START", 0),
@@ -137,6 +140,10 @@ DEFINE_bool(malloc_skip_sbrk,
 DEFINE_bool(malloc_skip_mmap,
             EnvToBool("TCMALLOC_SKIP_MMAP", false),
             "Whether mmap can be used to obtain memory.");
+DEFINE_bool(malloc_disable_memory_release,
+            EnvToBool("TCMALLOC_DISABLE_MEMORY_RELEASE", false),
+            "Whether MADV_FREE/MADV_DONTNEED should be used"
+            " to return unused memory to the system.");
 
 // static allocators
 class SbrkSysAllocator : public SysAllocator {
@@ -485,21 +492,24 @@ void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size,
     if (actual_size) {
       CheckAddressBits<kAddressBits>(
           reinterpret_cast<uintptr_t>(result) + *actual_size - 1);
+      TCMalloc_SystemTaken += *actual_size;
     } else {
       CheckAddressBits<kAddressBits>(
           reinterpret_cast<uintptr_t>(result) + size - 1);
+      TCMalloc_SystemTaken += size;
     }
   }
   return result;
 }
 
-void TCMalloc_SystemRelease(void* start, size_t length) {
+bool TCMalloc_SystemRelease(void* start, size_t length) {
 #ifdef MADV_FREE
   if (FLAGS_malloc_devmem_start) {
     // It's not safe to use MADV_FREE/MADV_DONTNEED if we've been
     // mapping /dev/mem for heap memory.
-    return;
+    return false;
   }
+  if (FLAGS_malloc_disable_memory_release) return false;
   if (pagesize == 0) pagesize = getpagesize();
   const size_t pagemask = pagesize - 1;
 
@@ -518,13 +528,14 @@ void TCMalloc_SystemRelease(void* start, size_t length) {
   ASSERT(new_end <= end);
 
   if (new_end > new_start) {
-    // Note -- ignoring most return codes, because if this fails it
-    // doesn't matter...
-    while (madvise(reinterpret_cast<char*>(new_start), new_end - new_start,
-                   MADV_FREE) == -1 &&
-           errno == EAGAIN) {
-      // NOP
-    }
+    int result;
+    do {
+      result = madvise(reinterpret_cast<char*>(new_start),
+          new_end - new_start, MADV_FREE);
+    } while (result == -1 && errno == EAGAIN);
+
+    return result != -1;
   }
 #endif
+  return false;
 }
diff --git a/src/system-alloc.h b/src/system-alloc.h
index 814b556..2d49fc3 100644
--- a/src/system-alloc.h
+++ b/src/system-alloc.h
@@ -69,9 +69,14 @@ extern void* TCMalloc_SystemAlloc(size_t bytes, size_t *actual_bytes,
 // the address space next time they are touched, which can impact
 // performance.  (Only pages fully covered by the memory region will
 // be released, partial pages will not.)
-extern void TCMalloc_SystemRelease(void* start, size_t length);
+//
+// Returns false if release failed or not supported.
+extern bool TCMalloc_SystemRelease(void* start, size_t length);
 
 // The current system allocator.
 extern PERFTOOLS_DLL_DECL SysAllocator* sys_alloc;
 
+// Number of bytes taken from system.
+extern PERFTOOLS_DLL_DECL size_t TCMalloc_SystemTaken;
+
 #endif /* TCMALLOC_SYSTEM_ALLOC_H_ */
diff --git a/src/tests/page_heap_test.cc b/src/tests/page_heap_test.cc
index 9f5f3c8..f08387c 100644
--- a/src/tests/page_heap_test.cc
+++ b/src/tests/page_heap_test.cc
@@ -3,17 +3,29 @@
 
 #include "config_for_unittests.h"
 #include "page_heap.h"
+#include "system-alloc.h"
 #include <stdio.h>
 #include "base/logging.h"
 #include "common.h"
 
+DECLARE_int64(tcmalloc_heap_limit_mb);
+
 namespace {
 
+static bool HaveSystemRelease =
+    TCMalloc_SystemRelease(TCMalloc_SystemAlloc(kPageSize, NULL, 0), kPageSize);
+
 static void CheckStats(const tcmalloc::PageHeap* ph,
                        uint64_t system_pages,
                        uint64_t free_pages,
                        uint64_t unmapped_pages) {
   tcmalloc::PageHeap::Stats stats = ph->stats();
+
+  if (!HaveSystemRelease) {
+    free_pages += unmapped_pages;
+    unmapped_pages = 0;
+  }
+
   EXPECT_EQ(system_pages, stats.system_bytes >> kPageShift);
   EXPECT_EQ(free_pages, stats.free_bytes >> kPageShift);
   EXPECT_EQ(unmapped_pages, stats.unmapped_bytes >> kPageShift);
@@ -36,7 +48,7 @@ static void TestPageHeap_Stats() {
   CheckStats(ph, 256, 128, 0);
 
   // Unmap deleted span 's2'
-  EXPECT_EQ(s2_len, ph->ReleaseAtLeastNPages(1));
+  ph->ReleaseAtLeastNPages(1);
   CheckStats(ph, 256, 0, 128);
 
   // Delete span 's1'
@@ -46,10 +58,99 @@ static void TestPageHeap_Stats() {
   delete ph;
 }
 
+static void TestPageHeap_Limit() {
+  tcmalloc::PageHeap* ph = new tcmalloc::PageHeap();
+
+  CHECK_EQ(kMaxPages, 1 << (20 - kPageShift));
+
+  // We do not know much is taken from the system for other purposes,
+  // so we detect the proper limit:
+  {
+    FLAGS_tcmalloc_heap_limit_mb = 1;
+    tcmalloc::Span* s = NULL;
+    while((s = ph->New(kMaxPages)) == NULL) {
+      FLAGS_tcmalloc_heap_limit_mb++;
+    }
+    FLAGS_tcmalloc_heap_limit_mb += 9;
+    ph->Delete(s);
+    // We are [10, 11) mb from the limit now.
+  }
+
+  // Test AllocLarge and GrowHeap first:
+  {
+    tcmalloc::Span * spans[10];
+    for (int i=0; i<10; ++i) {
+      spans[i] = ph->New(kMaxPages);
+      EXPECT_NE(spans[i], NULL);
+    }
+    EXPECT_EQ(ph->New(kMaxPages), NULL);
+
+    for (int i=0; i<10; i += 2) {
+      ph->Delete(spans[i]);
+    }
+
+    tcmalloc::Span *defragmented = ph->New(5 * kMaxPages);
+
+    if (HaveSystemRelease) {
+      // EnsureLimit should release deleted normal spans
+      EXPECT_NE(defragmented, NULL);
+      EXPECT_TRUE(ph->CheckExpensive());
+      ph->Delete(defragmented);
+    }
+    else
+    {
+      EXPECT_EQ(defragmented, NULL);
+      EXPECT_TRUE(ph->CheckExpensive());
+    }
+
+    for (int i=1; i<10; i += 2) {
+      ph->Delete(spans[i]);
+    }
+  }
+
+  // Once again, testing small lists this time (twice smaller spans):
+  {
+    tcmalloc::Span * spans[20];
+    for (int i=0; i<20; ++i) {
+      spans[i] = ph->New(kMaxPages >> 1);
+      EXPECT_NE(spans[i], NULL);
+    }
+    // one more half size allocation may be possible:
+    tcmalloc::Span * lastHalf = ph->New(kMaxPages >> 1);
+    EXPECT_EQ(ph->New(kMaxPages >> 1), NULL);
+
+    for (int i=0; i<20; i += 2) {
+      ph->Delete(spans[i]);
+    }
+
+    for(Length len = kMaxPages >> 2; len < 5 * kMaxPages; len = len << 1)
+    {
+      if(len <= kMaxPages >> 1 || HaveSystemRelease) {
+        tcmalloc::Span *s = ph->New(len);
+        EXPECT_NE(s, NULL);
+        ph->Delete(s);
+      }
+    }
+
+    EXPECT_TRUE(ph->CheckExpensive());
+
+    for (int i=1; i<20; i += 2) {
+      ph->Delete(spans[i]);
+    }
+
+    if (lastHalf != NULL) {
+      ph->Delete(lastHalf);
+    }
+  }
+
+  delete ph;
+}
+
 }  // namespace
 
 int main(int argc, char **argv) {
   TestPageHeap_Stats();
+  TestPageHeap_Limit();
   printf("PASS\n");
   return 0;
 }
diff --git a/src/tests/tcmalloc_unittest.cc b/src/tests/tcmalloc_unittest.cc
index e9526ae..50bc107 100644
--- a/src/tests/tcmalloc_unittest.cc
+++ b/src/tests/tcmalloc_unittest.cc
@@ -92,6 +92,7 @@
 #include "gperftools/malloc_extension.h"
 #include "gperftools/tcmalloc.h"
 #include "thread_cache.h"
+#include "system-alloc.h"
 #include "tests/testutil.h"
 
 // Windows doesn't define pvalloc and a few other obsolete unix
@@ -835,20 +836,26 @@ static void CheckRangeCallback(void* ptr, base::MallocRange::Type type,
 
 }
 
+static bool HaveSystemRelease =
+    TCMalloc_SystemRelease(TCMalloc_SystemAlloc(kPageSize, NULL, 0), kPageSize);
+
 static void TestRanges() {
   static const int MB = 1048576;
   void* a = malloc(MB);
   void* b = malloc(MB);
+  base::MallocRange::Type releasedType =
+      HaveSystemRelease ? base::MallocRange::UNMAPPED : base::MallocRange::FREE;
+
   CheckRangeCallback(a, base::MallocRange::INUSE, MB);
   CheckRangeCallback(b, base::MallocRange::INUSE, MB);
   free(a);
   CheckRangeCallback(a, base::MallocRange::FREE, MB);
   CheckRangeCallback(b, base::MallocRange::INUSE, MB);
   MallocExtension::instance()->ReleaseFreeMemory();
-  CheckRangeCallback(a, base::MallocRange::UNMAPPED, MB);
+  CheckRangeCallback(a, releasedType, MB);
   CheckRangeCallback(b, base::MallocRange::INUSE, MB);
   free(b);
-  CheckRangeCallback(a, base::MallocRange::UNMAPPED, MB);
+  CheckRangeCallback(a, releasedType, MB);
   CheckRangeCallback(b, base::MallocRange::FREE, MB);
 }
 
@@ -866,6 +873,9 @@ static void TestReleaseToSystem() {
   // messes up all the equality tests here.  I just disable the
   // teset in this mode.  TODO(csilvers): get it to work for debugalloc?
 #ifndef DEBUGALLOCATION
+
+  if(!HaveSystemRelease) return;
+
   const double old_tcmalloc_release_rate = FLAGS_tcmalloc_release_rate;
   FLAGS_tcmalloc_release_rate = 0;
 
diff --git a/src/windows/port.cc b/src/windows/port.cc
index e68de16..641c3c9 100644
--- a/src/windows/port.cc
+++ b/src/windows/port.cc
@@ -218,6 +218,11 @@ extern "C" int perftools_pthread_once(pthread_once_t *once_control,
 // -----------------------------------------------------------------------
 // These functions replace system-alloc.cc
 
+// The current system allocator declaration (unused here)
+SysAllocator* sys_alloc = NULL;
+// Number of bytes taken from system.
+size_t TCMalloc_SystemTaken = 0;
+
 // This is mostly like MmapSysAllocator::Alloc, except it does these weird
 // munmap's in the middle of the page, which is forbidden in windows.
 extern void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size,
@@ -243,6 +248,8 @@ extern void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size,
   if (result == NULL)
     return NULL;
 
+  TCMalloc_SystemTaken += size + extra;
+
   // Adjust the return memory so it is aligned
   uintptr_t ptr = reinterpret_cast<uintptr_t>(result);
   size_t adjust = 0;
@@ -254,8 +261,9 @@ extern void* TCMalloc_SystemAlloc(size_t size, size_t *actual_size,
   return reinterpret_cast<void*>(ptr);
 }
 
-void TCMalloc_SystemRelease(void* start, size_t length) {
+bool TCMalloc_SystemRelease(void* start, size_t length) {
   // TODO(csilvers): should I be calling VirtualFree here?
+  return false;
 }
 
 bool RegisterSystemAllocator(SysAllocator *allocator, int priority) {
@@ -266,9 +274,6 @@ void DumpSystemAllocatorStats(TCMalloc_Printer* printer) {
   // We don't dump stats on windows, right now
 }
 
-// The current system allocator
-SysAllocator* sys_alloc = NULL;
-
 
 // -----------------------------------------------------------------------
 // These functions rework existing functions of the same name in the