issue-502: Count m(un)map for each stacktrace in MemoryRegionMap

..instead of HeapProfileTable

This upstreams chromium commit reviewed at:
https://codereview.chromium.org/12388070

Original and upstreaming author is: Dai MIKURUBE

This patch fixes a bug that gperftools(TCMalloc)'s mmap profiler
(HEAP_PROFILE_MMAP) doesn't hook some memory pages used by the
profiler itself.

This problem has been lived in gperftools for a long time.
It is discussed in gperftools' issue 502.
https://code.google.com/p/gperftools/issues/detail?id=502

Some bugs in the mmap profiler were fixed by
https://code.google.com/p/gperftools/issues/detail?id=383,
but the patch in the issue 383 didn't fix the bug mentioned in
the issue 502.

This change reverts the previous patch and http://crrev.com/132771
at first. Then, it modifies MemoryRegionMap to count m(un)map
calls for each stacktrace in itself instead of merging the counts
for each stacktrace in HeapProfileTable.

This change also cleans up heap-profiler, heap-profile-table and
deep-heap-profile.

Chromium-BUG=https://code.google.com/p/chromium/issues/detail?id=181517
Chromium-Committed: https://src.chromium.org/viewvc/chrome?view=rev&revision=188176

7 files changed, 440 insertions, 226 deletions

diff --git a/src/heap-checker.cc b/src/heap-checker.cc
index c050866..040a33e 100644
--- a/src/heap-checker.cc
+++ b/src/heap-checker.cc
@@ -2206,13 +2206,14 @@ void HeapLeakChecker::BeforeConstructorsLocked() {
   RAW_CHECK(MallocHook::AddNewHook(&NewHook), "");
   RAW_CHECK(MallocHook::AddDeleteHook(&DeleteHook), "");
   constructor_heap_profiling = true;
-  MemoryRegionMap::Init(1);
+  MemoryRegionMap::Init(1, /* use_buckets */ false);
     // Set up MemoryRegionMap with (at least) one caller stack frame to record
     // (important that it's done before HeapProfileTable creation below).
   Allocator::Init();
   RAW_CHECK(heap_profile == NULL, "");
   heap_profile = new(Allocator::Allocate(sizeof(HeapProfileTable)))
-                   HeapProfileTable(&Allocator::Allocate, &Allocator::Free);
+      HeapProfileTable(&Allocator::Allocate, &Allocator::Free,
+                       /* profile_mmap */ false);
   RAW_VLOG(10, "Starting tracking the heap");
   heap_checker_on = true;
 }
diff --git a/src/heap-profile-stats.h b/src/heap-profile-stats.h
new file mode 100644
index 0000000..caf1c27
--- /dev/null
+++ b/src/heap-profile-stats.h
@@ -0,0 +1,77 @@
+// Copyright (c) 2013, Google Inc.
+// All rights reserved.
+// 
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+// 
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+// 
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+// This file defines structs to accumulate memory allocation and deallocation
+// counts.  These structs are commonly used for malloc (in HeapProfileTable)
+// and mmap (in MemoryRegionMap).
+
+// A bucket is data structure for heap profiling to store a pair of a stack
+// trace and counts of (de)allocation.  Buckets are stored in a hash table
+// which is declared as "HeapProfileBucket**".
+//
+// A hash value is computed from a stack trace.  Collision in the hash table
+// is resolved by separate chaining with linked lists.  The links in the list
+// are implemented with the member "HeapProfileBucket* next".
+//
+// A structure of a hash table HeapProfileBucket** bucket_table would be like:
+// bucket_table[0] => NULL
+// bucket_table[1] => HeapProfileBucket() => HeapProfileBucket() => NULL
+// ...
+// bucket_table[i] => HeapProfileBucket() => NULL
+// ...
+// bucket_table[n] => HeapProfileBucket() => NULL
+
+#ifndef HEAP_PROFILE_STATS_H_
+#define HEAP_PROFILE_STATS_H_
+
+struct HeapProfileStats {
+  // Returns true if the two HeapProfileStats are semantically equal.
+  bool Equivalent(const HeapProfileStats& other) const {
+    return allocs - frees == other.allocs - other.frees &&
+        alloc_size - free_size == other.alloc_size - other.free_size;
+  }
+
+  int32 allocs;      // Number of allocation calls.
+  int32 frees;       // Number of free calls.
+  int64 alloc_size;  // Total size of all allocated objects so far.
+  int64 free_size;   // Total size of all freed objects so far.
+};
+
+// Allocation and deallocation statistics per each stack trace.
+struct HeapProfileBucket : public HeapProfileStats {
+  // Longest stack trace we record.
+  static const int kMaxStackDepth = 32;
+
+  uintptr_t hash;           // Hash value of the stack trace.
+  int depth;                // Depth of stack trace.
+  const void** stack;       // Stack trace.
+  HeapProfileBucket* next;  // Next entry in hash-table.
+};
+
+#endif  // HEAP_PROFILE_STATS_H_
diff --git a/src/heap-profile-table.cc b/src/heap-profile-table.cc
index 6db2caa..b92251f 100644
--- a/src/heap-profile-table.cc
+++ b/src/heap-profile-table.cc
@@ -99,8 +99,7 @@ const char HeapProfileTable::kFileExt[] = ".heap";
 
 //----------------------------------------------------------------------
 
-// Size for alloc_table_ and mmap_table_.
-static const int kHashTableSize = 179999;
+static const int kHashTableSize = 179999;   // Size for bucket_table_.
 /*static*/ const int HeapProfileTable::kMaxStackDepth;
 
 //----------------------------------------------------------------------
@@ -122,62 +121,50 @@ static bool ByAllocatedSpace(HeapProfileTable::Stats* a,
 
 //----------------------------------------------------------------------
 
-HeapProfileTable::HeapProfileTable(Allocator alloc, DeAllocator dealloc)
-    : alloc_(alloc), dealloc_(dealloc) {
-  // Initialize the overall profile stats.
-  memset(&total_, 0, sizeof(total_));
-
-  // Make the malloc table.
-  const int alloc_table_bytes = kHashTableSize * sizeof(*alloc_table_);
-  alloc_table_ = reinterpret_cast<Bucket**>(alloc_(alloc_table_bytes));
-  memset(alloc_table_, 0, alloc_table_bytes);
-  num_alloc_buckets_ = 0;
-
-  // Initialize the mmap table.
-  mmap_table_ = NULL;
-  num_available_mmap_buckets_ = 0;
-
-  // Make malloc and mmap allocation maps.
-  alloc_address_map_ =
+HeapProfileTable::HeapProfileTable(Allocator alloc,
+                                   DeAllocator dealloc,
+                                   bool profile_mmap)
+    : alloc_(alloc),
+      dealloc_(dealloc),
+      bucket_table_(NULL),
+      profile_mmap_(profile_mmap),
+      num_buckets_(0),
+      address_map_(NULL) {
+  // Make a hash table for buckets.
+  const int table_bytes = kHashTableSize * sizeof(*bucket_table_);
+  bucket_table_ = static_cast<Bucket**>(alloc_(table_bytes));
+  memset(bucket_table_, 0, table_bytes);
+
+  // Make an allocation map.
+  address_map_ =
       new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_);
-  mmap_address_map_ = NULL;
-}
 
-HeapProfileTable::~HeapProfileTable() {
-  DeallocateBucketTable(alloc_table_);
-  alloc_table_ = NULL;
-  DeallocateBucketTable(mmap_table_);
-  mmap_table_ = NULL;
-  DeallocateAllocationMap(alloc_address_map_);
-  alloc_address_map_ = NULL;
-  DeallocateAllocationMap(mmap_address_map_);
-  mmap_address_map_ = NULL;
-}
-
-void HeapProfileTable::DeallocateAllocationMap(AllocationMap* allocation) {
-  if (allocation != NULL) {
-    alloc_address_map_->~AllocationMap();
-    dealloc_(allocation);
-  }
+  // Initialize.
+  memset(&total_, 0, sizeof(total_));
+  num_buckets_ = 0;
 }
 
-void HeapProfileTable::DeallocateBucketTable(Bucket** table) {
-  if (table != NULL) {
-    for (int b = 0; b < kHashTableSize; b++) {
-      for (Bucket* x = table[b]; x != 0; /**/) {
-        Bucket* b = x;
-        x = x->next;
-        dealloc_(b->stack);
-        dealloc_(b);
-      }
+HeapProfileTable::~HeapProfileTable() {
+  // Free the allocation map.
+  address_map_->~AllocationMap();
+  dealloc_(address_map_);
+  address_map_ = NULL;
+
+  // Free the hash table.
+  for (int i = 0; i < kHashTableSize; i++) {
+    for (Bucket* curr = bucket_table_[i]; curr != 0; /**/) {
+      Bucket* bucket = curr;
+      curr = curr->next;
+      dealloc_(bucket->stack);
+      dealloc_(bucket);
     }
-    dealloc_(table);
   }
+  dealloc_(bucket_table_);
+  bucket_table_ = NULL;
 }
 
-HeapProfileTable::Bucket* HeapProfileTable::GetBucket(
-    int depth, const void* const key[], Bucket** table,
-    int* bucket_count) {
+HeapProfileTable::Bucket* HeapProfileTable::GetBucket(int depth,
+                                                      const void* const key[]) {
   // Make hash-value
   uintptr_t h = 0;
   for (int i = 0; i < depth; i++) {
@@ -190,7 +177,7 @@ HeapProfileTable::Bucket* HeapProfileTable::GetBucket(
 
   // Lookup stack trace in table
   unsigned int buck = ((unsigned int) h) % kHashTableSize;
-  for (Bucket* b = table[buck]; b != 0; b = b->next) {
+  for (Bucket* b = bucket_table_[buck]; b != 0; b = b->next) {
     if ((b->hash == h) &&
         (b->depth == depth) &&
         equal(key, key + depth, b->stack)) {
@@ -207,11 +194,9 @@ HeapProfileTable::Bucket* HeapProfileTable::GetBucket(
   b->hash  = h;
   b->depth = depth;
   b->stack = kcopy;
-  b->next  = table[buck];
-  table[buck] = b;
-  if (bucket_count != NULL) {
-    ++(*bucket_count);
-  }
+  b->next  = bucket_table_[buck];
+  bucket_table_[buck] = b;
+  num_buckets_++;
   return b;
 }
 
@@ -224,8 +209,7 @@ int HeapProfileTable::GetCallerStackTrace(
 void HeapProfileTable::RecordAlloc(
     const void* ptr, size_t bytes, int stack_depth,
     const void* const call_stack[]) {
-  Bucket* b = GetBucket(stack_depth, call_stack, alloc_table_,
-                        &num_alloc_buckets_);
+  Bucket* b = GetBucket(stack_depth, call_stack);
   b->allocs++;
   b->alloc_size += bytes;
   total_.allocs++;
@@ -234,12 +218,12 @@ void HeapProfileTable::RecordAlloc(
   AllocValue v;
   v.set_bucket(b);  // also did set_live(false); set_ignore(false)
   v.bytes = bytes;
-  alloc_address_map_->Insert(ptr, v);
+  address_map_->Insert(ptr, v);
 }
 
 void HeapProfileTable::RecordFree(const void* ptr) {
   AllocValue v;
-  if (alloc_address_map_->FindAndRemove(ptr, &v)) {
+  if (address_map_->FindAndRemove(ptr, &v)) {
     Bucket* b = v.bucket();
     b->frees++;
     b->free_size += v.bytes;
@@ -249,14 +233,14 @@ void HeapProfileTable::RecordFree(const void* ptr) {
 }
 
 bool HeapProfileTable::FindAlloc(const void* ptr, size_t* object_size) const {
-  const AllocValue* alloc_value = alloc_address_map_->Find(ptr);
+  const AllocValue* alloc_value = address_map_->Find(ptr);
   if (alloc_value != NULL) *object_size = alloc_value->bytes;
   return alloc_value != NULL;
 }
 
 bool HeapProfileTable::FindAllocDetails(const void* ptr,
                                         AllocInfo* info) const {
-  const AllocValue* alloc_value = alloc_address_map_->Find(ptr);
+  const AllocValue* alloc_value = address_map_->Find(ptr);
   if (alloc_value != NULL) {
     info->object_size = alloc_value->bytes;
     info->call_stack = alloc_value->bucket()->stack;
@@ -270,13 +254,13 @@ bool HeapProfileTable::FindInsideAlloc(const void* ptr,
                                        const void** object_ptr,
                                        size_t* object_size) const {
   const AllocValue* alloc_value =
-    alloc_address_map_->FindInside(&AllocValueSize, max_size, ptr, object_ptr);
+    address_map_->FindInside(&AllocValueSize, max_size, ptr, object_ptr);
   if (alloc_value != NULL) *object_size = alloc_value->bytes;
   return alloc_value != NULL;
 }
 
 bool HeapProfileTable::MarkAsLive(const void* ptr) {
-  AllocValue* alloc = alloc_address_map_->FindMutable(ptr);
+  AllocValue* alloc = address_map_->FindMutable(ptr);
   if (alloc && !alloc->live()) {
     alloc->set_live(true);
     return true;
@@ -285,7 +269,7 @@ bool HeapProfileTable::MarkAsLive(const void* ptr) {
 }
 
 void HeapProfileTable::MarkAsIgnored(const void* ptr) {
-  AllocValue* alloc = alloc_address_map_->FindMutable(ptr);
+  AllocValue* alloc = address_map_->FindMutable(ptr);
   if (alloc) {
     alloc->set_ignore(true);
   }
@@ -326,81 +310,26 @@ int HeapProfileTable::UnparseBucket(const Bucket& b,
 
 HeapProfileTable::Bucket**
 HeapProfileTable::MakeSortedBucketList() const {
-  Bucket** list = reinterpret_cast<Bucket**>(alloc_(sizeof(Bucket) *
-      (num_alloc_buckets_ + num_available_mmap_buckets_)));
-
-  RAW_DCHECK(mmap_table_ != NULL || num_available_mmap_buckets_ == 0, "");
-
-  int n = 0;
+  Bucket** list = static_cast<Bucket**>(alloc_(sizeof(Bucket) * num_buckets_));
 
-  for (int b = 0; b < kHashTableSize; b++) {
-    for (Bucket* x = alloc_table_[b]; x != 0; x = x->next) {
-      list[n++] = x;
+  int bucket_count = 0;
+  for (int i = 0; i < kHashTableSize; i++) {
+    for (Bucket* curr = bucket_table_[i]; curr != 0; curr = curr->next) {
+      list[bucket_count++] = curr;
     }
   }
-  RAW_DCHECK(n == num_alloc_buckets_, "");
+  RAW_DCHECK(bucket_count == num_buckets_, "");
 
-  if (mmap_table_ != NULL) {
-    for (int b = 0; b < kHashTableSize; b++) {
-      for (Bucket* x = mmap_table_[b]; x != 0; x = x->next) {
-        list[n++] = x;
-      }
-    }
-  }
-  RAW_DCHECK(n == num_alloc_buckets_ + num_available_mmap_buckets_, "");
-
-  sort(list, list + num_alloc_buckets_ + num_available_mmap_buckets_,
-       ByAllocatedSpace);
+  sort(list, list + num_buckets_, ByAllocatedSpace);
 
   return list;
 }
 
-void HeapProfileTable::RefreshMMapData() {
-  // Make the table
-  static const int mmap_table_bytes = kHashTableSize * sizeof(*mmap_table_);
-  if (mmap_table_ == NULL) {
-    mmap_table_ = reinterpret_cast<Bucket**>(alloc_(mmap_table_bytes));
-    memset(mmap_table_, 0, mmap_table_bytes);
-  }
-  num_available_mmap_buckets_ = 0;
-
-  ClearMMapData();
-  mmap_address_map_ =
-      new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_);
-
-  MemoryRegionMap::LockHolder l;
-  for (MemoryRegionMap::RegionIterator r =
-           MemoryRegionMap::BeginRegionLocked();
-       r != MemoryRegionMap::EndRegionLocked(); ++r) {
-    Bucket* b =
-        GetBucket(r->call_stack_depth, r->call_stack, mmap_table_, NULL);
-    if (b->alloc_size == 0) {
-      num_available_mmap_buckets_ += 1;
-    }
-    b->allocs += 1;
-    b->alloc_size += r->end_addr - r->start_addr;
-
-    AllocValue v;
-    v.set_bucket(b);
-    v.bytes = r->end_addr - r->start_addr;
-    mmap_address_map_->Insert(reinterpret_cast<const void*>(r->start_addr), v);
-  }
-}
-
-void HeapProfileTable::ClearMMapData() {
-  if (mmap_address_map_ != NULL) {
-    mmap_address_map_->Iterate(ZeroBucketCountsIterator, this);
-    mmap_address_map_->~AllocationMap();
-    dealloc_(mmap_address_map_);
-    mmap_address_map_ = NULL;
-  }
-}
-
 void HeapProfileTable::IterateOrderedAllocContexts(
     AllocContextIterator callback) const {
   Bucket** list = MakeSortedBucketList();
   AllocContextInfo info;
-  for (int i = 0; i < num_alloc_buckets_; ++i) {
+  for (int i = 0; i < num_buckets_; ++i) {
     *static_cast<Stats*>(&info) = *static_cast<Stats*>(list[i]);
     info.stack_depth = list[i]->depth;
     info.call_stack = list[i]->stack;
@@ -432,14 +361,17 @@ int HeapProfileTable::FillOrderedProfile(char buf[], int size) const {
   memset(&stats, 0, sizeof(stats));
   int bucket_length = snprintf(buf, size, "%s", kProfileHeader);
   if (bucket_length < 0 || bucket_length >= size) return 0;
-  Bucket total_with_mmap(total_);
-  if (mmap_table_ != NULL) {
-    total_with_mmap.alloc_size += MemoryRegionMap::MapSize();
-    total_with_mmap.free_size += MemoryRegionMap::UnmapSize();
-  }
-  bucket_length = UnparseBucket(total_with_mmap, buf, bucket_length, size,
+  bucket_length = UnparseBucket(total_, buf, bucket_length, size,
                                 " heapprofile", &stats);
-  for (int i = 0; i < num_alloc_buckets_; i++) {
+
+  // Dump the mmap list first.
+  if (profile_mmap_) {
+    BufferArgs buffer(buf, bucket_length, size);
+    MemoryRegionMap::IterateBuckets<BufferArgs*>(DumpBucketIterator, &buffer);
+    bucket_length = buffer.buflen;
+  }
+
+  for (int i = 0; i < num_buckets_; i++) {
     bucket_length = UnparseBucket(*list[i], buf, bucket_length, size, "",
                                   &stats);
   }
@@ -453,6 +385,13 @@ int HeapProfileTable::FillOrderedProfile(char buf[], int size) const {
   return bucket_length + map_length;
 }
 
+// static
+void HeapProfileTable::DumpBucketIterator(const Bucket* bucket,
+                                          BufferArgs* args) {
+  args->buflen = UnparseBucket(*bucket, args->buf, args->buflen, args->bufsize,
+                               "", NULL);
+}
+
 inline
 void HeapProfileTable::DumpNonLiveIterator(const void* ptr, AllocValue* v,
                                            const DumpArgs& args) {
@@ -474,17 +413,6 @@ void HeapProfileTable::DumpNonLiveIterator(const void* ptr, AllocValue* v,
   RawWrite(args.fd, buf, len);
 }
 
-inline void HeapProfileTable::ZeroBucketCountsIterator(
-    const void* ptr, AllocValue* v, HeapProfileTable* heap_profile) {
-  Bucket* b = v->bucket();
-  if (b != NULL) {
-    b->allocs = 0;
-    b->alloc_size = 0;
-    b->free_size = 0;
-    b->frees = 0;
-  }
-}
-
 // Callback from NonLiveSnapshot; adds entry to arg->dest
 // if not the entry is not live and is not present in arg->base.
 void HeapProfileTable::AddIfNonLive(const void* ptr, AllocValue* v,
@@ -549,7 +477,7 @@ void HeapProfileTable::CleanupOldProfiles(const char* prefix) {
 
 HeapProfileTable::Snapshot* HeapProfileTable::TakeSnapshot() {
   Snapshot* s = new (alloc_(sizeof(Snapshot))) Snapshot(alloc_, dealloc_);
-  alloc_address_map_->Iterate(AddToSnapshot, s);
+  address_map_->Iterate(AddToSnapshot, s);
   return s;
 }
 
@@ -574,7 +502,7 @@ HeapProfileTable::Snapshot* HeapProfileTable::NonLiveSnapshot(
   AddNonLiveArgs args;
   args.dest = s;
   args.base = base;
-  alloc_address_map_->Iterate<AddNonLiveArgs*>(AddIfNonLive, &args);
+  address_map_->Iterate<AddNonLiveArgs*>(AddIfNonLive, &args);
   RAW_VLOG(2, "NonLiveSnapshot output: %d %d\n",
            int(s->total_.allocs - s->total_.frees),
            int(s->total_.alloc_size - s->total_.free_size));
diff --git a/src/heap-profile-table.h b/src/heap-profile-table.h
index abd3184..78f1e62 100644
--- a/src/heap-profile-table.h
+++ b/src/heap-profile-table.h
@@ -38,6 +38,7 @@
 #include "addressmap-inl.h"
 #include "base/basictypes.h"
 #include "base/logging.h"   // for RawFD
+#include "heap-profile-stats.h"
 
 // Table to maintain a heap profile data inside,
 // i.e. the set of currently active heap memory allocations.
@@ -58,18 +59,7 @@ class HeapProfileTable {
   // data types ----------------------------
 
   // Profile stats.
-  struct Stats {
-    int32 allocs;      // Number of allocation calls
-    int32 frees;       // Number of free calls
-    int64 alloc_size;  // Total size of all allocated objects so far
-    int64 free_size;   // Total size of all freed objects so far
-
-    // semantic equality
-    bool Equivalent(const Stats& x) const {
-      return allocs - frees == x.allocs - x.frees  &&
-             alloc_size - free_size == x.alloc_size - x.free_size;
-    }
-  };
+  typedef HeapProfileStats Stats;
 
   // Info we can return about an allocation.
   struct AllocInfo {
@@ -94,7 +84,7 @@ class HeapProfileTable {
 
   // interface ---------------------------
 
-  HeapProfileTable(Allocator alloc, DeAllocator dealloc);
+  HeapProfileTable(Allocator alloc, DeAllocator dealloc, bool profile_mmap);
   ~HeapProfileTable();
 
   // Collect the stack trace for the function that asked to do the
@@ -149,7 +139,7 @@ class HeapProfileTable {
   // Iterate over the allocation profile data calling "callback"
   // for every allocation.
   void IterateAllocs(AllocIterator callback) const {
-    alloc_address_map_->Iterate(MapArgsAllocIterator, callback);
+    address_map_->Iterate(MapArgsAllocIterator, callback);
   }
 
   // Allocation context profile data iteration callback
@@ -187,28 +177,13 @@ class HeapProfileTable {
   // Caller must call ReleaseSnapshot() on result when no longer needed.
   Snapshot* NonLiveSnapshot(Snapshot* base);
 
-  // Refresh the internal mmap information from MemoryRegionMap.  Results of
-  // FillOrderedProfile and IterateOrderedAllocContexts will contain mmap'ed
-  // memory regions as at calling RefreshMMapData.
-  void RefreshMMapData();
-
-  // Clear the internal mmap information.  Results of FillOrderedProfile and
-  // IterateOrderedAllocContexts won't contain mmap'ed memory regions after
-  // calling ClearMMapData.
-  void ClearMMapData();
-
  private:
 
   // data types ----------------------------
 
   // Hash table bucket to hold (de)allocation stats
   // for a given allocation call stack trace.
-  struct Bucket : public Stats {
-    uintptr_t    hash;   // Hash value of the stack trace
-    int          depth;  // Depth of stack trace
-    const void** stack;  // Stack trace
-    Bucket*      next;   // Next entry in hash-table
-  };
+  typedef HeapProfileBucket Bucket;
 
   // Info stored in the address map
   struct AllocValue {
@@ -247,13 +222,30 @@ class HeapProfileTable {
 
   typedef AddressMap<AllocValue> AllocationMap;
 
+  // Arguments that need to be passed DumpBucketIterator callback below.
+  struct BufferArgs {
+    BufferArgs(char* buf_arg, int buflen_arg, int bufsize_arg)
+        : buf(buf_arg),
+          buflen(buflen_arg),
+          bufsize(bufsize_arg) {
+    }
+
+    char* buf;
+    int buflen;
+    int bufsize;
+
+    DISALLOW_COPY_AND_ASSIGN(BufferArgs);
+  };
+
   // Arguments that need to be passed DumpNonLiveIterator callback below.
   struct DumpArgs {
+    DumpArgs(RawFD fd_arg, Stats* profile_stats_arg)
+        : fd(fd_arg),
+          profile_stats(profile_stats_arg) {
+    }
+
     RawFD fd;  // file to write to
     Stats* profile_stats;  // stats to update (may be NULL)
-
-    DumpArgs(RawFD a, Stats* d)
-      : fd(a), profile_stats(d) { }
   };
 
   // helpers ----------------------------
@@ -274,18 +266,9 @@ class HeapProfileTable {
                            const char* extra,
                            Stats* profile_stats);
 
-  // Deallocate a given allocation map.
-  void DeallocateAllocationMap(AllocationMap* allocation);
-
-  // Deallocate a given bucket table.
-  void DeallocateBucketTable(Bucket** table);
-
-  // Get the bucket for the caller stack trace 'key' of depth 'depth' from a
-  // bucket hash map 'table' creating the bucket if needed.  '*bucket_count'
-  // is incremented both when 'bucket_count' is not NULL and when a new
-  // bucket object is created.
-  Bucket* GetBucket(int depth, const void* const key[], Bucket** table,
-                    int* bucket_count);
+  // Get the bucket for the caller stack trace 'key' of depth 'depth'
+  // creating the bucket if needed.
+  Bucket* GetBucket(int depth, const void* const key[]);
 
   // Helper for IterateAllocs to do callback signature conversion
   // from AllocationMap::Iterate to AllocIterator.
@@ -300,18 +283,17 @@ class HeapProfileTable {
     callback(ptr, info);
   }
 
+  // Helper to dump a bucket.
+  inline static void DumpBucketIterator(const Bucket* bucket,
+                                        BufferArgs* args);
+
   // Helper for DumpNonLiveProfile to do object-granularity
   // heap profile dumping. It gets passed to AllocationMap::Iterate.
   inline static void DumpNonLiveIterator(const void* ptr, AllocValue* v,
                                          const DumpArgs& args);
 
-  // Helper for filling size variables in buckets by zero.
-  inline static void ZeroBucketCountsIterator(
-      const void* ptr, AllocValue* v, HeapProfileTable* heap_profile);
-
   // Helper for IterateOrderedAllocContexts and FillOrderedProfile.
-  // Creates a sorted list of Buckets whose length is num_alloc_buckets_ +
-  // num_avaliable_mmap_buckets_.
+  // Creates a sorted list of Buckets whose length is num_buckets_.
   // The caller is responsible for deallocating the returned list.
   Bucket** MakeSortedBucketList() const;
 
@@ -344,25 +326,19 @@ class HeapProfileTable {
 
   // Overall profile stats; we use only the Stats part,
   // but make it a Bucket to pass to UnparseBucket.
-  // It doesn't contain mmap'ed regions.
   Bucket total_;
 
+  bool profile_mmap_;
+
   // Bucket hash table for malloc.
   // We hand-craft one instead of using one of the pre-written
   // ones because we do not want to use malloc when operating on the table.
   // It is only few lines of code, so no big deal.
-  Bucket** alloc_table_;
-  int num_alloc_buckets_;
-
-  // Bucket hash table for mmap.
-  // This table is filled with the information from MemoryRegionMap by calling
-  // RefreshMMapData.
-  Bucket** mmap_table_;
-  int num_available_mmap_buckets_;
+  Bucket** bucket_table_;
+  int num_buckets_;
 
   // Map of all currently allocated objects and mapped regions we know about.
-  AllocationMap* alloc_address_map_;
-  AllocationMap* mmap_address_map_;
+  AllocationMap* address_map_;
 
   DISALLOW_COPY_AND_ASSIGN(HeapProfileTable);
 };
diff --git a/src/heap-profiler.cc b/src/heap-profiler.cc
index f1f2c21..aa95a51 100644
--- a/src/heap-profiler.cc
+++ b/src/heap-profiler.cc
@@ -190,13 +190,14 @@ static char* DoGetHeapProfileLocked(char* buf, int buflen) {
   RAW_DCHECK(heap_lock.IsHeld(), "");
   int bytes_written = 0;
   if (is_on) {
-    if (FLAGS_mmap_profile) {
-      heap_profile->RefreshMMapData();
-    }
+    HeapProfileTable::Stats const stats = heap_profile->total();
+    (void)stats;   // avoid an unused-variable warning in non-debug mode.
     bytes_written = heap_profile->FillOrderedProfile(buf, buflen - 1);
-    if (FLAGS_mmap_profile) {
-      heap_profile->ClearMMapData();
-    }
+    // FillOrderedProfile should not reduce the set of active mmap-ed regions,
+    // hence MemoryRegionMap will let us remove everything we've added above:
+    RAW_DCHECK(stats.Equivalent(heap_profile->total()), "");
+    // if this fails, we somehow removed by FillOrderedProfile
+    // more than we have added.
   }
   buf[bytes_written] = '\0';
   RAW_DCHECK(bytes_written == strlen(buf), "");
@@ -435,7 +436,8 @@ extern "C" void HeapProfilerStart(const char* prefix) {
   if (FLAGS_mmap_profile) {
     // Ask MemoryRegionMap to record all mmap, mremap, and sbrk
     // call stack traces of at least size kMaxStackDepth:
-    MemoryRegionMap::Init(HeapProfileTable::kMaxStackDepth);
+    MemoryRegionMap::Init(HeapProfileTable::kMaxStackDepth,
+                          /* use_buckets */ true);
   }
 
   if (FLAGS_mmap_log) {
@@ -455,7 +457,7 @@ extern "C" void HeapProfilerStart(const char* prefix) {
       reinterpret_cast<char*>(ProfilerMalloc(kProfileBufferSize));
 
   heap_profile = new(ProfilerMalloc(sizeof(HeapProfileTable)))
-                   HeapProfileTable(ProfilerMalloc, ProfilerFree);
+      HeapProfileTable(ProfilerMalloc, ProfilerFree, FLAGS_mmap_profile);
 
   last_dump_alloc = 0;
   last_dump_free = 0;
diff --git a/src/memory_region_map.cc b/src/memory_region_map.cc
index 0ce97c2..899b08c 100644
--- a/src/memory_region_map.cc
+++ b/src/memory_region_map.cc
@@ -84,9 +84,10 @@
 // which (sometimes) causes mmap, which calls our hook, and so on.
 // We do this as follows: on a recursive call of MemoryRegionMap's
 // mmap/sbrk/mremap hook we record the data about the allocation in a
-// static fixed-sized stack (saved_regions), when the recursion unwinds
-// but before returning from the outer hook call we unwind this stack and
-// move the data from saved_regions to its permanent place in the RegionSet,
+// static fixed-sized stack (saved_regions and saved_buckets), when the
+// recursion unwinds but before returning from the outer hook call we unwind
+// this stack and move the data from saved_regions and saved_buckets to its
+// permanent place in the RegionSet and "bucket_table" respectively,
 // which can cause more allocations and mmap-s and recursion and unwinding,
 // but the whole process ends eventually due to the fact that for the small
 // allocations we are doing LowLevelAlloc reuses one mmap call and parcels out
@@ -147,6 +148,13 @@ int MemoryRegionMap::recursion_count_ = 0;  // GUARDED_BY(owner_lock_)
 pthread_t MemoryRegionMap::lock_owner_tid_;  // GUARDED_BY(owner_lock_)
 int64 MemoryRegionMap::map_size_ = 0;
 int64 MemoryRegionMap::unmap_size_ = 0;
+HeapProfileBucket** MemoryRegionMap::bucket_table_ = NULL;  // GUARDED_BY(lock_)
+int MemoryRegionMap::num_buckets_ = 0;  // GUARDED_BY(lock_)
+int MemoryRegionMap::saved_buckets_count_ = 0;  // GUARDED_BY(lock_)
+HeapProfileBucket MemoryRegionMap::saved_buckets_[20];  // GUARDED_BY(lock_)
+
+// GUARDED_BY(lock_)
+const void* MemoryRegionMap::saved_buckets_keys_[20][kMaxStackDepth];
 
 // ========================================================================= //
 
@@ -182,7 +190,7 @@ static MemoryRegionMap::RegionSetRep regions_rep;
 // (or rather should we *not* use regions_ to record a hooked mmap).
 static bool recursive_insert = false;
 
-void MemoryRegionMap::Init(int max_stack_depth) {
+void MemoryRegionMap::Init(int max_stack_depth, bool use_buckets) {
   RAW_VLOG(10, "MemoryRegionMap Init");
   RAW_CHECK(max_stack_depth >= 0, "");
   // Make sure we don't overflow the memory in region stacks:
@@ -214,6 +222,15 @@ void MemoryRegionMap::Init(int max_stack_depth) {
     // Can't instead use HandleSavedRegionsLocked(&DoInsertRegionLocked) before
     // recursive_insert = false; as InsertRegionLocked will also construct
     // regions_ on demand for us.
+  if (use_buckets) {
+    const int table_bytes = kHashTableSize * sizeof(*bucket_table_);
+    recursive_insert = true;
+    bucket_table_ = static_cast<HeapProfileBucket**>(
+        MyAllocator::Allocate(table_bytes));
+    recursive_insert = false;
+    memset(bucket_table_, 0, table_bytes);
+    num_buckets_ = 0;
+  }
   Unlock();
   RAW_VLOG(10, "MemoryRegionMap Init done");
 }
@@ -228,6 +245,19 @@ bool MemoryRegionMap::Shutdown() {
     RAW_VLOG(10, "MemoryRegionMap Shutdown decrement done");
     return true;
   }
+  if (bucket_table_ != NULL) {
+    for (int i = 0; i < kHashTableSize; i++) {
+      for (HeapProfileBucket* curr = bucket_table_[i]; curr != 0; /**/) {
+        HeapProfileBucket* bucket = curr;
+        curr = curr->next;
+        MyAllocator::Free(bucket->stack, 0);
+        MyAllocator::Free(bucket, 0);
+      }
+    }
+    MyAllocator::Free(bucket_table_, 0);
+    num_buckets_ = 0;
+    bucket_table_ = NULL;
+  }
   RAW_CHECK(MallocHook::RemoveMmapHook(&MmapHook), "");
   RAW_CHECK(MallocHook::RemoveMremapHook(&MremapHook), "");
   RAW_CHECK(MallocHook::RemoveSbrkHook(&SbrkHook), "");
@@ -245,6 +275,11 @@ bool MemoryRegionMap::Shutdown() {
   return deleted_arena;
 }
 
+bool MemoryRegionMap::IsRecordingLocked() {
+  RAW_CHECK(LockIsHeld(), "should be held (by this thread)");
+  return client_count_ > 0;
+}
+
 // Invariants (once libpthread_initialized is true):
 //   * While lock_ is not held, recursion_count_ is 0 (and
 //     lock_owner_tid_ is the previous owner, but we don't rely on
@@ -336,6 +371,62 @@ bool MemoryRegionMap::FindAndMarkStackRegion(uintptr_t stack_top,
   return region != NULL;
 }
 
+HeapProfileBucket* MemoryRegionMap::GetBucket(int depth,
+                                              const void* const key[]) {
+  RAW_CHECK(LockIsHeld(), "should be held (by this thread)");
+  // Make hash-value
+  uintptr_t hash = 0;
+  for (int i = 0; i < depth; i++) {
+    hash += reinterpret_cast<uintptr_t>(key[i]);
+    hash += hash << 10;
+    hash ^= hash >> 6;
+  }
+  hash += hash << 3;
+  hash ^= hash >> 11;
+
+  // Lookup stack trace in table
+  unsigned int hash_index = (static_cast<unsigned int>(hash)) % kHashTableSize;
+  for (HeapProfileBucket* bucket = bucket_table_[hash_index];
+       bucket != 0;
+       bucket = bucket->next) {
+    if ((bucket->hash == hash) && (bucket->depth == depth) &&
+        std::equal(key, key + depth, bucket->stack)) {
+      return bucket;
+    }
+  }
+
+  // Create new bucket
+  const size_t key_size = sizeof(key[0]) * depth;
+  HeapProfileBucket* bucket;
+  if (recursive_insert) {  // recursion: save in saved_buckets_
+    const void** key_copy = saved_buckets_keys_[saved_buckets_count_];
+    std::copy(key, key + depth, key_copy);
+    bucket = &saved_buckets_[saved_buckets_count_];
+    memset(bucket, 0, sizeof(*bucket));
+    ++saved_buckets_count_;
+    bucket->stack = key_copy;
+    bucket->next  = NULL;
+  } else {
+    recursive_insert = true;
+    const void** key_copy = static_cast<const void**>(
+        MyAllocator::Allocate(key_size));
+    recursive_insert = false;
+    std::copy(key, key + depth, key_copy);
+    recursive_insert = true;
+    bucket = static_cast<HeapProfileBucket*>(
+        MyAllocator::Allocate(sizeof(HeapProfileBucket)));
+    recursive_insert = false;
+    memset(bucket, 0, sizeof(*bucket));
+    bucket->stack = key_copy;
+    bucket->next  = bucket_table_[hash_index];
+  }
+  bucket->hash = hash;
+  bucket->depth = depth;
+  bucket_table_[hash_index] = bucket;
+  ++num_buckets_;
+  return bucket;
+}
+
 MemoryRegionMap::RegionIterator MemoryRegionMap::BeginRegionLocked() {
   RAW_CHECK(LockIsHeld(), "should be held (by this thread)");
   RAW_CHECK(regions_ != NULL, "");
@@ -404,6 +495,44 @@ inline void MemoryRegionMap::HandleSavedRegionsLocked(
   }
 }
 
+void MemoryRegionMap::RestoreSavedBucketsLocked() {
+  RAW_CHECK(LockIsHeld(), "should be held (by this thread)");
+  while (saved_buckets_count_ > 0) {
+    HeapProfileBucket bucket = saved_buckets_[--saved_buckets_count_];
+    unsigned int hash_index =
+        static_cast<unsigned int>(bucket.hash) % kHashTableSize;
+    bool is_found = false;
+    for (HeapProfileBucket* curr = bucket_table_[hash_index];
+         curr != 0;
+         curr = curr->next) {
+      if ((curr->hash == bucket.hash) && (curr->depth == bucket.depth) &&
+          std::equal(bucket.stack, bucket.stack + bucket.depth, curr->stack)) {
+        curr->allocs += bucket.allocs;
+        curr->alloc_size += bucket.alloc_size;
+        curr->frees += bucket.frees;
+        curr->free_size += bucket.free_size;
+        is_found = true;
+        break;
+      }
+    }
+    if (is_found) continue;
+
+    const size_t key_size = sizeof(bucket.stack[0]) * bucket.depth;
+    const void** key_copy = static_cast<const void**>(
+        MyAllocator::Allocate(key_size));
+    std::copy(bucket.stack, bucket.stack + bucket.depth, key_copy);
+    HeapProfileBucket* new_bucket = static_cast<HeapProfileBucket*>(
+        MyAllocator::Allocate(sizeof(HeapProfileBucket)));
+    memset(new_bucket, 0, sizeof(*new_bucket));
+    new_bucket->hash = bucket.hash;
+    new_bucket->depth = bucket.depth;
+    new_bucket->stack = key_copy;
+    new_bucket->next = bucket_table_[hash_index];
+    bucket_table_[hash_index] = new_bucket;
+    ++num_buckets_;
+  }
+}
+
 inline void MemoryRegionMap::InsertRegionLocked(const Region& region) {
   RAW_CHECK(LockIsHeld(), "should be held (by this thread)");
   // We can be called recursively, because RegionSet constructor
@@ -468,6 +597,16 @@ void MemoryRegionMap::RecordRegionAddition(const void* start, size_t size) {
   InsertRegionLocked(region);
     // This will (eventually) allocate storage for and copy over the stack data
     // from region.call_stack_data_ that is pointed by region.call_stack().
+  if (bucket_table_ != NULL) {
+    HeapProfileBucket* b = GetBucket(depth, region.call_stack);
+    ++b->allocs;
+    b->alloc_size += size;
+    if (!recursive_insert) {
+      recursive_insert = true;
+      RestoreSavedBucketsLocked();
+      recursive_insert = false;
+    }
+  }
   Unlock();
 }
 
@@ -486,6 +625,7 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) {
       Region& r = saved_regions[i];
       if (r.start_addr == start_addr && r.end_addr == end_addr) {
         // An exact match, so it's safe to remove.
+        RecordRegionRemovalInBucket(r.call_stack_depth, r.call_stack, size);
         --saved_regions_count;
         --put_pos;
         RAW_VLOG(10, ("Insta-Removing saved region %p..%p; "
@@ -530,6 +670,8 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) {
       RAW_VLOG(12, "Deleting region %p..%p",
                   reinterpret_cast<void*>(region->start_addr),
                   reinterpret_cast<void*>(region->end_addr));
+      RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack,
+                                  region->end_addr - region->start_addr);
       RegionSet::iterator d = region;
       ++region;
       regions_->erase(d);
@@ -539,6 +681,8 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) {
       RAW_VLOG(12, "Splitting region %p..%p in two",
                   reinterpret_cast<void*>(region->start_addr),
                   reinterpret_cast<void*>(region->end_addr));
+      RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack,
+                                  end_addr - start_addr);
       // Make another region for the start portion:
       // The new region has to be the start portion because we can't
       // just modify region->end_addr as it's the sorting key.
@@ -552,12 +696,16 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) {
       RAW_VLOG(12, "Start-chopping region %p..%p",
                   reinterpret_cast<void*>(region->start_addr),
                   reinterpret_cast<void*>(region->end_addr));
+      RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack,
+                                  end_addr - region->start_addr);
       const_cast<Region&>(*region).set_start_addr(end_addr);
     } else if (start_addr > region->start_addr  &&
                start_addr < region->end_addr) {  // cut from end
       RAW_VLOG(12, "End-chopping region %p..%p",
                   reinterpret_cast<void*>(region->start_addr),
                   reinterpret_cast<void*>(region->end_addr));
+      RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack,
+                                  region->end_addr - start_addr);
       // Can't just modify region->end_addr (it's the sorting key):
       Region r = *region;
       r.set_end_addr(start_addr);
@@ -580,6 +728,16 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) {
   Unlock();
 }
 
+void MemoryRegionMap::RecordRegionRemovalInBucket(int depth,
+                                                  const void* const stack[],
+                                                  size_t size) {
+  RAW_CHECK(LockIsHeld(), "should be held (by this thread)");
+  if (bucket_table_ == NULL) return;
+  HeapProfileBucket* b = GetBucket(depth, stack);
+  ++b->frees;
+  b->free_size += size;
+}
+
 void MemoryRegionMap::MmapHook(const void* result,
                                const void* start, size_t size,
                                int prot, int flags,
diff --git a/src/memory_region_map.h b/src/memory_region_map.h
index 988ea70..be191a9 100644
--- a/src/memory_region_map.h
+++ b/src/memory_region_map.h
@@ -45,6 +45,7 @@
 #include "base/spinlock.h"
 #include "base/thread_annotations.h"
 #include "base/low_level_alloc.h"
+#include "heap-profile-stats.h"
 
 // TODO(maxim): add a unittest:
 //  execute a bunch of mmaps and compare memory map what strace logs
@@ -72,6 +73,10 @@ class MemoryRegionMap {
   // don't take the address of it!
   static const int kMaxStackDepth = 32;
 
+  // Size of the hash table of buckets.  A structure of the bucket table is
+  // described in heap-profile-stats.h.
+  static const int kHashTableSize = 179999;
+
  public:
   // interface ================================================================
 
@@ -87,11 +92,14 @@ class MemoryRegionMap {
   // are automatically shrunk to "max_stack_depth" when they are recorded.
   // Init() can be called more than once w/o harm, largest max_stack_depth
   // will be the effective one.
+  // When "use_buckets" is true, then counts of mmap and munmap sizes will be
+  // recorded with each stack trace.  If Init() is called more than once, then
+  // counting will be effective after any call contained "use_buckets" of true.
   // It will install mmap, munmap, mremap, sbrk hooks
   // and initialize arena_ and our hook and locks, hence one can use
   // MemoryRegionMap::Lock()/Unlock() to manage the locks.
   // Uses Lock/Unlock inside.
-  static void Init(int max_stack_depth);
+  static void Init(int max_stack_depth, bool use_buckets);
 
   // Try to shutdown this module undoing what Init() did.
   // Returns true iff could do full shutdown (or it was not attempted).
@@ -99,6 +107,10 @@ class MemoryRegionMap {
   // the number of Init() calls.
   static bool Shutdown();
 
+  // Return true if MemoryRegionMap is initialized and recording, i.e. when
+  // then number of Init() calls are more than the number of Shutdown() calls.
+  static bool IsRecordingLocked();
+
   // Locks to protect our internal data structures.
   // These also protect use of arena_ if our Init() has been done.
   // The lock is recursive.
@@ -214,6 +226,18 @@ class MemoryRegionMap {
   // Returns success. Uses Lock/Unlock inside.
   static bool FindAndMarkStackRegion(uintptr_t stack_top, Region* result);
 
+  // Iterate over the buckets which store mmap and munmap counts per stack
+  // trace.  It calls "callback" for each bucket, and passes "arg" to it.
+  template<class Type>
+  static void IterateBuckets(void (*callback)(const HeapProfileBucket*, Type),
+                             Type arg);
+
+  // Get the bucket whose caller stack trace is "key".  The stack trace is
+  // used to a depth of "depth" at most.  The requested bucket is created if
+  // needed.
+  // The bucket table is described in heap-profile-stats.h.
+  static HeapProfileBucket* GetBucket(int depth, const void* const key[]);
+
  private:  // our internal types ==============================================
 
   // Region comparator for sorting with STL
@@ -280,7 +304,7 @@ class MemoryRegionMap {
   // simply by acquiring our recursive Lock() before that.
   static RegionSet* regions_;
 
-  // Lock to protect regions_ variable and the data behind.
+  // Lock to protect regions_ and buckets_ variables and the data behind.
   static SpinLock lock_;
   // Lock to protect the recursive lock itself.
   static SpinLock owner_lock_;
@@ -295,6 +319,30 @@ class MemoryRegionMap {
   // Total size of all unmapped pages so far
   static int64 unmap_size_;
 
+  // Bucket hash table which is described in heap-profile-stats.h.
+  static HeapProfileBucket** bucket_table_ GUARDED_BY(lock_);
+  static int num_buckets_ GUARDED_BY(lock_);
+
+  // The following members are local to MemoryRegionMap::GetBucket()
+  // and MemoryRegionMap::HandleSavedBucketsLocked()
+  // and are file-level to ensure that they are initialized at load time.
+  //
+  // These are used as temporary storage to break the infinite cycle of mmap
+  // calling our hook which (sometimes) causes mmap.  It must be a static
+  // fixed-size array.  The size 20 is just an expected value for safety.
+  // The details are described in memory_region_map.cc.
+
+  // Number of unprocessed bucket inserts.
+  static int saved_buckets_count_ GUARDED_BY(lock_);
+
+  // Unprocessed inserts (must be big enough to hold all mmaps that can be
+  // caused by a GetBucket call).
+  // Bucket has no constructor, so that c-tor execution does not interfere
+  // with the any-time use of the static memory behind saved_buckets.
+  static HeapProfileBucket saved_buckets_[20] GUARDED_BY(lock_);
+
+  static const void* saved_buckets_keys_[20][kMaxStackDepth] GUARDED_BY(lock_);
+
   // helpers ==================================================================
 
   // Helper for FindRegion and FindAndMarkStackRegion:
@@ -308,6 +356,11 @@ class MemoryRegionMap {
   // by calling insert_func on them.
   inline static void HandleSavedRegionsLocked(
                        void (*insert_func)(const Region& region));
+
+  // Restore buckets saved in a tmp static array by GetBucket to the bucket
+  // table where all buckets eventually should be.
+  static void RestoreSavedBucketsLocked();
+
   // Wrapper around DoInsertRegionLocked
   // that handles the case of recursive allocator calls.
   inline static void InsertRegionLocked(const Region& region);
@@ -319,6 +372,13 @@ class MemoryRegionMap {
   // (called from our munmap/mremap/sbrk hooks).
   static void RecordRegionRemoval(const void* start, size_t size);
 
+  // Record deletion of a memory region of size "size" in a bucket whose
+  // caller stack trace is "key".  The stack trace is used to a depth of
+  // "depth" at most.
+  static void RecordRegionRemovalInBucket(int depth,
+                                          const void* const key[],
+                                          size_t size);
+
   // Hooks for MallocHook
   static void MmapHook(const void* result,
                        const void* start, size_t size,
@@ -337,4 +397,16 @@ class MemoryRegionMap {
   DISALLOW_COPY_AND_ASSIGN(MemoryRegionMap);
 };
 
+template <class Type>
+void MemoryRegionMap::IterateBuckets(
+    void (*callback)(const HeapProfileBucket*, Type), Type callback_arg) {
+  for (int index = 0; index < kHashTableSize; index++) {
+    for (HeapProfileBucket* bucket = bucket_table_[index];
+         bucket != NULL;
+         bucket = bucket->next) {
+      callback(bucket, callback_arg);
+    }
+  }
+}
+
 #endif  // BASE_MEMORY_REGION_MAP_H_