implemented sized free support via tc_free_sized

6 files changed, 66 insertions, 27 deletions

diff --git a/src/debugallocation.cc b/src/debugallocation.cc
index c170bc7..5d3a2e6 100644
--- a/src/debugallocation.cc
+++ b/src/debugallocation.cc
@@ -405,7 +405,7 @@ class MallocBlock {
     }
   }
 
-  size_t CheckAndClear(int type) {
+  size_t CheckAndClear(int type, size_t given_size) {
     alloc_map_lock_.Lock();
     CheckLocked(type);
     if (!IsMMapped()) {
@@ -416,6 +416,8 @@ class MallocBlock {
     alloc_map_lock_.Unlock();
     // clear us
     const size_t size = real_size();
+    RAW_CHECK(!given_size || given_size == size1_,
+              "right size must be passed to sized delete");
     memset(this, kMagicDeletedByte, size);
     return size;
   }
@@ -543,10 +545,10 @@ class MallocBlock {
     return b;
   }
 
-  void Deallocate(int type) {
+  void Deallocate(int type, size_t given_size) {
     if (IsMMapped()) {  // have to do this before CheckAndClear
 #ifdef HAVE_MMAP
-      int size = CheckAndClear(type);
+      int size = CheckAndClear(type, given_size);
       int pagesize = getpagesize();
       int num_pages = (size + pagesize - 1) / pagesize + 1;
       char* p = (char*) this;
@@ -559,7 +561,7 @@ class MallocBlock {
       }
 #endif
     } else {
-      const size_t size = CheckAndClear(type);
+      const size_t size = CheckAndClear(type, given_size);
       if (FLAGS_malloc_reclaim_memory) {
         // Instead of freeing the block immediately, push it onto a queue of
         // recently freed blocks.  Free only enough blocks to keep from
@@ -1030,11 +1032,11 @@ static inline void* DebugAllocate(size_t size, int type) {
   return ptr->data_addr();
 }
 
-static inline void DebugDeallocate(void* ptr, int type) {
+static inline void DebugDeallocate(void* ptr, int type, size_t given_size) {
   MALLOC_TRACE("free",
                (ptr != 0 ? MallocBlock::FromRawPointer(ptr)->data_size() : 0),
                ptr);
-  if (ptr)  MallocBlock::FromRawPointer(ptr)->Deallocate(type);
+  if (ptr)  MallocBlock::FromRawPointer(ptr)->Deallocate(type, given_size);
 }
 
 // ========================================================================= //
@@ -1218,7 +1220,12 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW {
 
 extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW {
   MallocHook::InvokeDeleteHook(ptr);
-  DebugDeallocate(ptr, MallocBlock::kMallocType);
+  DebugDeallocate(ptr, MallocBlock::kMallocType, 0);
+}
+
+extern "C" PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW {
+  MallocHook::InvokeDeleteHook(ptr);
+  DebugDeallocate(ptr, MallocBlock::kMallocType, size);
 }
 
 extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) __THROW {
@@ -1234,7 +1241,7 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) __THROW
 
 extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW {
   MallocHook::InvokeDeleteHook(ptr);
-  DebugDeallocate(ptr, MallocBlock::kMallocType);
+  DebugDeallocate(ptr, MallocBlock::kMallocType, 0);
 }
 
 extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW {
@@ -1245,7 +1252,7 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW {
   }
   if (size == 0) {
     MallocHook::InvokeDeleteHook(ptr);
-    DebugDeallocate(ptr, MallocBlock::kMallocType);
+    DebugDeallocate(ptr, MallocBlock::kMallocType, 0);
     return NULL;
   }
   MallocBlock* old = MallocBlock::FromRawPointer(ptr);
@@ -1270,7 +1277,7 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW {
   memcpy(p->data_addr(), ptr, (old_size < size) ? old_size : size);
   MallocHook::InvokeDeleteHook(ptr);
   MallocHook::InvokeNewHook(p->data_addr(), size);
-  DebugDeallocate(ptr, MallocBlock::kMallocType);
+  DebugDeallocate(ptr, MallocBlock::kMallocType, 0);
   MALLOC_TRACE("realloc", p->data_size(), p->data_addr());
   return p->data_addr();
 }
@@ -1292,14 +1299,19 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothr
 
 extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW {
   MallocHook::InvokeDeleteHook(p);
-  DebugDeallocate(p, MallocBlock::kNewType);
+  DebugDeallocate(p, MallocBlock::kNewType, 0);
+}
+
+extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void* p, size_t size) throw() {
+  MallocHook::InvokeDeleteHook(p);
+  DebugDeallocate(p, MallocBlock::kNewType, size);
 }
 
 // Some STL implementations explicitly invoke this.
 // It is completely equivalent to a normal delete (delete never throws).
 extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) __THROW {
   MallocHook::InvokeDeleteHook(p);
-  DebugDeallocate(p, MallocBlock::kNewType);
+  DebugDeallocate(p, MallocBlock::kNewType, 0);
 }
 
 extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) {
@@ -1320,14 +1332,19 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::
 
 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW {
   MallocHook::InvokeDeleteHook(p);
-  DebugDeallocate(p, MallocBlock::kArrayNewType);
+  DebugDeallocate(p, MallocBlock::kArrayNewType, 0);
+}
+
+extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void* p, size_t size) throw() {
+  MallocHook::InvokeDeleteHook(p);
+  DebugDeallocate(p, MallocBlock::kArrayNewType, size);
 }
 
 // Some STL implementations explicitly invoke this.
 // It is completely equivalent to a normal delete (delete never throws).
 extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) __THROW {
   MallocHook::InvokeDeleteHook(p);
-  DebugDeallocate(p, MallocBlock::kArrayNewType);
+  DebugDeallocate(p, MallocBlock::kArrayNewType, 0);
 }
 
 // This is mostly the same as do_memalign in tcmalloc.cc.
diff --git a/src/gperftools/tcmalloc.h.in b/src/gperftools/tcmalloc.h.in
index d43184d..0334d3f 100644
--- a/src/gperftools/tcmalloc.h.in
+++ b/src/gperftools/tcmalloc.h.in
@@ -91,6 +91,7 @@ extern "C" {
   PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW;
   PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) __THROW;
   PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW;
+  PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW;
   PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW;
   PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) __THROW;
   PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW;
diff --git a/src/tcmalloc.cc b/src/tcmalloc.cc
index 5bc400a..f168dfd 100644
--- a/src/tcmalloc.cc
+++ b/src/tcmalloc.cc
@@ -1239,7 +1239,9 @@ inline void free_null_or_invalid(void* ptr, void (*invalid_free_fn)(void*)) {
 ALWAYS_INLINE void do_free_helper(void* ptr,
                                   void (*invalid_free_fn)(void*),
                                   ThreadCache* heap,
-                                  bool heap_must_be_valid) {
+                                  bool heap_must_be_valid,
+                                  bool use_hint,
+                                  size_t size_hint) {
   ASSERT((Static::IsInited() && heap != NULL) || !heap_must_be_valid);
   if (!heap_must_be_valid && !Static::IsInited()) {
     // We called free() before malloc().  This can occur if the
@@ -1252,7 +1254,12 @@ ALWAYS_INLINE void do_free_helper(void* ptr,
   }
   Span* span = NULL;
   const PageID p = reinterpret_cast<uintptr_t>(ptr) >> kPageShift;
-  size_t cl = Static::pageheap()->GetSizeClassIfCached(p);
+  size_t cl;
+  if (use_hint && Static::sizemap()->MaybeSizeClass(size_hint, &cl)) {
+    goto non_zero;
+  }
+
+  cl = Static::pageheap()->GetSizeClassIfCached(p);
   if (UNLIKELY(cl == 0)) {
     span = Static::pageheap()->GetDescriptor(p);
     if (UNLIKELY(!span)) {
@@ -1269,8 +1276,10 @@ ALWAYS_INLINE void do_free_helper(void* ptr,
     cl = span->sizeclass;
     Static::pageheap()->CacheSizeClass(p, cl);
   }
+
   ASSERT(ptr != NULL);
   if (LIKELY(cl != 0)) {
+  non_zero:
     ASSERT(!Static::pageheap()->GetDescriptor(p)->sample);
     if (heap_must_be_valid || heap != NULL) {
       heap->Deallocate(ptr, cl);
@@ -1300,19 +1309,20 @@ ALWAYS_INLINE void do_free_helper(void* ptr,
 // We can usually detect the case where ptr is not pointing to a page that
 // tcmalloc is using, and in those cases we invoke invalid_free_fn.
 ALWAYS_INLINE void do_free_with_callback(void* ptr,
-                                         void (*invalid_free_fn)(void*)) {
+                                         void (*invalid_free_fn)(void*),
+                                         bool use_hint, size_t size_hint) {
   ThreadCache* heap = NULL;
   heap = ThreadCache::GetCacheIfPresent();
   if (LIKELY(heap)) {
-    do_free_helper(ptr, invalid_free_fn, heap, true);
+    do_free_helper(ptr, invalid_free_fn, heap, true, use_hint, size_hint);
   } else {
-    do_free_helper(ptr, invalid_free_fn, heap, false);
+    do_free_helper(ptr, invalid_free_fn, heap, false, use_hint, size_hint);
   }
 }
 
 // The default "do_free" that uses the default callback.
 ALWAYS_INLINE void do_free(void* ptr) {
-  return do_free_with_callback(ptr, &InvalidFree);
+  return do_free_with_callback(ptr, &InvalidFree, false, 0);
 }
 
 // NOTE: some logic here is duplicated in GetOwnership (above), for
@@ -1375,7 +1385,7 @@ ALWAYS_INLINE void* do_realloc_with_callback(
     // We could use a variant of do_free() that leverages the fact
     // that we already know the sizeclass of old_ptr.  The benefit
     // would be small, so don't bother.
-    do_free_with_callback(old_ptr, invalid_free_fn);
+    do_free_with_callback(old_ptr, invalid_free_fn, false, 0);
     return new_ptr;
   } else {
     // We still need to call hooks to report the updated size:
@@ -1576,6 +1586,15 @@ extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW {
   do_free(ptr);
 }
 
+extern "C" PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW {
+  if ((reinterpret_cast<uintptr_t>(ptr) & (kPageSize-1)) == 0) {
+    tc_free(ptr);
+    return;
+  }
+  MallocHook::InvokeDeleteHook(ptr);
+  do_free_with_callback(ptr, &InvalidFree, true, size);
+}
+
 extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t n,
                                               size_t elem_size) __THROW {
   void* result = do_calloc(n, elem_size);
diff --git a/src/windows/gperftools/tcmalloc.h b/src/windows/gperftools/tcmalloc.h
index 9ba79a9..5867c7c 100644
--- a/src/windows/gperftools/tcmalloc.h
+++ b/src/windows/gperftools/tcmalloc.h
@@ -81,6 +81,7 @@ extern "C" {
   PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW;
   PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) __THROW;
   PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW;
+  PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW;
   PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW;
   PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) __THROW;
   PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW;
diff --git a/src/windows/gperftools/tcmalloc.h.in b/src/windows/gperftools/tcmalloc.h.in
index 7458de1..a7ec70f 100644
--- a/src/windows/gperftools/tcmalloc.h.in
+++ b/src/windows/gperftools/tcmalloc.h.in
@@ -81,6 +81,7 @@ extern "C" {
   PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW;
   PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) __THROW;
   PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW;
+  PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW;
   PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW;
   PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) __THROW;
   PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW;
diff --git a/src/windows/patch_functions.cc b/src/windows/patch_functions.cc
index 52e3870..70771d2 100644
--- a/src/windows/patch_functions.cc
+++ b/src/windows/patch_functions.cc
@@ -814,7 +814,7 @@ void LibcInfoWithPatchFunctions<T>::Perftools_free(void* ptr) __THROW {
   // allocated by tcmalloc.  Note it calls the origstub_free from
   // *this* templatized instance of LibcInfo.  See "template
   // trickiness" above.
-  do_free_with_callback(ptr, (void (*)(void*))origstub_fn_[kFree]);
+  do_free_with_callback(ptr, (void (*)(void*))origstub_fn_[kFree], false, 0);
 }
 
 template<int T>
@@ -828,7 +828,7 @@ void* LibcInfoWithPatchFunctions<T>::Perftools_realloc(
   if (new_size == 0) {
     MallocHook::InvokeDeleteHook(old_ptr);
     do_free_with_callback(old_ptr,
-                          (void (*)(void*))origstub_fn_[kFree]);
+                          (void (*)(void*))origstub_fn_[kFree], false, 0);
     return NULL;
   }
   return do_realloc_with_callback(
@@ -862,13 +862,13 @@ void* LibcInfoWithPatchFunctions<T>::Perftools_newarray(size_t size) {
 template<int T>
 void LibcInfoWithPatchFunctions<T>::Perftools_delete(void *p) {
   MallocHook::InvokeDeleteHook(p);
-  do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree]);
+  do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0);
 }
 
 template<int T>
 void LibcInfoWithPatchFunctions<T>::Perftools_deletearray(void *p) {
   MallocHook::InvokeDeleteHook(p);
-  do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree]);
+  do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0);
 }
 
 template<int T>
@@ -891,14 +891,14 @@ template<int T>
 void LibcInfoWithPatchFunctions<T>::Perftools_delete_nothrow(
     void *p, const std::nothrow_t&) __THROW {
   MallocHook::InvokeDeleteHook(p);
-  do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree]);
+  do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0);
 }
 
 template<int T>
 void LibcInfoWithPatchFunctions<T>::Perftools_deletearray_nothrow(
     void *p, const std::nothrow_t&) __THROW {
   MallocHook::InvokeDeleteHook(p);
-  do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree]);
+  do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0);
 }