From baf1b6ce38bba20ff721bbd11736ab47d52f095f Mon Sep 17 00:00:00 2001 From: Aliaksey Kandratsenka Date: Sun, 4 Oct 2015 21:07:54 -0700 Subject: implemented sized free support via tc_free_sized --- src/debugallocation.cc | 45 +++++++++++++++++++++++++----------- src/gperftools/tcmalloc.h.in | 1 + src/tcmalloc.cc | 33 ++++++++++++++++++++------ src/windows/gperftools/tcmalloc.h | 1 + src/windows/gperftools/tcmalloc.h.in | 1 + src/windows/patch_functions.cc | 12 +++++----- 6 files changed, 66 insertions(+), 27 deletions(-) diff --git a/src/debugallocation.cc b/src/debugallocation.cc index c170bc7..5d3a2e6 100644 --- a/src/debugallocation.cc +++ b/src/debugallocation.cc @@ -405,7 +405,7 @@ class MallocBlock { } } - size_t CheckAndClear(int type) { + size_t CheckAndClear(int type, size_t given_size) { alloc_map_lock_.Lock(); CheckLocked(type); if (!IsMMapped()) { @@ -416,6 +416,8 @@ class MallocBlock { alloc_map_lock_.Unlock(); // clear us const size_t size = real_size(); + RAW_CHECK(!given_size || given_size == size1_, + "right size must be passed to sized delete"); memset(this, kMagicDeletedByte, size); return size; } @@ -543,10 +545,10 @@ class MallocBlock { return b; } - void Deallocate(int type) { + void Deallocate(int type, size_t given_size) { if (IsMMapped()) { // have to do this before CheckAndClear #ifdef HAVE_MMAP - int size = CheckAndClear(type); + int size = CheckAndClear(type, given_size); int pagesize = getpagesize(); int num_pages = (size + pagesize - 1) / pagesize + 1; char* p = (char*) this; @@ -559,7 +561,7 @@ class MallocBlock { } #endif } else { - const size_t size = CheckAndClear(type); + const size_t size = CheckAndClear(type, given_size); if (FLAGS_malloc_reclaim_memory) { // Instead of freeing the block immediately, push it onto a queue of // recently freed blocks. Free only enough blocks to keep from @@ -1030,11 +1032,11 @@ static inline void* DebugAllocate(size_t size, int type) { return ptr->data_addr(); } -static inline void DebugDeallocate(void* ptr, int type) { +static inline void DebugDeallocate(void* ptr, int type, size_t given_size) { MALLOC_TRACE("free", (ptr != 0 ? MallocBlock::FromRawPointer(ptr)->data_size() : 0), ptr); - if (ptr) MallocBlock::FromRawPointer(ptr)->Deallocate(type); + if (ptr) MallocBlock::FromRawPointer(ptr)->Deallocate(type, given_size); } // ========================================================================= // @@ -1218,7 +1220,12 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW { extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW { MallocHook::InvokeDeleteHook(ptr); - DebugDeallocate(ptr, MallocBlock::kMallocType); + DebugDeallocate(ptr, MallocBlock::kMallocType, 0); +} + +extern "C" PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW { + MallocHook::InvokeDeleteHook(ptr); + DebugDeallocate(ptr, MallocBlock::kMallocType, size); } extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) __THROW { @@ -1234,7 +1241,7 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) __THROW extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW { MallocHook::InvokeDeleteHook(ptr); - DebugDeallocate(ptr, MallocBlock::kMallocType); + DebugDeallocate(ptr, MallocBlock::kMallocType, 0); } extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW { @@ -1245,7 +1252,7 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW { } if (size == 0) { MallocHook::InvokeDeleteHook(ptr); - DebugDeallocate(ptr, MallocBlock::kMallocType); + DebugDeallocate(ptr, MallocBlock::kMallocType, 0); return NULL; } MallocBlock* old = MallocBlock::FromRawPointer(ptr); @@ -1270,7 +1277,7 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW { memcpy(p->data_addr(), ptr, (old_size < size) ? old_size : size); MallocHook::InvokeDeleteHook(ptr); MallocHook::InvokeNewHook(p->data_addr(), size); - DebugDeallocate(ptr, MallocBlock::kMallocType); + DebugDeallocate(ptr, MallocBlock::kMallocType, 0); MALLOC_TRACE("realloc", p->data_size(), p->data_addr()); return p->data_addr(); } @@ -1292,14 +1299,19 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothr extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW { MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kNewType); + DebugDeallocate(p, MallocBlock::kNewType, 0); +} + +extern "C" PERFTOOLS_DLL_DECL void tc_delete_sized(void* p, size_t size) throw() { + MallocHook::InvokeDeleteHook(p); + DebugDeallocate(p, MallocBlock::kNewType, size); } // Some STL implementations explicitly invoke this. // It is completely equivalent to a normal delete (delete never throws). extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) __THROW { MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kNewType); + DebugDeallocate(p, MallocBlock::kNewType, 0); } extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) { @@ -1320,14 +1332,19 @@ extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std:: extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW { MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kArrayNewType); + DebugDeallocate(p, MallocBlock::kArrayNewType, 0); +} + +extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_sized(void* p, size_t size) throw() { + MallocHook::InvokeDeleteHook(p); + DebugDeallocate(p, MallocBlock::kArrayNewType, size); } // Some STL implementations explicitly invoke this. // It is completely equivalent to a normal delete (delete never throws). extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) __THROW { MallocHook::InvokeDeleteHook(p); - DebugDeallocate(p, MallocBlock::kArrayNewType); + DebugDeallocate(p, MallocBlock::kArrayNewType, 0); } // This is mostly the same as do_memalign in tcmalloc.cc. diff --git a/src/gperftools/tcmalloc.h.in b/src/gperftools/tcmalloc.h.in index d43184d..0334d3f 100644 --- a/src/gperftools/tcmalloc.h.in +++ b/src/gperftools/tcmalloc.h.in @@ -91,6 +91,7 @@ extern "C" { PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW; PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) __THROW; PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW; + PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW; PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW; PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) __THROW; PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW; diff --git a/src/tcmalloc.cc b/src/tcmalloc.cc index 5bc400a..f168dfd 100644 --- a/src/tcmalloc.cc +++ b/src/tcmalloc.cc @@ -1239,7 +1239,9 @@ inline void free_null_or_invalid(void* ptr, void (*invalid_free_fn)(void*)) { ALWAYS_INLINE void do_free_helper(void* ptr, void (*invalid_free_fn)(void*), ThreadCache* heap, - bool heap_must_be_valid) { + bool heap_must_be_valid, + bool use_hint, + size_t size_hint) { ASSERT((Static::IsInited() && heap != NULL) || !heap_must_be_valid); if (!heap_must_be_valid && !Static::IsInited()) { // We called free() before malloc(). This can occur if the @@ -1252,7 +1254,12 @@ ALWAYS_INLINE void do_free_helper(void* ptr, } Span* span = NULL; const PageID p = reinterpret_cast(ptr) >> kPageShift; - size_t cl = Static::pageheap()->GetSizeClassIfCached(p); + size_t cl; + if (use_hint && Static::sizemap()->MaybeSizeClass(size_hint, &cl)) { + goto non_zero; + } + + cl = Static::pageheap()->GetSizeClassIfCached(p); if (UNLIKELY(cl == 0)) { span = Static::pageheap()->GetDescriptor(p); if (UNLIKELY(!span)) { @@ -1269,8 +1276,10 @@ ALWAYS_INLINE void do_free_helper(void* ptr, cl = span->sizeclass; Static::pageheap()->CacheSizeClass(p, cl); } + ASSERT(ptr != NULL); if (LIKELY(cl != 0)) { + non_zero: ASSERT(!Static::pageheap()->GetDescriptor(p)->sample); if (heap_must_be_valid || heap != NULL) { heap->Deallocate(ptr, cl); @@ -1300,19 +1309,20 @@ ALWAYS_INLINE void do_free_helper(void* ptr, // We can usually detect the case where ptr is not pointing to a page that // tcmalloc is using, and in those cases we invoke invalid_free_fn. ALWAYS_INLINE void do_free_with_callback(void* ptr, - void (*invalid_free_fn)(void*)) { + void (*invalid_free_fn)(void*), + bool use_hint, size_t size_hint) { ThreadCache* heap = NULL; heap = ThreadCache::GetCacheIfPresent(); if (LIKELY(heap)) { - do_free_helper(ptr, invalid_free_fn, heap, true); + do_free_helper(ptr, invalid_free_fn, heap, true, use_hint, size_hint); } else { - do_free_helper(ptr, invalid_free_fn, heap, false); + do_free_helper(ptr, invalid_free_fn, heap, false, use_hint, size_hint); } } // The default "do_free" that uses the default callback. ALWAYS_INLINE void do_free(void* ptr) { - return do_free_with_callback(ptr, &InvalidFree); + return do_free_with_callback(ptr, &InvalidFree, false, 0); } // NOTE: some logic here is duplicated in GetOwnership (above), for @@ -1375,7 +1385,7 @@ ALWAYS_INLINE void* do_realloc_with_callback( // We could use a variant of do_free() that leverages the fact // that we already know the sizeclass of old_ptr. The benefit // would be small, so don't bother. - do_free_with_callback(old_ptr, invalid_free_fn); + do_free_with_callback(old_ptr, invalid_free_fn, false, 0); return new_ptr; } else { // We still need to call hooks to report the updated size: @@ -1576,6 +1586,15 @@ extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW { do_free(ptr); } +extern "C" PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW { + if ((reinterpret_cast(ptr) & (kPageSize-1)) == 0) { + tc_free(ptr); + return; + } + MallocHook::InvokeDeleteHook(ptr); + do_free_with_callback(ptr, &InvalidFree, true, size); +} + extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t n, size_t elem_size) __THROW { void* result = do_calloc(n, elem_size); diff --git a/src/windows/gperftools/tcmalloc.h b/src/windows/gperftools/tcmalloc.h index 9ba79a9..5867c7c 100644 --- a/src/windows/gperftools/tcmalloc.h +++ b/src/windows/gperftools/tcmalloc.h @@ -81,6 +81,7 @@ extern "C" { PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW; PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) __THROW; PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW; + PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW; PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW; PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) __THROW; PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW; diff --git a/src/windows/gperftools/tcmalloc.h.in b/src/windows/gperftools/tcmalloc.h.in index 7458de1..a7ec70f 100644 --- a/src/windows/gperftools/tcmalloc.h.in +++ b/src/windows/gperftools/tcmalloc.h.in @@ -81,6 +81,7 @@ extern "C" { PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW; PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) __THROW; PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW; + PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) __THROW; PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW; PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) __THROW; PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW; diff --git a/src/windows/patch_functions.cc b/src/windows/patch_functions.cc index 52e3870..70771d2 100644 --- a/src/windows/patch_functions.cc +++ b/src/windows/patch_functions.cc @@ -814,7 +814,7 @@ void LibcInfoWithPatchFunctions::Perftools_free(void* ptr) __THROW { // allocated by tcmalloc. Note it calls the origstub_free from // *this* templatized instance of LibcInfo. See "template // trickiness" above. - do_free_with_callback(ptr, (void (*)(void*))origstub_fn_[kFree]); + do_free_with_callback(ptr, (void (*)(void*))origstub_fn_[kFree], false, 0); } template @@ -828,7 +828,7 @@ void* LibcInfoWithPatchFunctions::Perftools_realloc( if (new_size == 0) { MallocHook::InvokeDeleteHook(old_ptr); do_free_with_callback(old_ptr, - (void (*)(void*))origstub_fn_[kFree]); + (void (*)(void*))origstub_fn_[kFree], false, 0); return NULL; } return do_realloc_with_callback( @@ -862,13 +862,13 @@ void* LibcInfoWithPatchFunctions::Perftools_newarray(size_t size) { template void LibcInfoWithPatchFunctions::Perftools_delete(void *p) { MallocHook::InvokeDeleteHook(p); - do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree]); + do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0); } template void LibcInfoWithPatchFunctions::Perftools_deletearray(void *p) { MallocHook::InvokeDeleteHook(p); - do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree]); + do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0); } template @@ -891,14 +891,14 @@ template void LibcInfoWithPatchFunctions::Perftools_delete_nothrow( void *p, const std::nothrow_t&) __THROW { MallocHook::InvokeDeleteHook(p); - do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree]); + do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0); } template void LibcInfoWithPatchFunctions::Perftools_deletearray_nothrow( void *p, const std::nothrow_t&) __THROW { MallocHook::InvokeDeleteHook(p); - do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree]); + do_free_with_callback(p, (void (*)(void*))origstub_fn_[kFree], false, 0); } -- cgit v1.2.1