always inline a number of hot functions

4 files changed, 8 insertions, 4 deletions

diff --git a/src/page_heap.h b/src/page_heap.h
index 27d54ba..193bc97 100644
--- a/src/page_heap.h
+++ b/src/page_heap.h
@@ -143,7 +143,8 @@ class PERFTOOLS_DLL_DECL PageHeap {
 
   // Return the descriptor for the specified page.  Returns NULL if
   // this PageID was not allocated previously.
-  inline Span* GetDescriptor(PageID p) const {
+  inline ATTRIBUTE_ALWAYS_INLINE
+  Span* GetDescriptor(PageID p) const {
     return reinterpret_cast<Span*>(pagemap_.get(p));
   }
 
diff --git a/src/pagemap.h b/src/pagemap.h
index a072a0f..50c17db 100644
--- a/src/pagemap.h
+++ b/src/pagemap.h
@@ -89,6 +89,7 @@ class TCMalloc_PageMap1 {
 
   // Return the current value for KEY.  Returns NULL if not yet set,
   // or if k is out of range.
+  ATTRIBUTE_ALWAYS_INLINE
   void* get(Number k) const {
     if ((k >> BITS) > 0) {
       return NULL;
@@ -141,6 +142,7 @@ class TCMalloc_PageMap2 {
     memset(root_, 0, sizeof(root_));
   }
 
+  ATTRIBUTE_ALWAYS_INLINE
   void* get(Number k) const {
     const Number i1 = k >> LEAF_BITS;
     const Number i2 = k & (LEAF_LENGTH-1);
@@ -246,6 +248,7 @@ class TCMalloc_PageMap3 {
     root_ = NewNode();
   }
 
+  ATTRIBUTE_ALWAYS_INLINE
   void* get(Number k) const {
     const Number i1 = k >> (LEAF_BITS + INTERIOR_BITS);
     const Number i2 = (k >> LEAF_BITS) & (INTERIOR_LENGTH-1);
diff --git a/src/tcmalloc.cc b/src/tcmalloc.cc
index 07a7ff7..e6c236d 100644
--- a/src/tcmalloc.cc
+++ b/src/tcmalloc.cc
@@ -1050,12 +1050,12 @@ static inline bool CheckCachedSizeClass(void *ptr) {
   return cached_value == Static::pageheap()->GetDescriptor(p)->sizeclass;
 }
 
-static inline void* CheckedMallocResult(void *result) {
+static inline ATTRIBUTE_ALWAYS_INLINE void* CheckedMallocResult(void *result) {
   ASSERT(result == NULL || CheckCachedSizeClass(result));
   return result;
 }
 
-static inline void* SpanToMallocResult(Span *span) {
+static inline ATTRIBUTE_ALWAYS_INLINE void* SpanToMallocResult(Span *span) {
   Static::pageheap()->InvalidateCachedSizeClass(span->start);
   return
       CheckedMallocResult(reinterpret_cast<void*>(span->start << kPageShift));
diff --git a/src/thread_cache.h b/src/thread_cache.h
index eda3034..bc43992 100644
--- a/src/thread_cache.h
+++ b/src/thread_cache.h
@@ -353,7 +353,7 @@ inline bool ThreadCache::SampleAllocation(size_t k) {
 #endif
 }
 
-inline void* ThreadCache::Allocate(size_t size, size_t cl) {
+inline ATTRIBUTE_ALWAYS_INLINE void* ThreadCache::Allocate(size_t size, size_t cl) {
   ASSERT(size <= kMaxSize);
   ASSERT(size == Static::sizemap()->ByteSizeForClass(cl));