capture growthz backtraces outside of pageheap_lock

Actual growthz list is now lockless since we never delete anything
from it. And we now pass special 'locking context' object down page
heap allocation path, both as a documentation that it is under lock
and for tracking whether we needed to grow heap and by how much. Then
whenever lock is released in RAII fashion, we're able to trigger
growthz recording outside of lock.

Closes #1159

src/page_heap.cc

@@ -65,6 +65,19 @@ DEFINE_int64(tcmalloc_heap_limit_mb,
 
 namespace tcmalloc {
 
+struct PageHeap::LockingContext {
+  PageHeap * const heap;
+  size_t grown_by = 0;
+
+  explicit LockingContext(PageHeap* heap) : heap(heap) {
+    heap->lock_.Lock();
+  }
+  ~LockingContext() {
+    heap->HandleUnlock(this);
+  }
+};
+
+
 PageHeap::PageHeap(Length smallest_span_size)
     : smallest_span_size_(smallest_span_size),
       pagemap_(MetaDataAlloc),
@@ -128,9 +141,25 @@ Length PageHeap::RoundUpSize(Length n) {
   return rounded_n;
 }
 
+void PageHeap::HandleUnlock(LockingContext* context) {
+  StackTrace* t = nullptr;
+  if (context->grown_by) {
+    t = Static::stacktrace_allocator()->New();
+    t->size = context->grown_by;
+  }
+
+  lock_.Unlock();
+
+  if (t) {
+    t->depth = GetStackTrace(t->stack, kMaxStackDepth-1, 0);
+    Static::push_growth_stack(t);
+  }
+}
+
 Span* PageHeap::NewWithSizeClass(Length n, uint32 sizeclass) {
-  SpinLockHolder h(&lock_);
-  Span* span = NewLocked(n);
+  LockingContext context{this};
+
+  Span* span = NewLocked(n, &context);
   if (!span) {
     return span;
   }
@@ -141,7 +170,7 @@ Span* PageHeap::NewWithSizeClass(Length n, uint32 sizeclass) {
   return span;
 }
 
-Span* PageHeap::NewLocked(Length n) {
+Span* PageHeap::NewLocked(Length n, LockingContext* context) {
   ASSERT(lock_.IsHeld());
   ASSERT(Check());
   n = RoundUpSize(n);
@@ -183,7 +212,7 @@ Span* PageHeap::NewLocked(Length n) {
   }
 
   // Grow the heap and try again.
-  if (!GrowHeap(n)) {
+  if (!GrowHeap(n, context)) {
     ASSERT(stats_.unmapped_bytes+ stats_.committed_bytes==stats_.system_bytes);
     ASSERT(Check());
     // underlying SysAllocator likely set ENOMEM but we can get here
@@ -210,9 +239,9 @@ Span* PageHeap::NewAligned(Length n, Length align_pages) {
     return nullptr;
   }
 
-  SpinLockHolder h(&lock_);
+  LockingContext context{this};
 
-  Span* span = NewLocked(alloc);
+  Span* span = NewLocked(alloc, &context);
   if (PREDICT_FALSE(span == nullptr)) return nullptr;
 
   // Skip starting portion so that we end up aligned
@@ -699,15 +728,7 @@ bool PageHeap::GetNextRange(PageID start, base::MallocRange* r) {
   return true;
 }
 
-static void RecordGrowth(size_t growth) {
-  StackTrace* t = Static::stacktrace_allocator()->New();
-  t->depth = GetStackTrace(t->stack, kMaxStackDepth-1, 3);
-  t->size = growth;
-  t->stack[kMaxStackDepth-1] = reinterpret_cast<void*>(Static::growth_stacks());
-  Static::set_growth_stacks(t);
-}
-
-bool PageHeap::GrowHeap(Length n) {
+bool PageHeap::GrowHeap(Length n, LockingContext* context) {
   ASSERT(lock_.IsHeld());
   ASSERT(kMaxPages >= kMinSystemAlloc);
   if (n > kMaxValidPages) return false;
@@ -728,7 +749,7 @@ bool PageHeap::GrowHeap(Length n) {
     if (ptr == NULL) return false;
   }
   ask = actual_size >> kPageShift;
-  RecordGrowth(ask << kPageShift);
+  context->grown_by += ask << kPageShift;
 
   ++stats_.reserve_count;
   ++stats_.commit_count;

src/page_heap.h

@@ -243,6 +243,10 @@ class PERFTOOLS_DLL_DECL PageHeap {
   }
 
  private:
+  struct LockingContext;
+
+  void HandleUnlock(LockingContext* context);
+
   // Allocates a big block of memory for the pagemap once we reach more than
   // 128MB
   static const size_t kPageMapBigAllocationThreshold = 128 << 20;
@@ -298,7 +302,7 @@ class PERFTOOLS_DLL_DECL PageHeap {
   // Statistics on system, free, and unmapped bytes
   Stats stats_;
 
-  Span* NewLocked(Length n);
+  Span* NewLocked(Length n, LockingContext* context);
   void DeleteLocked(Span* span);
 
   // Split an allocated span into two spans: one of length "n" pages
@@ -313,7 +317,7 @@ class PERFTOOLS_DLL_DECL PageHeap {
 
   Span* SearchFreeAndLargeLists(Length n);
 
-  bool GrowHeap(Length n);
+  bool GrowHeap(Length n, LockingContext* context);
 
   // REQUIRES: span->length >= n
   // REQUIRES: span->location != IN_USE

src/static_vars.cc

@@ -72,7 +72,7 @@ CentralFreeListPadded Static::central_cache_[kClassSizesMax];
 PageHeapAllocator<Span> Static::span_allocator_;
 PageHeapAllocator<StackTrace> Static::stacktrace_allocator_;
 Span Static::sampled_objects_;
-StackTrace* Static::growth_stacks_ = NULL;
+std::atomic<StackTrace*> Static::growth_stacks_;
 Static::PageHeapStorage Static::pageheap_;
 
 void Static::InitStaticVars() {

src/static_vars.h

@@ -36,7 +36,10 @@
 #ifndef TCMALLOC_STATIC_VARS_H_
 #define TCMALLOC_STATIC_VARS_H_
 
-#include <config.h>
+#include "config.h"
+
+#include <atomic>
+
 #include "base/basictypes.h"
 #include "base/spinlock.h"
 #include "central_freelist.h"
@@ -78,8 +81,16 @@ class Static {
     return &stacktrace_allocator_;
   }
 
-  static StackTrace* growth_stacks() { return growth_stacks_; }
-  static void set_growth_stacks(StackTrace* s) { growth_stacks_ = s; }
+  static StackTrace* growth_stacks() { return growth_stacks_.load(std::memory_order_seq_cst); }
+  static void push_growth_stack(StackTrace* s) {
+    ASSERT(s->depth <= kMaxStackDepth - 1);
+    StackTrace* old_top = growth_stacks_.load(std::memory_order_relaxed);
+    do {
+      s->stack[kMaxStackDepth-1] = reinterpret_cast<void*>(old_top);
+    } while (!growth_stacks_.compare_exchange_strong(
+               old_top, s,
+               std::memory_order_seq_cst, std::memory_order_seq_cst));
+  }
 
   // State kept for sampled allocations (/pprof/heap support)
   static Span* sampled_objects() { return &sampled_objects_; }
@@ -108,7 +119,7 @@ class Static {
   // from the system.  Useful for finding allocation sites that cause
   // increase in the footprint of the system.  The linked list pointer
   // is stored in trace->stack[kMaxStackDepth-1].
-  ATTRIBUTE_HIDDEN static StackTrace* growth_stacks_;
+  ATTRIBUTE_HIDDEN static std::atomic<StackTrace*> growth_stacks_;
 
   // PageHeap uses a constructor for initialization.  Like the members above,
   // we can't depend on initialization order, so pageheap is new'd