diff --git a/src/heap-checker.cc b/src/heap-checker.cc index c050866..040a33e 100644 --- a/src/heap-checker.cc +++ b/src/heap-checker.cc @@ -2206,13 +2206,14 @@ void HeapLeakChecker::BeforeConstructorsLocked() { RAW_CHECK(MallocHook::AddNewHook(&NewHook), ""); RAW_CHECK(MallocHook::AddDeleteHook(&DeleteHook), ""); constructor_heap_profiling = true; - MemoryRegionMap::Init(1); + MemoryRegionMap::Init(1, /* use_buckets */ false); // Set up MemoryRegionMap with (at least) one caller stack frame to record // (important that it's done before HeapProfileTable creation below). Allocator::Init(); RAW_CHECK(heap_profile == NULL, ""); heap_profile = new(Allocator::Allocate(sizeof(HeapProfileTable))) - HeapProfileTable(&Allocator::Allocate, &Allocator::Free); + HeapProfileTable(&Allocator::Allocate, &Allocator::Free, + /* profile_mmap */ false); RAW_VLOG(10, "Starting tracking the heap"); heap_checker_on = true; } diff --git a/src/heap-profile-stats.h b/src/heap-profile-stats.h new file mode 100644 index 0000000..caf1c27 --- /dev/null +++ b/src/heap-profile-stats.h @@ -0,0 +1,77 @@ +// Copyright (c) 2013, Google Inc. +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file defines structs to accumulate memory allocation and deallocation +// counts. These structs are commonly used for malloc (in HeapProfileTable) +// and mmap (in MemoryRegionMap). + +// A bucket is data structure for heap profiling to store a pair of a stack +// trace and counts of (de)allocation. Buckets are stored in a hash table +// which is declared as "HeapProfileBucket**". +// +// A hash value is computed from a stack trace. Collision in the hash table +// is resolved by separate chaining with linked lists. The links in the list +// are implemented with the member "HeapProfileBucket* next". +// +// A structure of a hash table HeapProfileBucket** bucket_table would be like: +// bucket_table[0] => NULL +// bucket_table[1] => HeapProfileBucket() => HeapProfileBucket() => NULL +// ... +// bucket_table[i] => HeapProfileBucket() => NULL +// ... +// bucket_table[n] => HeapProfileBucket() => NULL + +#ifndef HEAP_PROFILE_STATS_H_ +#define HEAP_PROFILE_STATS_H_ + +struct HeapProfileStats { + // Returns true if the two HeapProfileStats are semantically equal. + bool Equivalent(const HeapProfileStats& other) const { + return allocs - frees == other.allocs - other.frees && + alloc_size - free_size == other.alloc_size - other.free_size; + } + + int32 allocs; // Number of allocation calls. + int32 frees; // Number of free calls. + int64 alloc_size; // Total size of all allocated objects so far. + int64 free_size; // Total size of all freed objects so far. +}; + +// Allocation and deallocation statistics per each stack trace. +struct HeapProfileBucket : public HeapProfileStats { + // Longest stack trace we record. + static const int kMaxStackDepth = 32; + + uintptr_t hash; // Hash value of the stack trace. + int depth; // Depth of stack trace. + const void** stack; // Stack trace. + HeapProfileBucket* next; // Next entry in hash-table. +}; + +#endif // HEAP_PROFILE_STATS_H_ diff --git a/src/heap-profile-table.cc b/src/heap-profile-table.cc index 6db2caa..b92251f 100644 --- a/src/heap-profile-table.cc +++ b/src/heap-profile-table.cc @@ -99,8 +99,7 @@ const char HeapProfileTable::kFileExt[] = ".heap"; //---------------------------------------------------------------------- -// Size for alloc_table_ and mmap_table_. -static const int kHashTableSize = 179999; +static const int kHashTableSize = 179999; // Size for bucket_table_. /*static*/ const int HeapProfileTable::kMaxStackDepth; //---------------------------------------------------------------------- @@ -122,62 +121,50 @@ static bool ByAllocatedSpace(HeapProfileTable::Stats* a, //---------------------------------------------------------------------- -HeapProfileTable::HeapProfileTable(Allocator alloc, DeAllocator dealloc) - : alloc_(alloc), dealloc_(dealloc) { - // Initialize the overall profile stats. - memset(&total_, 0, sizeof(total_)); +HeapProfileTable::HeapProfileTable(Allocator alloc, + DeAllocator dealloc, + bool profile_mmap) + : alloc_(alloc), + dealloc_(dealloc), + bucket_table_(NULL), + profile_mmap_(profile_mmap), + num_buckets_(0), + address_map_(NULL) { + // Make a hash table for buckets. + const int table_bytes = kHashTableSize * sizeof(*bucket_table_); + bucket_table_ = static_cast(alloc_(table_bytes)); + memset(bucket_table_, 0, table_bytes); - // Make the malloc table. - const int alloc_table_bytes = kHashTableSize * sizeof(*alloc_table_); - alloc_table_ = reinterpret_cast(alloc_(alloc_table_bytes)); - memset(alloc_table_, 0, alloc_table_bytes); - num_alloc_buckets_ = 0; - - // Initialize the mmap table. - mmap_table_ = NULL; - num_available_mmap_buckets_ = 0; - - // Make malloc and mmap allocation maps. - alloc_address_map_ = + // Make an allocation map. + address_map_ = new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_); - mmap_address_map_ = NULL; + + // Initialize. + memset(&total_, 0, sizeof(total_)); + num_buckets_ = 0; } HeapProfileTable::~HeapProfileTable() { - DeallocateBucketTable(alloc_table_); - alloc_table_ = NULL; - DeallocateBucketTable(mmap_table_); - mmap_table_ = NULL; - DeallocateAllocationMap(alloc_address_map_); - alloc_address_map_ = NULL; - DeallocateAllocationMap(mmap_address_map_); - mmap_address_map_ = NULL; -} + // Free the allocation map. + address_map_->~AllocationMap(); + dealloc_(address_map_); + address_map_ = NULL; -void HeapProfileTable::DeallocateAllocationMap(AllocationMap* allocation) { - if (allocation != NULL) { - alloc_address_map_->~AllocationMap(); - dealloc_(allocation); - } -} - -void HeapProfileTable::DeallocateBucketTable(Bucket** table) { - if (table != NULL) { - for (int b = 0; b < kHashTableSize; b++) { - for (Bucket* x = table[b]; x != 0; /**/) { - Bucket* b = x; - x = x->next; - dealloc_(b->stack); - dealloc_(b); - } + // Free the hash table. + for (int i = 0; i < kHashTableSize; i++) { + for (Bucket* curr = bucket_table_[i]; curr != 0; /**/) { + Bucket* bucket = curr; + curr = curr->next; + dealloc_(bucket->stack); + dealloc_(bucket); } - dealloc_(table); } + dealloc_(bucket_table_); + bucket_table_ = NULL; } -HeapProfileTable::Bucket* HeapProfileTable::GetBucket( - int depth, const void* const key[], Bucket** table, - int* bucket_count) { +HeapProfileTable::Bucket* HeapProfileTable::GetBucket(int depth, + const void* const key[]) { // Make hash-value uintptr_t h = 0; for (int i = 0; i < depth; i++) { @@ -190,7 +177,7 @@ HeapProfileTable::Bucket* HeapProfileTable::GetBucket( // Lookup stack trace in table unsigned int buck = ((unsigned int) h) % kHashTableSize; - for (Bucket* b = table[buck]; b != 0; b = b->next) { + for (Bucket* b = bucket_table_[buck]; b != 0; b = b->next) { if ((b->hash == h) && (b->depth == depth) && equal(key, key + depth, b->stack)) { @@ -207,11 +194,9 @@ HeapProfileTable::Bucket* HeapProfileTable::GetBucket( b->hash = h; b->depth = depth; b->stack = kcopy; - b->next = table[buck]; - table[buck] = b; - if (bucket_count != NULL) { - ++(*bucket_count); - } + b->next = bucket_table_[buck]; + bucket_table_[buck] = b; + num_buckets_++; return b; } @@ -224,8 +209,7 @@ int HeapProfileTable::GetCallerStackTrace( void HeapProfileTable::RecordAlloc( const void* ptr, size_t bytes, int stack_depth, const void* const call_stack[]) { - Bucket* b = GetBucket(stack_depth, call_stack, alloc_table_, - &num_alloc_buckets_); + Bucket* b = GetBucket(stack_depth, call_stack); b->allocs++; b->alloc_size += bytes; total_.allocs++; @@ -234,12 +218,12 @@ void HeapProfileTable::RecordAlloc( AllocValue v; v.set_bucket(b); // also did set_live(false); set_ignore(false) v.bytes = bytes; - alloc_address_map_->Insert(ptr, v); + address_map_->Insert(ptr, v); } void HeapProfileTable::RecordFree(const void* ptr) { AllocValue v; - if (alloc_address_map_->FindAndRemove(ptr, &v)) { + if (address_map_->FindAndRemove(ptr, &v)) { Bucket* b = v.bucket(); b->frees++; b->free_size += v.bytes; @@ -249,14 +233,14 @@ void HeapProfileTable::RecordFree(const void* ptr) { } bool HeapProfileTable::FindAlloc(const void* ptr, size_t* object_size) const { - const AllocValue* alloc_value = alloc_address_map_->Find(ptr); + const AllocValue* alloc_value = address_map_->Find(ptr); if (alloc_value != NULL) *object_size = alloc_value->bytes; return alloc_value != NULL; } bool HeapProfileTable::FindAllocDetails(const void* ptr, AllocInfo* info) const { - const AllocValue* alloc_value = alloc_address_map_->Find(ptr); + const AllocValue* alloc_value = address_map_->Find(ptr); if (alloc_value != NULL) { info->object_size = alloc_value->bytes; info->call_stack = alloc_value->bucket()->stack; @@ -270,13 +254,13 @@ bool HeapProfileTable::FindInsideAlloc(const void* ptr, const void** object_ptr, size_t* object_size) const { const AllocValue* alloc_value = - alloc_address_map_->FindInside(&AllocValueSize, max_size, ptr, object_ptr); + address_map_->FindInside(&AllocValueSize, max_size, ptr, object_ptr); if (alloc_value != NULL) *object_size = alloc_value->bytes; return alloc_value != NULL; } bool HeapProfileTable::MarkAsLive(const void* ptr) { - AllocValue* alloc = alloc_address_map_->FindMutable(ptr); + AllocValue* alloc = address_map_->FindMutable(ptr); if (alloc && !alloc->live()) { alloc->set_live(true); return true; @@ -285,7 +269,7 @@ bool HeapProfileTable::MarkAsLive(const void* ptr) { } void HeapProfileTable::MarkAsIgnored(const void* ptr) { - AllocValue* alloc = alloc_address_map_->FindMutable(ptr); + AllocValue* alloc = address_map_->FindMutable(ptr); if (alloc) { alloc->set_ignore(true); } @@ -326,81 +310,26 @@ int HeapProfileTable::UnparseBucket(const Bucket& b, HeapProfileTable::Bucket** HeapProfileTable::MakeSortedBucketList() const { - Bucket** list = reinterpret_cast(alloc_(sizeof(Bucket) * - (num_alloc_buckets_ + num_available_mmap_buckets_))); + Bucket** list = static_cast(alloc_(sizeof(Bucket) * num_buckets_)); - RAW_DCHECK(mmap_table_ != NULL || num_available_mmap_buckets_ == 0, ""); - - int n = 0; - - for (int b = 0; b < kHashTableSize; b++) { - for (Bucket* x = alloc_table_[b]; x != 0; x = x->next) { - list[n++] = x; + int bucket_count = 0; + for (int i = 0; i < kHashTableSize; i++) { + for (Bucket* curr = bucket_table_[i]; curr != 0; curr = curr->next) { + list[bucket_count++] = curr; } } - RAW_DCHECK(n == num_alloc_buckets_, ""); + RAW_DCHECK(bucket_count == num_buckets_, ""); - if (mmap_table_ != NULL) { - for (int b = 0; b < kHashTableSize; b++) { - for (Bucket* x = mmap_table_[b]; x != 0; x = x->next) { - list[n++] = x; - } - } - } - RAW_DCHECK(n == num_alloc_buckets_ + num_available_mmap_buckets_, ""); - - sort(list, list + num_alloc_buckets_ + num_available_mmap_buckets_, - ByAllocatedSpace); + sort(list, list + num_buckets_, ByAllocatedSpace); return list; } -void HeapProfileTable::RefreshMMapData() { - // Make the table - static const int mmap_table_bytes = kHashTableSize * sizeof(*mmap_table_); - if (mmap_table_ == NULL) { - mmap_table_ = reinterpret_cast(alloc_(mmap_table_bytes)); - memset(mmap_table_, 0, mmap_table_bytes); - } - num_available_mmap_buckets_ = 0; - - ClearMMapData(); - mmap_address_map_ = - new(alloc_(sizeof(AllocationMap))) AllocationMap(alloc_, dealloc_); - - MemoryRegionMap::LockHolder l; - for (MemoryRegionMap::RegionIterator r = - MemoryRegionMap::BeginRegionLocked(); - r != MemoryRegionMap::EndRegionLocked(); ++r) { - Bucket* b = - GetBucket(r->call_stack_depth, r->call_stack, mmap_table_, NULL); - if (b->alloc_size == 0) { - num_available_mmap_buckets_ += 1; - } - b->allocs += 1; - b->alloc_size += r->end_addr - r->start_addr; - - AllocValue v; - v.set_bucket(b); - v.bytes = r->end_addr - r->start_addr; - mmap_address_map_->Insert(reinterpret_cast(r->start_addr), v); - } -} - -void HeapProfileTable::ClearMMapData() { - if (mmap_address_map_ != NULL) { - mmap_address_map_->Iterate(ZeroBucketCountsIterator, this); - mmap_address_map_->~AllocationMap(); - dealloc_(mmap_address_map_); - mmap_address_map_ = NULL; - } -} - void HeapProfileTable::IterateOrderedAllocContexts( AllocContextIterator callback) const { Bucket** list = MakeSortedBucketList(); AllocContextInfo info; - for (int i = 0; i < num_alloc_buckets_; ++i) { + for (int i = 0; i < num_buckets_; ++i) { *static_cast(&info) = *static_cast(list[i]); info.stack_depth = list[i]->depth; info.call_stack = list[i]->stack; @@ -432,14 +361,17 @@ int HeapProfileTable::FillOrderedProfile(char buf[], int size) const { memset(&stats, 0, sizeof(stats)); int bucket_length = snprintf(buf, size, "%s", kProfileHeader); if (bucket_length < 0 || bucket_length >= size) return 0; - Bucket total_with_mmap(total_); - if (mmap_table_ != NULL) { - total_with_mmap.alloc_size += MemoryRegionMap::MapSize(); - total_with_mmap.free_size += MemoryRegionMap::UnmapSize(); - } - bucket_length = UnparseBucket(total_with_mmap, buf, bucket_length, size, + bucket_length = UnparseBucket(total_, buf, bucket_length, size, " heapprofile", &stats); - for (int i = 0; i < num_alloc_buckets_; i++) { + + // Dump the mmap list first. + if (profile_mmap_) { + BufferArgs buffer(buf, bucket_length, size); + MemoryRegionMap::IterateBuckets(DumpBucketIterator, &buffer); + bucket_length = buffer.buflen; + } + + for (int i = 0; i < num_buckets_; i++) { bucket_length = UnparseBucket(*list[i], buf, bucket_length, size, "", &stats); } @@ -453,6 +385,13 @@ int HeapProfileTable::FillOrderedProfile(char buf[], int size) const { return bucket_length + map_length; } +// static +void HeapProfileTable::DumpBucketIterator(const Bucket* bucket, + BufferArgs* args) { + args->buflen = UnparseBucket(*bucket, args->buf, args->buflen, args->bufsize, + "", NULL); +} + inline void HeapProfileTable::DumpNonLiveIterator(const void* ptr, AllocValue* v, const DumpArgs& args) { @@ -474,17 +413,6 @@ void HeapProfileTable::DumpNonLiveIterator(const void* ptr, AllocValue* v, RawWrite(args.fd, buf, len); } -inline void HeapProfileTable::ZeroBucketCountsIterator( - const void* ptr, AllocValue* v, HeapProfileTable* heap_profile) { - Bucket* b = v->bucket(); - if (b != NULL) { - b->allocs = 0; - b->alloc_size = 0; - b->free_size = 0; - b->frees = 0; - } -} - // Callback from NonLiveSnapshot; adds entry to arg->dest // if not the entry is not live and is not present in arg->base. void HeapProfileTable::AddIfNonLive(const void* ptr, AllocValue* v, @@ -549,7 +477,7 @@ void HeapProfileTable::CleanupOldProfiles(const char* prefix) { HeapProfileTable::Snapshot* HeapProfileTable::TakeSnapshot() { Snapshot* s = new (alloc_(sizeof(Snapshot))) Snapshot(alloc_, dealloc_); - alloc_address_map_->Iterate(AddToSnapshot, s); + address_map_->Iterate(AddToSnapshot, s); return s; } @@ -574,7 +502,7 @@ HeapProfileTable::Snapshot* HeapProfileTable::NonLiveSnapshot( AddNonLiveArgs args; args.dest = s; args.base = base; - alloc_address_map_->Iterate(AddIfNonLive, &args); + address_map_->Iterate(AddIfNonLive, &args); RAW_VLOG(2, "NonLiveSnapshot output: %d %d\n", int(s->total_.allocs - s->total_.frees), int(s->total_.alloc_size - s->total_.free_size)); diff --git a/src/heap-profile-table.h b/src/heap-profile-table.h index abd3184..78f1e62 100644 --- a/src/heap-profile-table.h +++ b/src/heap-profile-table.h @@ -38,6 +38,7 @@ #include "addressmap-inl.h" #include "base/basictypes.h" #include "base/logging.h" // for RawFD +#include "heap-profile-stats.h" // Table to maintain a heap profile data inside, // i.e. the set of currently active heap memory allocations. @@ -58,18 +59,7 @@ class HeapProfileTable { // data types ---------------------------- // Profile stats. - struct Stats { - int32 allocs; // Number of allocation calls - int32 frees; // Number of free calls - int64 alloc_size; // Total size of all allocated objects so far - int64 free_size; // Total size of all freed objects so far - - // semantic equality - bool Equivalent(const Stats& x) const { - return allocs - frees == x.allocs - x.frees && - alloc_size - free_size == x.alloc_size - x.free_size; - } - }; + typedef HeapProfileStats Stats; // Info we can return about an allocation. struct AllocInfo { @@ -94,7 +84,7 @@ class HeapProfileTable { // interface --------------------------- - HeapProfileTable(Allocator alloc, DeAllocator dealloc); + HeapProfileTable(Allocator alloc, DeAllocator dealloc, bool profile_mmap); ~HeapProfileTable(); // Collect the stack trace for the function that asked to do the @@ -149,7 +139,7 @@ class HeapProfileTable { // Iterate over the allocation profile data calling "callback" // for every allocation. void IterateAllocs(AllocIterator callback) const { - alloc_address_map_->Iterate(MapArgsAllocIterator, callback); + address_map_->Iterate(MapArgsAllocIterator, callback); } // Allocation context profile data iteration callback @@ -187,28 +177,13 @@ class HeapProfileTable { // Caller must call ReleaseSnapshot() on result when no longer needed. Snapshot* NonLiveSnapshot(Snapshot* base); - // Refresh the internal mmap information from MemoryRegionMap. Results of - // FillOrderedProfile and IterateOrderedAllocContexts will contain mmap'ed - // memory regions as at calling RefreshMMapData. - void RefreshMMapData(); - - // Clear the internal mmap information. Results of FillOrderedProfile and - // IterateOrderedAllocContexts won't contain mmap'ed memory regions after - // calling ClearMMapData. - void ClearMMapData(); - private: // data types ---------------------------- // Hash table bucket to hold (de)allocation stats // for a given allocation call stack trace. - struct Bucket : public Stats { - uintptr_t hash; // Hash value of the stack trace - int depth; // Depth of stack trace - const void** stack; // Stack trace - Bucket* next; // Next entry in hash-table - }; + typedef HeapProfileBucket Bucket; // Info stored in the address map struct AllocValue { @@ -247,13 +222,30 @@ class HeapProfileTable { typedef AddressMap AllocationMap; + // Arguments that need to be passed DumpBucketIterator callback below. + struct BufferArgs { + BufferArgs(char* buf_arg, int buflen_arg, int bufsize_arg) + : buf(buf_arg), + buflen(buflen_arg), + bufsize(bufsize_arg) { + } + + char* buf; + int buflen; + int bufsize; + + DISALLOW_COPY_AND_ASSIGN(BufferArgs); + }; + // Arguments that need to be passed DumpNonLiveIterator callback below. struct DumpArgs { + DumpArgs(RawFD fd_arg, Stats* profile_stats_arg) + : fd(fd_arg), + profile_stats(profile_stats_arg) { + } + RawFD fd; // file to write to Stats* profile_stats; // stats to update (may be NULL) - - DumpArgs(RawFD a, Stats* d) - : fd(a), profile_stats(d) { } }; // helpers ---------------------------- @@ -274,18 +266,9 @@ class HeapProfileTable { const char* extra, Stats* profile_stats); - // Deallocate a given allocation map. - void DeallocateAllocationMap(AllocationMap* allocation); - - // Deallocate a given bucket table. - void DeallocateBucketTable(Bucket** table); - - // Get the bucket for the caller stack trace 'key' of depth 'depth' from a - // bucket hash map 'table' creating the bucket if needed. '*bucket_count' - // is incremented both when 'bucket_count' is not NULL and when a new - // bucket object is created. - Bucket* GetBucket(int depth, const void* const key[], Bucket** table, - int* bucket_count); + // Get the bucket for the caller stack trace 'key' of depth 'depth' + // creating the bucket if needed. + Bucket* GetBucket(int depth, const void* const key[]); // Helper for IterateAllocs to do callback signature conversion // from AllocationMap::Iterate to AllocIterator. @@ -300,18 +283,17 @@ class HeapProfileTable { callback(ptr, info); } + // Helper to dump a bucket. + inline static void DumpBucketIterator(const Bucket* bucket, + BufferArgs* args); + // Helper for DumpNonLiveProfile to do object-granularity // heap profile dumping. It gets passed to AllocationMap::Iterate. inline static void DumpNonLiveIterator(const void* ptr, AllocValue* v, const DumpArgs& args); - // Helper for filling size variables in buckets by zero. - inline static void ZeroBucketCountsIterator( - const void* ptr, AllocValue* v, HeapProfileTable* heap_profile); - // Helper for IterateOrderedAllocContexts and FillOrderedProfile. - // Creates a sorted list of Buckets whose length is num_alloc_buckets_ + - // num_avaliable_mmap_buckets_. + // Creates a sorted list of Buckets whose length is num_buckets_. // The caller is responsible for deallocating the returned list. Bucket** MakeSortedBucketList() const; @@ -344,25 +326,19 @@ class HeapProfileTable { // Overall profile stats; we use only the Stats part, // but make it a Bucket to pass to UnparseBucket. - // It doesn't contain mmap'ed regions. Bucket total_; + bool profile_mmap_; + // Bucket hash table for malloc. // We hand-craft one instead of using one of the pre-written // ones because we do not want to use malloc when operating on the table. // It is only few lines of code, so no big deal. - Bucket** alloc_table_; - int num_alloc_buckets_; - - // Bucket hash table for mmap. - // This table is filled with the information from MemoryRegionMap by calling - // RefreshMMapData. - Bucket** mmap_table_; - int num_available_mmap_buckets_; + Bucket** bucket_table_; + int num_buckets_; // Map of all currently allocated objects and mapped regions we know about. - AllocationMap* alloc_address_map_; - AllocationMap* mmap_address_map_; + AllocationMap* address_map_; DISALLOW_COPY_AND_ASSIGN(HeapProfileTable); }; diff --git a/src/heap-profiler.cc b/src/heap-profiler.cc index f1f2c21..aa95a51 100644 --- a/src/heap-profiler.cc +++ b/src/heap-profiler.cc @@ -190,13 +190,14 @@ static char* DoGetHeapProfileLocked(char* buf, int buflen) { RAW_DCHECK(heap_lock.IsHeld(), ""); int bytes_written = 0; if (is_on) { - if (FLAGS_mmap_profile) { - heap_profile->RefreshMMapData(); - } + HeapProfileTable::Stats const stats = heap_profile->total(); + (void)stats; // avoid an unused-variable warning in non-debug mode. bytes_written = heap_profile->FillOrderedProfile(buf, buflen - 1); - if (FLAGS_mmap_profile) { - heap_profile->ClearMMapData(); - } + // FillOrderedProfile should not reduce the set of active mmap-ed regions, + // hence MemoryRegionMap will let us remove everything we've added above: + RAW_DCHECK(stats.Equivalent(heap_profile->total()), ""); + // if this fails, we somehow removed by FillOrderedProfile + // more than we have added. } buf[bytes_written] = '\0'; RAW_DCHECK(bytes_written == strlen(buf), ""); @@ -435,7 +436,8 @@ extern "C" void HeapProfilerStart(const char* prefix) { if (FLAGS_mmap_profile) { // Ask MemoryRegionMap to record all mmap, mremap, and sbrk // call stack traces of at least size kMaxStackDepth: - MemoryRegionMap::Init(HeapProfileTable::kMaxStackDepth); + MemoryRegionMap::Init(HeapProfileTable::kMaxStackDepth, + /* use_buckets */ true); } if (FLAGS_mmap_log) { @@ -455,7 +457,7 @@ extern "C" void HeapProfilerStart(const char* prefix) { reinterpret_cast(ProfilerMalloc(kProfileBufferSize)); heap_profile = new(ProfilerMalloc(sizeof(HeapProfileTable))) - HeapProfileTable(ProfilerMalloc, ProfilerFree); + HeapProfileTable(ProfilerMalloc, ProfilerFree, FLAGS_mmap_profile); last_dump_alloc = 0; last_dump_free = 0; diff --git a/src/memory_region_map.cc b/src/memory_region_map.cc index 0ce97c2..899b08c 100644 --- a/src/memory_region_map.cc +++ b/src/memory_region_map.cc @@ -84,9 +84,10 @@ // which (sometimes) causes mmap, which calls our hook, and so on. // We do this as follows: on a recursive call of MemoryRegionMap's // mmap/sbrk/mremap hook we record the data about the allocation in a -// static fixed-sized stack (saved_regions), when the recursion unwinds -// but before returning from the outer hook call we unwind this stack and -// move the data from saved_regions to its permanent place in the RegionSet, +// static fixed-sized stack (saved_regions and saved_buckets), when the +// recursion unwinds but before returning from the outer hook call we unwind +// this stack and move the data from saved_regions and saved_buckets to its +// permanent place in the RegionSet and "bucket_table" respectively, // which can cause more allocations and mmap-s and recursion and unwinding, // but the whole process ends eventually due to the fact that for the small // allocations we are doing LowLevelAlloc reuses one mmap call and parcels out @@ -147,6 +148,13 @@ int MemoryRegionMap::recursion_count_ = 0; // GUARDED_BY(owner_lock_) pthread_t MemoryRegionMap::lock_owner_tid_; // GUARDED_BY(owner_lock_) int64 MemoryRegionMap::map_size_ = 0; int64 MemoryRegionMap::unmap_size_ = 0; +HeapProfileBucket** MemoryRegionMap::bucket_table_ = NULL; // GUARDED_BY(lock_) +int MemoryRegionMap::num_buckets_ = 0; // GUARDED_BY(lock_) +int MemoryRegionMap::saved_buckets_count_ = 0; // GUARDED_BY(lock_) +HeapProfileBucket MemoryRegionMap::saved_buckets_[20]; // GUARDED_BY(lock_) + +// GUARDED_BY(lock_) +const void* MemoryRegionMap::saved_buckets_keys_[20][kMaxStackDepth]; // ========================================================================= // @@ -182,7 +190,7 @@ static MemoryRegionMap::RegionSetRep regions_rep; // (or rather should we *not* use regions_ to record a hooked mmap). static bool recursive_insert = false; -void MemoryRegionMap::Init(int max_stack_depth) { +void MemoryRegionMap::Init(int max_stack_depth, bool use_buckets) { RAW_VLOG(10, "MemoryRegionMap Init"); RAW_CHECK(max_stack_depth >= 0, ""); // Make sure we don't overflow the memory in region stacks: @@ -214,6 +222,15 @@ void MemoryRegionMap::Init(int max_stack_depth) { // Can't instead use HandleSavedRegionsLocked(&DoInsertRegionLocked) before // recursive_insert = false; as InsertRegionLocked will also construct // regions_ on demand for us. + if (use_buckets) { + const int table_bytes = kHashTableSize * sizeof(*bucket_table_); + recursive_insert = true; + bucket_table_ = static_cast( + MyAllocator::Allocate(table_bytes)); + recursive_insert = false; + memset(bucket_table_, 0, table_bytes); + num_buckets_ = 0; + } Unlock(); RAW_VLOG(10, "MemoryRegionMap Init done"); } @@ -228,6 +245,19 @@ bool MemoryRegionMap::Shutdown() { RAW_VLOG(10, "MemoryRegionMap Shutdown decrement done"); return true; } + if (bucket_table_ != NULL) { + for (int i = 0; i < kHashTableSize; i++) { + for (HeapProfileBucket* curr = bucket_table_[i]; curr != 0; /**/) { + HeapProfileBucket* bucket = curr; + curr = curr->next; + MyAllocator::Free(bucket->stack, 0); + MyAllocator::Free(bucket, 0); + } + } + MyAllocator::Free(bucket_table_, 0); + num_buckets_ = 0; + bucket_table_ = NULL; + } RAW_CHECK(MallocHook::RemoveMmapHook(&MmapHook), ""); RAW_CHECK(MallocHook::RemoveMremapHook(&MremapHook), ""); RAW_CHECK(MallocHook::RemoveSbrkHook(&SbrkHook), ""); @@ -245,6 +275,11 @@ bool MemoryRegionMap::Shutdown() { return deleted_arena; } +bool MemoryRegionMap::IsRecordingLocked() { + RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); + return client_count_ > 0; +} + // Invariants (once libpthread_initialized is true): // * While lock_ is not held, recursion_count_ is 0 (and // lock_owner_tid_ is the previous owner, but we don't rely on @@ -336,6 +371,62 @@ bool MemoryRegionMap::FindAndMarkStackRegion(uintptr_t stack_top, return region != NULL; } +HeapProfileBucket* MemoryRegionMap::GetBucket(int depth, + const void* const key[]) { + RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); + // Make hash-value + uintptr_t hash = 0; + for (int i = 0; i < depth; i++) { + hash += reinterpret_cast(key[i]); + hash += hash << 10; + hash ^= hash >> 6; + } + hash += hash << 3; + hash ^= hash >> 11; + + // Lookup stack trace in table + unsigned int hash_index = (static_cast(hash)) % kHashTableSize; + for (HeapProfileBucket* bucket = bucket_table_[hash_index]; + bucket != 0; + bucket = bucket->next) { + if ((bucket->hash == hash) && (bucket->depth == depth) && + std::equal(key, key + depth, bucket->stack)) { + return bucket; + } + } + + // Create new bucket + const size_t key_size = sizeof(key[0]) * depth; + HeapProfileBucket* bucket; + if (recursive_insert) { // recursion: save in saved_buckets_ + const void** key_copy = saved_buckets_keys_[saved_buckets_count_]; + std::copy(key, key + depth, key_copy); + bucket = &saved_buckets_[saved_buckets_count_]; + memset(bucket, 0, sizeof(*bucket)); + ++saved_buckets_count_; + bucket->stack = key_copy; + bucket->next = NULL; + } else { + recursive_insert = true; + const void** key_copy = static_cast( + MyAllocator::Allocate(key_size)); + recursive_insert = false; + std::copy(key, key + depth, key_copy); + recursive_insert = true; + bucket = static_cast( + MyAllocator::Allocate(sizeof(HeapProfileBucket))); + recursive_insert = false; + memset(bucket, 0, sizeof(*bucket)); + bucket->stack = key_copy; + bucket->next = bucket_table_[hash_index]; + } + bucket->hash = hash; + bucket->depth = depth; + bucket_table_[hash_index] = bucket; + ++num_buckets_; + return bucket; +} + MemoryRegionMap::RegionIterator MemoryRegionMap::BeginRegionLocked() { RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); RAW_CHECK(regions_ != NULL, ""); @@ -404,6 +495,44 @@ inline void MemoryRegionMap::HandleSavedRegionsLocked( } } +void MemoryRegionMap::RestoreSavedBucketsLocked() { + RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); + while (saved_buckets_count_ > 0) { + HeapProfileBucket bucket = saved_buckets_[--saved_buckets_count_]; + unsigned int hash_index = + static_cast(bucket.hash) % kHashTableSize; + bool is_found = false; + for (HeapProfileBucket* curr = bucket_table_[hash_index]; + curr != 0; + curr = curr->next) { + if ((curr->hash == bucket.hash) && (curr->depth == bucket.depth) && + std::equal(bucket.stack, bucket.stack + bucket.depth, curr->stack)) { + curr->allocs += bucket.allocs; + curr->alloc_size += bucket.alloc_size; + curr->frees += bucket.frees; + curr->free_size += bucket.free_size; + is_found = true; + break; + } + } + if (is_found) continue; + + const size_t key_size = sizeof(bucket.stack[0]) * bucket.depth; + const void** key_copy = static_cast( + MyAllocator::Allocate(key_size)); + std::copy(bucket.stack, bucket.stack + bucket.depth, key_copy); + HeapProfileBucket* new_bucket = static_cast( + MyAllocator::Allocate(sizeof(HeapProfileBucket))); + memset(new_bucket, 0, sizeof(*new_bucket)); + new_bucket->hash = bucket.hash; + new_bucket->depth = bucket.depth; + new_bucket->stack = key_copy; + new_bucket->next = bucket_table_[hash_index]; + bucket_table_[hash_index] = new_bucket; + ++num_buckets_; + } +} + inline void MemoryRegionMap::InsertRegionLocked(const Region& region) { RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); // We can be called recursively, because RegionSet constructor @@ -468,6 +597,16 @@ void MemoryRegionMap::RecordRegionAddition(const void* start, size_t size) { InsertRegionLocked(region); // This will (eventually) allocate storage for and copy over the stack data // from region.call_stack_data_ that is pointed by region.call_stack(). + if (bucket_table_ != NULL) { + HeapProfileBucket* b = GetBucket(depth, region.call_stack); + ++b->allocs; + b->alloc_size += size; + if (!recursive_insert) { + recursive_insert = true; + RestoreSavedBucketsLocked(); + recursive_insert = false; + } + } Unlock(); } @@ -486,6 +625,7 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) { Region& r = saved_regions[i]; if (r.start_addr == start_addr && r.end_addr == end_addr) { // An exact match, so it's safe to remove. + RecordRegionRemovalInBucket(r.call_stack_depth, r.call_stack, size); --saved_regions_count; --put_pos; RAW_VLOG(10, ("Insta-Removing saved region %p..%p; " @@ -530,6 +670,8 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) { RAW_VLOG(12, "Deleting region %p..%p", reinterpret_cast(region->start_addr), reinterpret_cast(region->end_addr)); + RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack, + region->end_addr - region->start_addr); RegionSet::iterator d = region; ++region; regions_->erase(d); @@ -539,6 +681,8 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) { RAW_VLOG(12, "Splitting region %p..%p in two", reinterpret_cast(region->start_addr), reinterpret_cast(region->end_addr)); + RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack, + end_addr - start_addr); // Make another region for the start portion: // The new region has to be the start portion because we can't // just modify region->end_addr as it's the sorting key. @@ -552,12 +696,16 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) { RAW_VLOG(12, "Start-chopping region %p..%p", reinterpret_cast(region->start_addr), reinterpret_cast(region->end_addr)); + RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack, + end_addr - region->start_addr); const_cast(*region).set_start_addr(end_addr); } else if (start_addr > region->start_addr && start_addr < region->end_addr) { // cut from end RAW_VLOG(12, "End-chopping region %p..%p", reinterpret_cast(region->start_addr), reinterpret_cast(region->end_addr)); + RecordRegionRemovalInBucket(region->call_stack_depth, region->call_stack, + region->end_addr - start_addr); // Can't just modify region->end_addr (it's the sorting key): Region r = *region; r.set_end_addr(start_addr); @@ -580,6 +728,16 @@ void MemoryRegionMap::RecordRegionRemoval(const void* start, size_t size) { Unlock(); } +void MemoryRegionMap::RecordRegionRemovalInBucket(int depth, + const void* const stack[], + size_t size) { + RAW_CHECK(LockIsHeld(), "should be held (by this thread)"); + if (bucket_table_ == NULL) return; + HeapProfileBucket* b = GetBucket(depth, stack); + ++b->frees; + b->free_size += size; +} + void MemoryRegionMap::MmapHook(const void* result, const void* start, size_t size, int prot, int flags, diff --git a/src/memory_region_map.h b/src/memory_region_map.h index 988ea70..be191a9 100644 --- a/src/memory_region_map.h +++ b/src/memory_region_map.h @@ -45,6 +45,7 @@ #include "base/spinlock.h" #include "base/thread_annotations.h" #include "base/low_level_alloc.h" +#include "heap-profile-stats.h" // TODO(maxim): add a unittest: // execute a bunch of mmaps and compare memory map what strace logs @@ -72,6 +73,10 @@ class MemoryRegionMap { // don't take the address of it! static const int kMaxStackDepth = 32; + // Size of the hash table of buckets. A structure of the bucket table is + // described in heap-profile-stats.h. + static const int kHashTableSize = 179999; + public: // interface ================================================================ @@ -87,11 +92,14 @@ class MemoryRegionMap { // are automatically shrunk to "max_stack_depth" when they are recorded. // Init() can be called more than once w/o harm, largest max_stack_depth // will be the effective one. + // When "use_buckets" is true, then counts of mmap and munmap sizes will be + // recorded with each stack trace. If Init() is called more than once, then + // counting will be effective after any call contained "use_buckets" of true. // It will install mmap, munmap, mremap, sbrk hooks // and initialize arena_ and our hook and locks, hence one can use // MemoryRegionMap::Lock()/Unlock() to manage the locks. // Uses Lock/Unlock inside. - static void Init(int max_stack_depth); + static void Init(int max_stack_depth, bool use_buckets); // Try to shutdown this module undoing what Init() did. // Returns true iff could do full shutdown (or it was not attempted). @@ -99,6 +107,10 @@ class MemoryRegionMap { // the number of Init() calls. static bool Shutdown(); + // Return true if MemoryRegionMap is initialized and recording, i.e. when + // then number of Init() calls are more than the number of Shutdown() calls. + static bool IsRecordingLocked(); + // Locks to protect our internal data structures. // These also protect use of arena_ if our Init() has been done. // The lock is recursive. @@ -214,6 +226,18 @@ class MemoryRegionMap { // Returns success. Uses Lock/Unlock inside. static bool FindAndMarkStackRegion(uintptr_t stack_top, Region* result); + // Iterate over the buckets which store mmap and munmap counts per stack + // trace. It calls "callback" for each bucket, and passes "arg" to it. + template + static void IterateBuckets(void (*callback)(const HeapProfileBucket*, Type), + Type arg); + + // Get the bucket whose caller stack trace is "key". The stack trace is + // used to a depth of "depth" at most. The requested bucket is created if + // needed. + // The bucket table is described in heap-profile-stats.h. + static HeapProfileBucket* GetBucket(int depth, const void* const key[]); + private: // our internal types ============================================== // Region comparator for sorting with STL @@ -280,7 +304,7 @@ class MemoryRegionMap { // simply by acquiring our recursive Lock() before that. static RegionSet* regions_; - // Lock to protect regions_ variable and the data behind. + // Lock to protect regions_ and buckets_ variables and the data behind. static SpinLock lock_; // Lock to protect the recursive lock itself. static SpinLock owner_lock_; @@ -295,6 +319,30 @@ class MemoryRegionMap { // Total size of all unmapped pages so far static int64 unmap_size_; + // Bucket hash table which is described in heap-profile-stats.h. + static HeapProfileBucket** bucket_table_ GUARDED_BY(lock_); + static int num_buckets_ GUARDED_BY(lock_); + + // The following members are local to MemoryRegionMap::GetBucket() + // and MemoryRegionMap::HandleSavedBucketsLocked() + // and are file-level to ensure that they are initialized at load time. + // + // These are used as temporary storage to break the infinite cycle of mmap + // calling our hook which (sometimes) causes mmap. It must be a static + // fixed-size array. The size 20 is just an expected value for safety. + // The details are described in memory_region_map.cc. + + // Number of unprocessed bucket inserts. + static int saved_buckets_count_ GUARDED_BY(lock_); + + // Unprocessed inserts (must be big enough to hold all mmaps that can be + // caused by a GetBucket call). + // Bucket has no constructor, so that c-tor execution does not interfere + // with the any-time use of the static memory behind saved_buckets. + static HeapProfileBucket saved_buckets_[20] GUARDED_BY(lock_); + + static const void* saved_buckets_keys_[20][kMaxStackDepth] GUARDED_BY(lock_); + // helpers ================================================================== // Helper for FindRegion and FindAndMarkStackRegion: @@ -308,6 +356,11 @@ class MemoryRegionMap { // by calling insert_func on them. inline static void HandleSavedRegionsLocked( void (*insert_func)(const Region& region)); + + // Restore buckets saved in a tmp static array by GetBucket to the bucket + // table where all buckets eventually should be. + static void RestoreSavedBucketsLocked(); + // Wrapper around DoInsertRegionLocked // that handles the case of recursive allocator calls. inline static void InsertRegionLocked(const Region& region); @@ -319,6 +372,13 @@ class MemoryRegionMap { // (called from our munmap/mremap/sbrk hooks). static void RecordRegionRemoval(const void* start, size_t size); + // Record deletion of a memory region of size "size" in a bucket whose + // caller stack trace is "key". The stack trace is used to a depth of + // "depth" at most. + static void RecordRegionRemovalInBucket(int depth, + const void* const key[], + size_t size); + // Hooks for MallocHook static void MmapHook(const void* result, const void* start, size_t size, @@ -337,4 +397,16 @@ class MemoryRegionMap { DISALLOW_COPY_AND_ASSIGN(MemoryRegionMap); }; +template +void MemoryRegionMap::IterateBuckets( + void (*callback)(const HeapProfileBucket*, Type), Type callback_arg) { + for (int index = 0; index < kHashTableSize; index++) { + for (HeapProfileBucket* bucket = bucket_table_[index]; + bucket != NULL; + bucket = bucket->next) { + callback(bucket, callback_arg); + } + } +} + #endif // BASE_MEMORY_REGION_MAP_H_