* Make kHideMask use all 64 bits (ppluzhnikov)

* Add new IsDebuggerAttached method (ppluzhnikov)
	* Document some tricks for maybe getting perftools to work on OS X
	* Redo file-top pprof commands (csilvers)
	* Clean up pprof input-file handling (csilvers)
	* 16-byte align debug allocs (jyasskin)
	* Ignore JVM memory leakage in the heap checker (davidyu, kkurimoto)
	* Better internal-function list for contentionz (ruemmler)
	* mmap2 on i386 takes an off_t, not an off64_t (csilvers)
	* Fix up fake-VDSO handling for unittest (ppluzhnikov)
	* Don't try to check valgrind for windows (csilvers)


git-svn-id: http://gperftools.googlecode.com/svn/trunk@101 6b5cf1ce-ec42-a296-1ba9-69fdba395a50
This commit is contained in:
csilvers 2011-01-19 21:37:15 +00:00
parent a0a2ff3b49
commit 3d77cbf7d5
13 changed files with 188 additions and 88 deletions

22
README
View File

@ -196,6 +196,28 @@ that error. To fix it, just comment out (or delete) the line
in your config.h file before building.
OS X ISSUES
-----------
You may need to set the environment variable DYLD_FORCE_FLAT_NAMESPACE
to use perftools with OS X. Because of how OS X does symbol binding,
libc routines will use libc malloc even when the binary is linked with
-ltcmalloc. This is not usually a problem, but becomes one if the
application is responsible for freeing that memory: the application
will use tcmalloc's free() to try to free memory allocated with libc's
malloc(), which will cause no end of confusion.
One (or both) of these workaround may fix the problem:
DYLD_FORCE_FLAT_NAMESPACE=1 myapp
DYLD_INSERT_LIBRARIES=path/to/libtcmalloc.dylib myapp
The best solution may depend on the version of OS X being used.
Neither solution is likely to work if you dlopen() libraries from
within your application. If you have any experience with this, we'd
appreciate you sharing it at
http://groups.google.com/group/google-perftools
64-BIT ISSUES
-------------

View File

@ -127,13 +127,23 @@ static int GetRunningOnValgrind(void) {
#ifdef RUNNING_ON_VALGRIND
if (RUNNING_ON_VALGRIND) return 1;
#endif
// TODO(csilvers): use GetenvBeforeMain() instead? Will need to
// change it to be extern "C".
#ifdef _MSC_VER
/* Visual Studio can complain about getenv, so use a windows equivalent. */
char value[100] = "1"; /* something that is not "0" */
int res = GetEnvironmentVariableA("RUNNING_ON_VALGRIND",
value, sizeof(value));
/* value will remain "1" if the called failed for some reason. */
return (res > 0 && strcmp(value, "0") != 0);
#else
/* TODO(csilvers): use GetenvBeforeMain() instead? Will need to
* change it to be extern "C".
*/
char *running_on_valgrind_str = getenv("RUNNING_ON_VALGRIND");
if (running_on_valgrind_str) {
return strcmp(running_on_valgrind_str, "0") != 0;
}
return 0;
#endif
}
/* See the comments in dynamic_annotations.h */

View File

@ -2593,7 +2593,7 @@ struct kernel_statfs {
LSS_INLINE _syscall6(void*, mmap2, void*, s,
size_t, l, int, p,
int, f, int, d,
__off64_t, o)
off_t, o)
#endif
LSS_INLINE _syscall3(int, _sigaction, int, s,
const struct kernel_old_sigaction*, a,

View File

@ -207,6 +207,10 @@ void VDSOSupport::ElfMemImage::Init(const void *base) {
if (!base) {
return;
}
const intptr_t base_as_uintptr_t = reinterpret_cast<uintptr_t>(base);
// Fake VDSO has low bit set.
const bool fake_vdso = ((base_as_uintptr_t & 1) != 0);
base = reinterpret_cast<const void *>(base_as_uintptr_t & ~1);
const char *const base_as_char = reinterpret_cast<const char *>(base);
if (base_as_char[EI_MAG0] != ELFMAG0 || base_as_char[EI_MAG1] != ELFMAG1 ||
base_as_char[EI_MAG2] != ELFMAG2 || base_as_char[EI_MAG3] != ELFMAG3) {
@ -266,17 +270,6 @@ void VDSOSupport::ElfMemImage::Init(const void *base) {
ElfW(Dyn) *dynamic_entry =
reinterpret_cast<ElfW(Dyn) *>(dynamic_program_header->p_vaddr +
relocation);
bool fake_vdso = false; // Assume we are dealing with the real VDSO.
for (ElfW(Dyn) *de = dynamic_entry; de->d_tag != DT_NULL; ++de) {
ElfW(Sxword) tag = de->d_tag;
if (tag == DT_PLTGOT || tag == DT_RELA || tag == DT_JMPREL ||
tag == DT_NEEDED || tag == DT_RPATH || tag == DT_VERNEED ||
tag == DT_INIT || tag == DT_FINI) {
/* Real vdso can not reasonably have any of the above entries. */
fake_vdso = true;
break;
}
}
for (; dynamic_entry->d_tag != DT_NULL; ++dynamic_entry) {
ElfW(Xword) value = dynamic_entry->d_un.d_val;
if (fake_vdso) {

View File

@ -147,6 +147,10 @@ class VDSOSupport {
// kInvalidBase => value hasn't been determined yet.
// 0 => there is no VDSO.
// else => vma of VDSO Elf{32,64}_Ehdr.
//
// When testing with mock VDSO, low bit is set.
// The low bit is always available because vdso_base_ is
// page-aligned.
static const void *vdso_base_;
// NOLINT on 'long' because these routines mimic kernel api.

View File

@ -273,12 +273,13 @@ class MallocBlock {
// NOTE: tcmalloc.cc depends on the value of kMagicDeletedByte
// to work around a bug in the pthread library.
static const int kMagicDeletedByte = 0xCD;
// An int (type of alloc_type_ below) in a deallocated storage
// A size_t (type of alloc_type_ below) in a deallocated storage
// filled with kMagicDeletedByte.
static const int kMagicDeletedInt = 0xCDCDCDCD | ((0xCDCDCDCD << 16) << 16);
// Initializer works for 32 and 64 bit ints;
static const size_t kMagicDeletedSizeT =
0xCDCDCDCD | (((size_t)0xCDCDCDCD << 16) << 16);
// Initializer works for 32 and 64 bit size_ts;
// "<< 16 << 16" is to fool gcc from issuing a warning
// when ints are 32 bits.
// when size_ts are 32 bits.
// NOTE: on Linux, you can enable malloc debugging support in libc by
// setting the environment variable MALLOC_CHECK_ to 1 before you
@ -297,12 +298,17 @@ class MallocBlock {
private: // data layout
// The four fields size1_,offset_,magic1_,alloc_type_
// should together occupy a multiple of 8 bytes.
// should together occupy a multiple of 16 bytes. (At the
// moment, sizeof(size_t) == 4 or 8 depending on piii vs
// k8, and 4 of those sum to 16 or 32 bytes).
// This, combined with BASE_MALLOC's alignment guarantees,
// ensures that SSE types can be stored into the returned
// block, at &size2_.
size_t size1_;
size_t offset_; // normally 0 unless memaligned memory
// see comments in memalign() and FromRawPointer().
int magic1_;
int alloc_type_;
size_t magic1_;
size_t alloc_type_;
// here comes the actual data (variable length)
// ...
// then come the size2_ and magic2_, or a full page of mprotect-ed memory
@ -435,7 +441,7 @@ class MallocBlock {
"has been already deallocated (it was allocated with %s)",
data_addr(), AllocName(map_type & ~kDeallocatedTypeBit));
}
if (alloc_type_ == kMagicDeletedInt) {
if (alloc_type_ == kMagicDeletedSizeT) {
RAW_LOG(FATAL, "memory stomping bug: a word before object at %p "
"has been corrupted; or else the object has been already "
"deallocated and our memory map has been corrupted",
@ -701,8 +707,8 @@ class MallocBlock {
// Find the header just before client's memory.
MallocBlock *mb = reinterpret_cast<MallocBlock *>(
reinterpret_cast<char *>(p) - data_offset);
// If mb->alloc_type_ is kMagicDeletedInt, we're not an ok pointer.
if (mb->alloc_type_ == kMagicDeletedInt) {
// If mb->alloc_type_ is kMagicDeletedSizeT, we're not an ok pointer.
if (mb->alloc_type_ == kMagicDeletedSizeT) {
RAW_LOG(FATAL, "memory allocation bug: object at %p has been already"
" deallocated; or else a word before the object has been"
" corrupted (memory stomping bug)", p);

View File

@ -106,6 +106,32 @@ using std::max;
using std::less;
using std::char_traits;
// If current process is being ptrace()d, 'TracerPid' in /proc/self/status
// will be non-zero.
static bool IsDebuggerAttached(void) { // only works under linux, probably
// Since we could be called from FailureSignalHandler, avoid stdio
// which could have been corrupted.
// Limit stack usage as well. Currently, TracerPid is at max offset 76
// (depending on length of argv[0]) into /proc/self/status.
char buf[100];
int fd = open("/proc/self/status", O_RDONLY);
if (fd == -1) {
return false; // Can't tell for sure.
}
const int len = read(fd, buf, sizeof(buf));
bool rc = false;
if (len > 0) {
const char *const kTracerPid = "TracerPid:\t";
buf[len - 1] = '\0';
const char *p = strstr(buf, kTracerPid);
if (p != NULL) {
rc = (strncmp(p + strlen(kTracerPid), "0\n", 2) != 0);
}
}
close(fd);
return rc;
}
// This is the default if you don't link in -lprofiler
extern "C" {
ATTRIBUTE_WEAK PERFTOOLS_DLL_DECL bool ProfilingIsEnabledForAllThreads();
@ -798,7 +824,7 @@ void HeapLeakChecker::DisableLibraryAllocsLocked(const char* library,
// pthread_setspecific (which can be the only pointer to a heap object).
IsLibraryNamed(library, "/libdl") ||
// library loaders leak some "system" heap that we don't care about
IsLibraryNamed(library, "/libcrypto")
IsLibraryNamed(library, "/libcrypto") ||
// Sometimes libcrypto of OpenSSH is compiled with -fomit-frame-pointer
// (any library can be, of course, but this one often is because speed
// is so important for making crypto usable). We ignore all its
@ -806,6 +832,10 @@ void HeapLeakChecker::DisableLibraryAllocsLocked(const char* library,
// to ignore allocations done in files/symbols that match
// "default_malloc_ex|default_realloc_ex"
// but that doesn't work when the end-result binary is stripped.
IsLibraryNamed(library, "/libjvm") ||
// JVM has a lot of leaks we don't care about.
IsLibraryNamed(library, "/libzip")
// The JVM leaks java.util.zip.Inflater after loading classes.
) {
depth = 1; // only disable allocation calls directly from the library code
} else if (IsLibraryNamed(library, "/ld")
@ -1637,6 +1667,13 @@ bool HeapLeakChecker::DoNoLeaks(ShouldSymbolize should_symbolize) {
return true;
}
// Update global_region_caller_ranges. They may need to change since
// e.g. initialization because shared libraries might have been loaded or
// unloaded.
Allocator::DeleteAndNullIfNot(&global_region_caller_ranges);
ProcMapsResult pm_result = UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS);
RAW_CHECK(pm_result == PROC_MAPS_USED, "");
// Keep track of number of internally allocated objects so we
// can detect leaks in the heap-leak-checket itself
const int initial_allocs = Allocator::alloc_count();
@ -1867,25 +1904,11 @@ static bool internal_init_start_has_run = false;
}
// Changing this to false can be useful when debugging heap-checker itself:
if (!FLAGS_heap_check_run_under_gdb) {
// See if heap checker should turn itself off because we are
// running under gdb (to avoid conflicts over ptrace-ing rights):
char name_buf[15+15];
snprintf(name_buf, sizeof(name_buf),
"/proc/%d/cmdline", static_cast<int>(getppid()));
char cmdline[1024*8]; // /proc/*/cmdline is at most 4Kb anyway usually
int size = GetCommandLineFrom(name_buf, cmdline, sizeof(cmdline)-1);
cmdline[size] = '\0';
// look for "gdb" in the executable's name:
const char* last = strrchr(cmdline, '/');
if (last) last += 1;
else last = cmdline;
if (strncmp(last, "gdb", 3) == 0) {
RAW_LOG(WARNING, "We seem to be running under gdb; will turn itself off");
SpinLockHolder l(&heap_checker_lock);
TurnItselfOffLocked();
return;
}
if (!FLAGS_heap_check_run_under_gdb && IsDebuggerAttached()) {
RAW_LOG(WARNING, "Someone is ptrace()ing us; will turn itself off");
SpinLockHolder l(&heap_checker_lock);
TurnItselfOffLocked();
return;
}
{ SpinLockHolder l(&heap_checker_lock);

View File

@ -412,7 +412,8 @@ static inline void* do_mmap64(void *start, size_t length,
}
result = (void *)syscall(SYS_mmap2,
start, length, prot, flags, fd, offset / pagesize);
start, length, prot, flags, fd,
(off_t) (offset / pagesize));
if (result != MAP_FAILED || errno != ENOSYS) goto out;
// We don't have mmap2() after all - don't bother trying it in future

103
src/pprof
View File

@ -2437,7 +2437,16 @@ sub RemoveUninterestingFrames {
# old code out of the system.
$skip_regexp = "TCMalloc|^tcmalloc::";
} elsif ($main::profile_type eq 'contention') {
foreach my $vname ('Mutex::Unlock', 'Mutex::UnlockSlow') {
foreach my $vname ('base::RecordLockProfileData',
'base::SubmitMutexProfileData',
'base::SubmitSpinLockProfileData',
'Mutex::Unlock',
'Mutex::UnlockSlow',
'Mutex::ReaderUnlock',
'MutexLock::~MutexLock',
'SpinLock::Unlock',
'SpinLock::SlowUnlock',
'SpinLockHolder::~SpinLockHolder') {
$skip{$vname} = 1;
}
} elsif ($main::profile_type eq 'cpu') {
@ -3176,24 +3185,47 @@ BEGIN {
}
}
# Return the next line from the profile file, assuming it's a text
# line (which in this case means, doesn't start with a NUL byte). If
# it's not a text line, return "". At EOF, return undef, like perl does.
# Input file should be in binmode.
sub ReadProfileLine {
# Reads the top, 'header' section of a profile, and returns the last
# line of the header, commonly called a 'header line'. The header
# section of a profile consists of zero or more 'command' lines that
# are instructions to pprof, which pprof executes when reading the
# header. All 'command' lines start with a %. After the command
# lines is the 'header line', which is a profile-specific line that
# indicates what type of profile it is, and perhaps other global
# information about the profile. For instance, here's a header line
# for a heap profile:
# heap profile: 53: 38236 [ 5525: 1284029] @ heapprofile
# For historical reasons, the CPU profile does not contain a text-
# readable header line. If the profile looks like a CPU profile,
# this function returns "". If no header line could be found, this
# function returns undef.
#
# The following commands are recognized:
# %warn -- emit the rest of this line to stderr, prefixed by 'WARNING:'
#
# The input file should be in binmode.
sub ReadProfileHeader {
local *PROFILE = shift;
my $firstchar = "";
my $line = "";
read(PROFILE, $firstchar, 1);
seek(PROFILE, -1, 1); # unread the firstchar
if ($firstchar eq "\0") {
seek(PROFILE, -1, 1); # unread the firstchar
if ($firstchar !~ /[[:print:]]/) { # is not a text character
return "";
}
$line = <PROFILE>;
if (defined($line)) {
while (defined($line = <PROFILE>)) {
$line =~ s/\r//g; # turn windows-looking lines into unix-looking lines
if ($line =~ /^%warn\s+(.*)/) { # 'warn' command
# Note this matches both '%warn blah\n' and '%warn\n'.
print STDERR "WARNING: $1\n"; # print the rest of the line
} elsif ($line =~ /^%/) {
print STDERR "Ignoring unknown command from profile header: $line";
} else {
# End of commands, must be the header line.
return $line;
}
}
return $line;
return undef; # got to EOF without seeing a header line
}
sub IsSymbolizedProfileFile {
@ -3204,7 +3236,7 @@ sub IsSymbolizedProfileFile {
# Check if the file contains a symbol-section marker.
open(TFILE, "<$file_name");
binmode TFILE;
my $firstline = ReadProfileLine(*TFILE);
my $firstline = ReadProfileHeader(*TFILE);
close(TFILE);
if (!$firstline) {
return 0;
@ -3224,14 +3256,7 @@ sub IsSymbolizedProfileFile {
sub ReadProfile {
my $prog = shift;
my $fname = shift;
if (IsSymbolizedProfileFile($fname) && !$main::use_symbolized_profile) {
# we have both a binary and symbolized profiles, abort
usage("Symbolized profile '$fname' cannot be used with a binary arg. " .
"Try again without passing '$prog'.");
}
$main::profile_type = '';
my $result; # return value
$CONTENTION_PAGE =~ m,[^/]+$,; # matches everything after the last slash
my $contention_marker = $&;
@ -3248,40 +3273,45 @@ sub ReadProfile {
# whole firstline, since it may be gigabytes(!) of data.
open(PROFILE, "<$fname") || error("$fname: $!\n");
binmode PROFILE; # New perls do UTF-8 processing
my $header = ReadProfileLine(*PROFILE);
my $header = ReadProfileHeader(*PROFILE);
if (!defined($header)) { # means "at EOF"
error("Profile is empty.\n");
}
my $symbols;
if ($header =~ m/^--- *$symbol_marker/o) {
# Verify that the user asked for a symbolized profile
if (!$main::use_symbolized_profile) {
# we have both a binary and symbolized profiles, abort
error("FATAL ERROR: Symbolized profile\n $fname\ncannot be used with " .
"a binary arg. Try again without passing\n $prog\n");
}
# Read the symbol section of the symbolized profile file.
$symbols = ReadSymbols(*PROFILE{IO});
# Read the next line to get the header for the remaining profile.
$header = ReadProfileLine(*PROFILE) || "";
$header = ReadProfileHeader(*PROFILE) || "";
}
my $result;
$main::profile_type = '';
if ($header =~ m/^heap profile:.*$growth_marker/o) {
$main::profile_type = 'growth';
$result = ReadHeapProfile($prog, $fname, $header);
$result = ReadHeapProfile($prog, *PROFILE, $header);
} elsif ($header =~ m/^heap profile:/) {
$main::profile_type = 'heap';
$result = ReadHeapProfile($prog, $fname, $header);
$result = ReadHeapProfile($prog, *PROFILE, $header);
} elsif ($header =~ m/^--- *$contention_marker/o) {
$main::profile_type = 'contention';
$result = ReadSynchProfile($prog, $fname);
$result = ReadSynchProfile($prog, *PROFILE);
} elsif ($header =~ m/^--- *Stacks:/) {
print STDERR
"Old format contention profile: mistakenly reports " .
"condition variable signals as lock contentions.\n";
$main::profile_type = 'contention';
$result = ReadSynchProfile($prog, $fname);
$result = ReadSynchProfile($prog, *PROFILE);
} elsif ($header =~ m/^--- *$profile_marker/) {
# the binary cpu profile data starts immediately after this line
$main::profile_type = 'cpu';
$result = ReadCPUProfile($prog, $fname);
$result = ReadCPUProfile($prog, $fname, *PROFILE);
} else {
if (defined($symbols)) {
# a symbolized profile contains a format we don't recognize, bail out
@ -3289,9 +3319,11 @@ sub ReadProfile {
}
# no ascii header present -- must be a CPU profile
$main::profile_type = 'cpu';
$result = ReadCPUProfile($prog, $fname);
$result = ReadCPUProfile($prog, $fname, *PROFILE);
}
close(PROFILE);
# if we got symbols along with the profile, return those as well
if (defined($symbols)) {
$result->{symbols} = $symbols;
@ -3330,7 +3362,8 @@ sub FixCallerAddresses {
# CPU profile reader
sub ReadCPUProfile {
my $prog = shift;
my $fname = shift;
my $fname = shift; # just used for logging
local *PROFILE = shift;
my $version;
my $period;
my $i;
@ -3397,7 +3430,6 @@ sub ReadCPUProfile {
my $map = '';
seek(PROFILE, $i * 4, 0);
read(PROFILE, $map, (stat PROFILE)[7]);
close(PROFILE);
my $r = {};
$r->{version} = $version;
@ -3411,7 +3443,7 @@ sub ReadCPUProfile {
sub ReadHeapProfile {
my $prog = shift;
my $fname = shift;
local *PROFILE = shift;
my $header = shift;
my $index = 1;
@ -3596,7 +3628,9 @@ sub ReadHeapProfile {
}
sub ReadSynchProfile {
my ($prog, $fname, $header) = @_;
my $prog = shift;
local *PROFILE = shift;
my $header = shift;
my $map = '';
my $profile = {};
@ -3671,7 +3705,6 @@ sub ReadSynchProfile {
$map .= $line;
}
}
close PROFILE;
if (!$seen_clockrate) {
printf STDERR ("No cycles/second entry in profile; Guessing %.1f GHz\n",

View File

@ -630,11 +630,13 @@ class TCMallocImplementation : public MallocExtension {
virtual void GetHeapSample(MallocExtensionWriter* writer) {
if (FLAGS_tcmalloc_sample_parameter == 0) {
const char* const kWarningMsg =
"#\n# WARNING: This heap profile does not have any data in it,\n"
"# because the application was run with heap sampling turned off.\n"
"# To get useful data from from GetHeapSample(), you must first\n"
"# set the environment variable TCMALLOC_SAMPLE_PARAMETER to a\n"
"# positive sampling period, such as 524288.\n#\n";
"%warn\n"
"%warn This heap profile does not have any data in it, because\n"
"%warn the application was run with heap sampling turned off.\n"
"%warn To get useful data from GetHeapSample(), you must\n"
"%warn set the environment variable TCMALLOC_SAMPLE_PARAMETER to\n"
"%warn a positive sampling period, such as 524288.\n"
"%warn\n";
writer->append(kWarningMsg, strlen(kWarningMsg));
}
MallocExtension::GetHeapSample(writer);

View File

@ -291,7 +291,8 @@ static void Use(T** foo) {
// Arbitrary value, but not such that xor'ing with it is likely
// to map one valid pointer to another valid pointer:
static const uintptr_t kHideMask = 0xF03A5F7B;
static const uintptr_t kHideMask =
static_cast<uintptr_t>(0xF03A5F7BF03A5F7BLL);
// Helpers to hide a pointer from live data traversal.
// We just xor the pointer so that (with high probability)

View File

@ -721,11 +721,9 @@ static void TestAlignmentForSize(int size) {
CHECK((p % sizeof(double)) == 0);
// Must have 16-byte alignment for large enough objects
#ifndef DEBUGALLOCATION // debug allocation doesn't need to align like this
if (size >= 16) {
CHECK((p % 16) == 0);
}
#endif
}
for (int i = 0; i < kNum; i++) {
free(ptrs[i]);

View File

@ -136,6 +136,9 @@
/* Define to 1 if the system has the type `struct mallinfo'. */
#undef HAVE_STRUCT_MALLINFO
/* Define to 1 if you have the <sys/param.h> header file. */
#undef HAVE_SYS_PARAM_H
/* Define to 1 if you have the <sys/prctl.h> header file. */
#undef HAVE_SYS_PRCTL_H
@ -190,6 +193,10 @@
/* Define to 1 if int32_t is equivalent to intptr_t */
#undef INT32_EQUALS_INTPTR
/* Define to the sub-directory in which libtool stores uninstalled libraries.
*/
#undef LT_OBJDIR
/* Define to 1 if your C compiler doesn't accept -c and -o together. */
#undef NO_MINUS_C_MINUS_O