// Copyright (c) 2005, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// ---
// All Rights Reserved.
//
// Author: Maxim Lifantsev
//
#include "config.h"
#include <fcntl.h> // for O_RDONLY (we use syscall to do actual reads)
#include <string.h>
#include <errno.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_MMAP
#include <sys/mman.h>
#endif
#include <sys/poll.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <assert.h>
#ifdef HAVE_LINUX_PTRACE_H
#include <linux/ptrace.h>
#endif
#ifdef HAVE_SYS_SYSCALL_H
#include <sys/syscall.h>
#endif
#include <string>
#include <vector>
#include <map>
#include <set>
#include <algorithm>
#include <functional>
#include <google/heap-checker.h>
#include "base/basictypes.h"
#include "base/googleinit.h"
#include "base/logging.h"
#include <google/stacktrace.h>
#include "base/commandlineflags.h"
#include "base/elfcore.h" // for i386_regs
#include "base/thread_lister.h"
#include "heap-profile-table.h"
#include "base/low_level_alloc.h"
#include <google/malloc_hook.h>
#include <google/malloc_extension.h>
#include "memory_region_map.h"
#include "base/spinlock.h"
#include "base/sysinfo.h"
#include "base/stl_allocator.h"
using std::string;
using std::basic_string;
using std::pair;
using std::map;
using std::set;
using std::vector;
using std::swap;
using std::make_pair;
using std::min;
using std::max;
using std::less;
using std::char_traits;
//----------------------------------------------------------------------
// Flags that control heap-checking
//----------------------------------------------------------------------
DEFINE_string(heap_check,
EnvToString("HEAPCHECK", ""),
"The heap leak checking to be done over the whole executable: "
"\"minimal\", \"normal\", \"strict\", "
"\"draconian\", \"as-is\", and \"local\" "
" or the empty string are the supported choices. "
"(See HeapLeakChecker::InternalInitStart for details.)");
DEFINE_bool(heap_check_report,
EnvToBool("HEAP_CHECK_REPORT", true),
"If overall heap check should report the found leaks via pprof");
DEFINE_bool(heap_check_before_constructors,
true,
"deprecated; pretty much always true now");
DEFINE_bool(heap_check_after_destructors,
EnvToBool("HEAP_CHECK_AFTER_DESTRUCTORS", false),
"If overall heap check is to end after global destructors "
"or right after all REGISTER_HEAPCHECK_CLEANUP's");
DEFINE_bool(heap_check_strict_check,
EnvToBool("HEAP_CHECK_STRICT_CHECK", true),
"If overall heap check is to be done "
"via HeapLeakChecker::*SameHeap "
"or HeapLeakChecker::*NoLeaks call");
// heap_check_strict_check == false
// is useful only when heap_check_before_constructors == false
DEFINE_bool(heap_check_ignore_global_live,
EnvToBool("HEAP_CHECK_IGNORE_GLOBAL_LIVE", true),
"If overall heap check is to ignore heap objects reachable "
"from the global data");
DEFINE_bool(heap_check_identify_leaks,
EnvToBool("HEAP_CHECK_IDENTIFY_LEAKS", false),
"If heap check should generate the addresses of the leaked objects "
"in the memory leak profiles");
DEFINE_bool(heap_check_ignore_thread_live,
EnvToBool("HEAP_CHECK_IGNORE_THREAD_LIVE", true),
"If set to true, objects reachable from thread stacks "
"and registers are not reported as leaks");
DEFINE_bool(heap_check_test_pointer_alignment,
EnvToBool("HEAP_CHECK_TEST_POINTER_ALIGNMENT", false),
"Set to true to check if the found leak can be due to "
"use of unaligned pointers");
DEFINE_bool(heap_check_run_under_gdb,
EnvToBool("HEAP_CHECK_RUN_UNDER_GDB", false),
"If false, turns off heap-checking library when running under gdb "
"(normally, set to 'true' only when debugging the heap-checker)");
//----------------------------------------------------------------------
DEFINE_string(heap_profile_pprof,
EnvToString("PPROF_PATH", "pprof"),
"Path to pprof to call for full leak checking.");
DEFINE_string(heap_check_dump_directory,
EnvToString("HEAP_CHECK_DUMP_DIRECTORY", "/tmp"),
"Directory to put heap-checker leak dump information");
// Copy of FLAGS_heap_profile_pprof.
// Need this since DoNoLeaks can happen
// after FLAGS_heap_profile_pprof is destroyed.
static string* flags_heap_profile_pprof = &FLAGS_heap_profile_pprof;
//----------------------------------------------------------------------
// HeapLeakChecker global data
//----------------------------------------------------------------------
// Global lock for (most of) the global data of this module.
// We could use pthread's lock here, but spinlock is faster.
static SpinLock heap_checker_lock(SpinLock::LINKER_INITIALIZED);
//----------------------------------------------------------------------
// Heap profile prefix for leak checking profiles
static string* profile_prefix = NULL;
// Whole-program heap leak checker
static HeapLeakChecker* main_heap_checker = NULL;
// Whether we will use main_heap_checker to do a check at program exit
static bool do_main_heap_check = false;
// The heap profile we use to collect info about the heap.
static HeapProfileTable* heap_profile = NULL;
// If we are doing (or going to do) any kind of heap-checking.
static bool heap_checker_on = false;
// pid of the process that does whole-program heap leak checking
static pid_t heap_checker_pid = 0;
// If we did heap profiling during global constructors execution
static bool constructor_heap_profiling = false;
//----------------------------------------------------------------------
// HeapLeakChecker's own memory allocator that is
// independent of the normal program allocator.
//----------------------------------------------------------------------
// Wrapper of LowLevelAlloc for STL_Allocator and direct use.
// We always access Allocate/Free in this class under held heap_checker_lock,
// this allows us to protect the period when threads are stopped
// at random spots with ListAllProcessThreads by heap_checker_lock,
// w/o worrying about the lock in LowLevelAlloc::Arena.
// We rely on the fact that we use an own arena with an own lock here.
class HeapLeakChecker::Allocator {
public:
static void Init() {
RAW_DCHECK(arena_ == NULL, "");
arena_ = LowLevelAlloc::NewArena(0, LowLevelAlloc::DefaultArena());
}
static void Shutdown() {
if (!LowLevelAlloc::DeleteArena(arena_) || alloc_count_ != 0) {
RAW_LOG(FATAL, "Internal heap checker leak of %d objects", alloc_count_);
}
}
static int alloc_count() { return alloc_count_; }
static void* Allocate(size_t n) {
RAW_DCHECK(arena_ && heap_checker_lock.IsHeld(), "");
void* p = LowLevelAlloc::AllocWithArena(n, arena_);
if (p) alloc_count_ += 1;
return p;
}
static void Free(void* p) {
RAW_DCHECK(heap_checker_lock.IsHeld(), "");
if (p) alloc_count_ -= 1;
LowLevelAlloc::Free(p);
}
// destruct, free, and make *p to be NULL
template<typename T> static void DeleteAndNull(T** p) {
(*p)->~T();
Free(*p);
*p = NULL;
}
template<typename T> static void DeleteAndNullIfNot(T** p) {
if (*p != NULL) DeleteAndNull(p);
}
private:
static LowLevelAlloc::Arena* arena_;
static int alloc_count_;
};
LowLevelAlloc::Arena* HeapLeakChecker::Allocator::arena_ = NULL;
int HeapLeakChecker::Allocator::alloc_count_ = 0;
//----------------------------------------------------------------------
// HeapLeakChecker live object tracking components
//----------------------------------------------------------------------
// Cases of live object placement we distinguish
enum ObjectPlacement {
MUST_BE_ON_HEAP, // Must point to a live object of the matching size in the
// heap_profile map of the heap when we get to it
IGNORED_ON_HEAP, // Is a live (ignored) object on heap
MAYBE_LIVE, // Is simply a piece of writable memory from /proc/self/maps
IN_GLOBAL_DATA, // Is part of global data region of the executable
THREAD_DATA, // Part of a thread stack (and a thread descriptor with TLS)
THREAD_REGISTERS, // Values in registers of some thread
};
// Information about an allocated object
struct AllocObject {
const void* ptr; // the object
uintptr_t size; // its size
ObjectPlacement place; // where ptr points to
AllocObject(const void* p, size_t s, ObjectPlacement l)
: ptr(p), size(s), place(l) { }
};
typedef basic_string<char, char_traits<char>,
STL_Allocator<char, HeapLeakChecker::Allocator>
> HCL_string;
// the disabled regexp accumulated
// via HeapLeakChecker::DisableChecksIn
static HCL_string* disabled_regexp = NULL;
// All objects (memory ranges) ignored via HeapLeakChecker::IgnoreObject
// Key is the object's address; value is its size.
typedef map<uintptr_t, size_t, less<uintptr_t>,
STL_Allocator<pair<const uintptr_t, size_t>,
HeapLeakChecker::Allocator>
> IgnoredObjectsMap;
static IgnoredObjectsMap* ignored_objects = NULL;
// All objects (memory ranges) that we consider to be the sources of pointers
// to live (not leaked) objects.
// At different times this holds (what can be reached from) global data regions
// and the objects we've been told to ignore.
// For any AllocObject::ptr "live_objects" is supposed to contain at most one
// record at any time. We maintain this by checking with the heap_profile map
// of the heap and removing the live heap objects we've handled from it.
// This vector is maintained as a stack and the frontier of reachable
// live heap objects in our flood traversal of them.
typedef vector<AllocObject,
STL_Allocator<AllocObject, HeapLeakChecker::Allocator>
> LiveObjectsStack;
static LiveObjectsStack* live_objects = NULL;
// A placeholder to fill-in the starting values for live_objects
// for each library so we can keep the library-name association for logging.
typedef map<HCL_string, LiveObjectsStack, less<HCL_string>,
STL_Allocator<pair<const HCL_string, LiveObjectsStack>,
HeapLeakChecker::Allocator>
> LibraryLiveObjectsStacks;
static LibraryLiveObjectsStacks* library_live_objects = NULL;
// Objects to be removed from the heap profile when we dump it.
typedef set<const void*, less<const void*>,
STL_Allocator<const void*, HeapLeakChecker::Allocator>
> ProfileAdjustObjectSet;
static ProfileAdjustObjectSet* profile_adjust_objects = NULL;
// The disabled program counter addresses for profile dumping
// that are registered with HeapLeakChecker::DisableChecksUp
typedef set<uintptr_t, less<uintptr_t>,
STL_Allocator<uintptr_t, HeapLeakChecker::Allocator>
> DisabledAddressSet;
static DisabledAddressSet* disabled_addresses = NULL;
// Value stored in the map of disabled address ranges;
// its key is the end of the address range.
// We'll ignore allocations with a return address in a disabled range
// if the address occurs at 'max_depth' or less in the stack trace.
struct HeapLeakChecker::RangeValue {
uintptr_t start_address; // the start of the range
int max_depth; // the maximal stack depth to disable at
};
typedef map<uintptr_t, HeapLeakChecker::RangeValue, less<uintptr_t>,
STL_Allocator<pair<const uintptr_t, HeapLeakChecker::RangeValue>,
HeapLeakChecker::Allocator>
> DisabledRangeMap;
// The disabled program counter address ranges for profile dumping
// that are registered with HeapLeakChecker::DisableChecksFromToLocked.
static DisabledRangeMap* disabled_ranges = NULL;
// Set of stack tops.
// These are used to consider live only appropriate chunks of the memory areas
// that are used for stacks (and maybe thread-specific data as well)
// so that we do not treat pointers from outdated stack frames as live.
typedef set<uintptr_t, less<uintptr_t>,
STL_Allocator<uintptr_t, HeapLeakChecker::Allocator>
> StackTopSet;
static StackTopSet* stack_tops = NULL;
// A map of ranges of code addresses for the system libraries
// that can mmap/mremap/sbrk-allocate memory regions for stacks
// and thread-local storage that we want to consider as live global data.
// Maps from the end address to the start address.
typedef map<uintptr_t, uintptr_t, less<uintptr_t>,
STL_Allocator<pair<const uintptr_t, uintptr_t>,
HeapLeakChecker::Allocator>
> GlobalRegionCallerRangeMap;
static GlobalRegionCallerRangeMap* global_region_caller_ranges = NULL;
// TODO(maxim): make our big data structs into own modules
//----------------------------------------------------------------------
// Simple hook into execution of global object constructors,
// so that we do not call pthread_self() when it does not yet work.
static bool libpthread_initialized = false;
static bool initializer = (libpthread_initialized = true, true);
// Our hooks for MallocHook
static void NewHook(const void* ptr, size_t size) {
if (ptr != NULL) {
RAW_VLOG(7, "Recording Alloc: %p of %"PRIuS, ptr, size);
heap_checker_lock.Lock();
heap_profile->RecordAlloc(ptr, size, 0);
heap_checker_lock.Unlock();
RAW_VLOG(8, "Alloc Recorded: %p of %"PRIuS"", ptr, size);
}
}
static void DeleteHook(const void* ptr) {
if (ptr != NULL) {
RAW_VLOG(7, "Recording Free %p", ptr);
heap_checker_lock.Lock();
heap_profile->RecordFree(ptr);
heap_checker_lock.Unlock();
RAW_VLOG(8, "Free Recorded: %p", ptr);
}
}
//----------------------------------------------------------------------
enum StackDirection {
GROWS_TOWARDS_HIGH_ADDRESSES,
GROWS_TOWARDS_LOW_ADDRESSES,
UNKNOWN_DIRECTION
};
static StackDirection GetStackDirection(const int* ptr); // defined below
// Function pointer to trick compiler into not inlining a call:
static StackDirection (*do_stack_direction)(const int* ptr) = GetStackDirection;
// Determine which way the stack grows:
// Call with NULL argument.
static StackDirection GetStackDirection(const int* ptr) {
int a_local;
if (ptr == NULL) return do_stack_direction(&a_local);
if (&a_local > ptr) return GROWS_TOWARDS_HIGH_ADDRESSES;
if (&a_local < ptr) return GROWS_TOWARDS_LOW_ADDRESSES;
RAW_CHECK(0, ""); // &a_local == ptr, i.e. the recursive call got inlined
// and we can't do it (need more hoops to prevent inlining)
return UNKNOWN_DIRECTION;
}
// Direction of stack growth (will initialize via GetStackDirection())
static StackDirection stack_direction = UNKNOWN_DIRECTION;
// This routine is called for every thread stack we know about to register it.
static void RegisterStack(const void* top_ptr) {
RAW_VLOG(1, "Thread stack at %p", top_ptr);
uintptr_t top = reinterpret_cast<uintptr_t>(top_ptr);
stack_tops->insert(top); // add for later use
// make sure stack_direction is initialized
if (stack_direction == UNKNOWN_DIRECTION) {
stack_direction = GetStackDirection(NULL);
}
// Find memory region with this stack
MemoryRegionMap::Region region;
if (MemoryRegionMap::FindStackRegion(top, ®ion)) {
// Make the proper portion of the stack live:
if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) {
RAW_VLOG(2, "Live stack at %p of %"PRIuS" bytes",
top_ptr, region.end_addr - top);
live_objects->push_back(AllocObject(top_ptr, region.end_addr - top,
THREAD_DATA));
} else { // GROWS_TOWARDS_HIGH_ADDRESSES
RAW_VLOG(2, "Live stack at %p of %"PRIuS" bytes",
(void*)region.start_addr, top - region.start_addr);
live_objects->push_back(AllocObject((void*)region.start_addr,
top - region.start_addr,
THREAD_DATA));
}
} else { // not in MemoryRegionMap, look in library_live_objects
for (LibraryLiveObjectsStacks::iterator lib = library_live_objects->begin();
lib != library_live_objects->end(); ++lib) {
for (LiveObjectsStack::iterator span = lib->second.begin();
span != lib->second.end(); ++span) {
uintptr_t start = reinterpret_cast<uintptr_t>(span->ptr);
uintptr_t end = start + span->size;
if (start <= top && top < end) {
RAW_VLOG(2, "Stack at %p is inside /proc/self/maps chunk %p..%p",
top_ptr, (void*)start, (void*)end);
// Shrink start..end region by chopping away the memory regions in
// MemoryRegionMap that land in it to undo merging of regions
// in /proc/self/maps, so that we correctly identify what portion
// of start..end is actually the stack region.
uintptr_t stack_start = start;
uintptr_t stack_end = end;
// can optimize-away this loop, but it does not run often
for (MemoryRegionMap::RegionIterator r =
MemoryRegionMap::BeginRegionLocked();
r != MemoryRegionMap::EndRegionLocked(); ++r) {
if (top < r->start_addr && r->start_addr < stack_end) {
stack_end = r->start_addr;
}
if (stack_start < r->end_addr && r->end_addr <= top) {
stack_start = r->end_addr;
}
}
if (stack_start != start || stack_end != end) {
RAW_VLOG(2, "Stack at %p is actually inside memory chunk %p..%p",
top_ptr, (void*)stack_start, (void*)stack_end);
}
// Make the proper portion of the stack live:
if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) {
RAW_VLOG(2, "Live stack at %p of %"PRIuS" bytes",
top_ptr, stack_end - top);
live_objects->push_back(
AllocObject(top_ptr, stack_end - top, THREAD_DATA));
} else { // GROWS_TOWARDS_HIGH_ADDRESSES
RAW_VLOG(2, "Live stack at %p of %"PRIuS" bytes",
(void*)stack_start, top - stack_start);
live_objects->push_back(
AllocObject((void*)stack_start, top - stack_start, THREAD_DATA));
}
lib->second.erase(span); // kill the rest of the region
// Put the non-stack part(s) of the region back:
if (stack_start != start) {
lib->second.push_back(AllocObject((void*)start, stack_start - start,
MAYBE_LIVE));
}
if (stack_end != end) {
lib->second.push_back(AllocObject((void*)stack_end, end - stack_end,
MAYBE_LIVE));
}
return;
}
}
}
RAW_LOG(ERROR, "Memory region for stack at %p not found. "
"Will likely report false leak positives.", top_ptr);
}
}
// Iterator for heap allocation map data to make objects allocated from
// disabled regions of code to be live.
static void MakeDisabledLiveCallback(const void* ptr,
const HeapProfileTable::AllocInfo& info) {
bool stack_disable = false;
bool range_disable = false;
for (int depth = 0; depth < info.stack_depth; depth++) {
uintptr_t addr = reinterpret_cast<uintptr_t>(info.call_stack[depth]);
if (disabled_addresses &&
disabled_addresses->find(addr) != disabled_addresses->end()) {
stack_disable = true; // found; dropping
break;
}
if (disabled_ranges) {
DisabledRangeMap::const_iterator iter
= disabled_ranges->upper_bound(addr);
if (iter != disabled_ranges->end()) {
RAW_DCHECK(iter->first > addr, "");
if (iter->second.start_address < addr &&
iter->second.max_depth > depth) {
range_disable = true; // in range; dropping
break;
}
}
}
}
if (stack_disable || range_disable) {
uintptr_t start_address = reinterpret_cast<uintptr_t>(ptr);
uintptr_t end_address = start_address + info.object_size;
StackTopSet::const_iterator iter
= stack_tops->lower_bound(start_address);
if (iter != stack_tops->end()) {
RAW_DCHECK(*iter >= start_address, "");
if (*iter < end_address) {
// We do not disable (treat as live) whole allocated regions
// if they are used to hold thread call stacks
// (i.e. when we find a stack inside).
// The reason is that we'll treat as live the currently used
// stack portions anyway (see RegisterStack),
// and the rest of the region where the stack lives can well
// contain outdated stack variables which are not live anymore,
// hence should not be treated as such.
RAW_VLOG(2, "Not %s-disabling %"PRIuS" bytes at %p"
": have stack inside: %p",
(stack_disable ? "stack" : "range"),
info.object_size, ptr, (void*)*iter);
return;
}
}
RAW_VLOG(2, "%s-disabling %"PRIuS" bytes at %p",
(stack_disable ? "Stack" : "Range"), info.object_size, ptr);
live_objects->push_back(AllocObject(ptr, info.object_size,
MUST_BE_ON_HEAP));
}
}
// This function takes some fields from a /proc/self/maps line:
//
// start_address start address of a memory region.
// end_address end address of a memory region
// permissions rwx + private/shared bit
// filename filename of the mapped file
//
// If the region is not writeable, then it cannot have any heap
// pointers in it, otherwise we record it as a candidate live region
// to get filtered later.
static void RecordGlobalDataLocked(uintptr_t start_address,
uintptr_t end_address,
const char* permissions,
const char* filename) {
// Ignore non-writeable regions.
if (strchr(permissions, 'w') == NULL) return;
if (filename == NULL || *filename == '\0') filename = "UNNAMED";
RAW_VLOG(2, "Looking into %s: 0x%" PRIxPTR "..0x%" PRIxPTR,
filename, start_address, end_address);
(*library_live_objects)[filename].
push_back(AllocObject(reinterpret_cast<void*>(start_address),
end_address - start_address,
MAYBE_LIVE));
}
// See if 'library' from /proc/self/maps has base name 'library_base'
// i.e. contains it and has '.' or '-' after it.
static bool IsLibraryNamed(const char* library, const char* library_base) {
const char* p = strstr(library, library_base);
size_t sz = strlen(library_base);
return p != NULL && (p[sz] == '.' || p[sz] == '-');
}
void HeapLeakChecker::DisableLibraryAllocsLocked(const char* library,
uintptr_t start_address,
uintptr_t end_address) {
RAW_DCHECK(heap_checker_lock.IsHeld(), "");
int depth = 0;
// TODO(maxim): maybe this should be extended to also use objdump
// and pick the text portion of the library more precisely.
if (IsLibraryNamed(library, "/libpthread") ||
// libpthread has a lot of small "system" leaks we don't care about.
// In particular it allocates memory to store data supplied via
// pthread_setspecific (which can be the only pointer to a heap object).
IsLibraryNamed(library, "/libdl") ||
// library loaders leak some "system" heap that we don't care about
IsLibraryNamed(library, "/libcrypto")
// Sometimes libcrypto of OpenSSH is compiled with -fomit-frame-pointer
// (any library can be, of course, but this one often is because speed
// is so important for making crypto usable). We ignore all its
// allocations because we can't see the call stacks. We'd prefer
// HeapLeakChecker::DisableChecksIn("default_malloc_ex"
// "|default_realloc_ex")
// but that doesn't work when the end-result binary is stripped.
) {
depth = 1; // only disable allocation calls directly from the library code
} else if (IsLibraryNamed(library, "/ld")
// library loader leaks some "system" heap
// (e.g. thread-local storage) that we don't care about
) {
depth = 2; // disable allocation calls directly from the library code
// and at depth 2 from it.
// We need depth 2 here solely because of a libc bug that
// forces us to jump through __memalign_hook and MemalignOverride hoops
// in tcmalloc.cc.
// Those buggy __libc_memalign() calls are in ld-linux.so and happen for
// thread-local storage allocations that we want to ignore here.
// We go with the depth-2 hack as a workaround for this libc bug:
// otherwise we'd need to extend MallocHook interface
// so that correct stack depth adjustment can be propagated from
// the exceptional case of MemalignOverride.
// Using depth 2 here should not mask real leaks because ld-linux.so
// does not call user code.
}
if (depth) {
RAW_VLOG(1, "Disabling allocations from %s at depth %d:", library, depth);
DisableChecksFromToLocked(reinterpret_cast<void*>(start_address),
reinterpret_cast<void*>(end_address),
depth);
if (IsLibraryNamed(library, "/libpthread") ||
IsLibraryNamed(library, "/libdl") ||
IsLibraryNamed(library, "/ld")) {
RAW_VLOG(1, "Global memory regions made by %s will be live data",
library);
if (global_region_caller_ranges == NULL) {
global_region_caller_ranges =
new (Allocator::Allocate(sizeof(GlobalRegionCallerRangeMap)))
GlobalRegionCallerRangeMap;
}
global_region_caller_ranges
->insert(make_pair(end_address, start_address));
}
}
}
HeapLeakChecker::ProcMapsResult HeapLeakChecker::UseProcMapsLocked(
ProcMapsTask proc_maps_task) {
RAW_DCHECK(heap_checker_lock.IsHeld(), "");
// Need to provide own scratch memory to ProcMapsIterator:
ProcMapsIterator::Buffer buffer;
ProcMapsIterator it(0, &buffer);
if (!it.Valid()) {
int errsv = errno;
RAW_LOG(ERROR, "Could not open /proc/self/maps: errno=%d. "
"Libraries will not be handled correctly.", errsv);
return CANT_OPEN_PROC_MAPS;
}
uint64 start_address, end_address, file_offset;
int64 inode;
char *permissions, *filename;
bool saw_shared_lib = false;
while (it.Next(&start_address, &end_address, &permissions,
&file_offset, &inode, &filename)) {
if (start_address >= end_address) {
// Warn if a line we can be interested in is ill-formed:
if (inode != 0) {
RAW_LOG(ERROR, "Errors reading /proc/self/maps. "
"Some global memory regions will not "
"be handled correctly.");
}
// Silently skip other ill-formed lines: some are possible
// probably due to the interplay of how /proc/self/maps is updated
// while we read it in chunks in ProcMapsIterator and
// do things in this loop.
continue;
}
// Determine if any shared libraries are present.
if (inode != 0 && strstr(filename, "lib") && strstr(filename, ".so")) {
saw_shared_lib = true;
}
switch (proc_maps_task) {
case DISABLE_LIBRARY_ALLOCS:
// All lines starting like
// "401dc000-4030f000 r??p 00132000 03:01 13991972 lib/bin"
// identify a data and code sections of a shared library or our binary
if (inode != 0 && strncmp(permissions, "r-xp", 4) == 0) {
DisableLibraryAllocsLocked(filename, start_address, end_address);
}
break;
case RECORD_GLOBAL_DATA:
RecordGlobalDataLocked(start_address, end_address,
permissions, filename);
break;
default:
RAW_CHECK(0, "");
}
}
if (!saw_shared_lib) {
RAW_LOG(ERROR, "No shared libs detected. Will likely report false leak "
"positives for statically linked executables.");
return NO_SHARED_LIBS_IN_PROC_MAPS;
}
return PROC_MAPS_USED;
}
// Total number and size of live objects dropped from the profile.
static int64 live_objects_total = 0;
static int64 live_bytes_total = 0;
// pid of the thread that is doing the current leak check
// (protected by our lock; IgnoreAllLiveObjectsLocked sets it)
static pid_t self_thread_pid = 0;
// Status of our thread listing callback execution
// (protected by our lock; used from within IgnoreAllLiveObjectsLocked)
static enum {
CALLBACK_NOT_STARTED,
CALLBACK_STARTED,
CALLBACK_COMPLETED,
} thread_listing_status = CALLBACK_NOT_STARTED;
// Ideally to avoid deadlocks this function should not result in any libc
// or other function calls that might need to lock a mutex:
// It is called when all threads of a process are stopped
// at arbitrary points thus potentially holding those locks.
//
// In practice we are calling some simple i/o and sprintf-type library functions
// for logging messages, but use only our own LowLevelAlloc::Arena allocator.
//
// This is known to be buggy: the library i/o function calls are able to cause
// deadlocks when they request a lock that a stopped thread happens to hold.
// This issue as far as we know have so far not resulted in any deadlocks
// in practice, so for now we are taking our chance that the deadlocks
// have insignificant frequency.
//
// If such deadlocks become a problem we should make the i/o calls
// into appropriately direct system calls (or eliminate them),
// in particular write() is not safe and vsnprintf() is potentially dangerous
// due to reliance on locale functions (these are called through RAW_LOG
// and in other ways).
//
int HeapLeakChecker::IgnoreLiveThreads(void* parameter,
int num_threads,
pid_t* thread_pids,
va_list ap) {
thread_listing_status = CALLBACK_STARTED;
RAW_VLOG(2, "Found %d threads (from pid %d)", num_threads, getpid());
if (FLAGS_heap_check_ignore_global_live) {
UseProcMapsLocked(RECORD_GLOBAL_DATA);
}
// We put the registers from other threads here
// to make pointers stored in them live.
vector<void*, STL_Allocator<void*, Allocator> > thread_registers;
int failures = 0;
for (int i = 0; i < num_threads; ++i) {
// the leak checking thread itself is handled
// specially via self_thread_stack, not here:
if (thread_pids[i] == self_thread_pid) continue;
RAW_VLOG(2, "Handling thread with pid %d", thread_pids[i]);
#if defined(HAVE_LINUX_PTRACE_H) && defined(HAVE_SYS_SYSCALL_H) && defined(DUMPER)
i386_regs thread_regs;
#define sys_ptrace(r, p, a, d) syscall(SYS_ptrace, (r), (p), (a), (d))
// We use sys_ptrace to avoid thread locking
// because this is called from ListAllProcessThreads
// when all but this thread are suspended.
if (sys_ptrace(PTRACE_GETREGS, thread_pids[i], NULL, &thread_regs) == 0) {
// Need to use SP to get all the data from the very last stack frame:
RegisterStack((void*) thread_regs.SP);
// Make registers live (just in case PTRACE_ATTACH resulted in some
// register pointers still being in the registers and not on the stack):
for (void** p = (void**)&thread_regs;
p < (void**)(&thread_regs + 1); ++p) {
RAW_VLOG(3, "Thread register %p", *p);
thread_registers.push_back(*p);
}
} else {
failures += 1;
}
#else
failures += 1;
#endif
}
// Use all the collected thread (stack) liveness sources:
IgnoreLiveObjectsLocked("threads stack data", "");
if (thread_registers.size()) {
// Make thread registers be live heap data sources.
// we rely here on the fact that vector is in one memory chunk:
RAW_VLOG(2, "Live registers at %p of %"PRIuS" bytes",
&thread_registers[0], thread_registers.size() * sizeof(void*));
live_objects->push_back(AllocObject(&thread_registers[0],
thread_registers.size() * sizeof(void*),
THREAD_REGISTERS));
IgnoreLiveObjectsLocked("threads register data", "");
}
// Do all other liveness walking while all threads are stopped:
IgnoreNonThreadLiveObjectsLocked();
// Can now resume the threads:
ResumeAllProcessThreads(num_threads, thread_pids);
thread_listing_status = CALLBACK_COMPLETED;
return failures;
}
// Stack top of the thread that is doing the current leak check
// (protected by our lock; IgnoreAllLiveObjectsLocked sets it)
static const void* self_thread_stack_top;
void HeapLeakChecker::IgnoreNonThreadLiveObjectsLocked() {
RAW_VLOG(2, "Handling self thread with pid %d", self_thread_pid);
// Register our own stack:
// Important that all stack ranges (including the one here)
// are known before we start looking at them in MakeDisabledLiveCallback:
RegisterStack(self_thread_stack_top);
IgnoreLiveObjectsLocked("stack data", "");
// Make objects we were told to ignore live:
if (ignored_objects) {
for (IgnoredObjectsMap::const_iterator object = ignored_objects->begin();
object != ignored_objects->end(); ++object) {
const void* ptr = reinterpret_cast<const void*>(object->first);
RAW_VLOG(2, "Ignored live object at %p of %"PRIuS" bytes",
ptr, object->second);
live_objects->
push_back(AllocObject(ptr, object->second, MUST_BE_ON_HEAP));
// we do this liveness check for ignored_objects before doing any
// live heap walking to make sure it does not fail needlessly:
size_t object_size;
if (!(HaveOnHeapLocked(&ptr, &object_size) &&
object->second == object_size)) {
RAW_LOG(FATAL, "Object at %p of %"PRIuS" bytes from an"
" IgnoreObject() has disappeared", ptr, object->second);
}
}
IgnoreLiveObjectsLocked("ignored objects", "");
}
// Make code-address-disabled objects live and ignored:
// This in particular makes all thread-specific data live
// because the basic data structure to hold pointers to thread-specific data
// is allocated from libpthreads and we have range-disabled that
// library code with UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS);
// so now we declare all thread-specific data reachable from there as live.
heap_profile->IterateAllocs(MakeDisabledLiveCallback);
IgnoreLiveObjectsLocked("disabled code", "");
// Actually make global data live:
if (FLAGS_heap_check_ignore_global_live) {
bool have_null_region_callers = false;
for (LibraryLiveObjectsStacks::iterator l = library_live_objects->begin();
l != library_live_objects->end(); ++l) {
RAW_CHECK(live_objects->empty(), "");
// Process library_live_objects in l->second
// filtering them by MemoryRegionMap:
// It's safe to iterate over MemoryRegionMap
// w/o locks here as we are inside MemoryRegionMap::Lock().
// The only change to MemoryRegionMap possible in this loop
// is region addition as a result of allocating more memory
// for live_objects. This won't invalidate the RegionIterator
// or the intent of the loop.
// --see the comment by MemoryRegionMap::BeginRegionLocked().
for (MemoryRegionMap::RegionIterator region =
MemoryRegionMap::BeginRegionLocked();
region != MemoryRegionMap::EndRegionLocked(); ++region) {
// "region" from MemoryRegionMap is to be subtracted from
// (tentatively live) regions in l->second
// if it has a stack inside or it was allocated by
// a non-special caller (not one covered by a range
// in global_region_caller_ranges).
// This will in particular exclude all memory chunks used
// by the heap itself as well as what's been allocated with
// any allocator on top of mmap.
bool subtract = true;
if (!region->is_stack && global_region_caller_ranges) {
if (region->caller == static_cast<uintptr_t>(NULL)) {
have_null_region_callers = true;
} else {
GlobalRegionCallerRangeMap::const_iterator iter
= global_region_caller_ranges->upper_bound(region->caller);
if (iter != global_region_caller_ranges->end()) {
RAW_DCHECK(iter->first > region->caller, "");
if (iter->second < region->caller) { // in special region
subtract = false;
}
}
}
}
if (subtract) {
// The loop puts the result of filtering l->second into live_objects:
for (LiveObjectsStack::const_iterator i = l->second.begin();
i != l->second.end(); ++i) {
// subtract *region from *i
uintptr_t start = reinterpret_cast<uintptr_t>(i->ptr);
uintptr_t end = start + i->size;
if (region->start_addr <= start && end <= region->end_addr) {
// full deletion due to subsumption
} else if (start < region->start_addr &&
region->end_addr < end) { // cutting-out split
live_objects->push_back(AllocObject(i->ptr,
region->start_addr - start,
IN_GLOBAL_DATA));
live_objects->push_back(AllocObject((void*)region->end_addr,
end - region->end_addr,
IN_GLOBAL_DATA));
} else if (region->end_addr > start &&
region->start_addr <= start) { // cut from start
live_objects->push_back(AllocObject((void*)region->end_addr,
end - region->end_addr,
IN_GLOBAL_DATA));
} else if (region->start_addr > start &&
region->start_addr < end) { // cut from end
live_objects->push_back(AllocObject(i->ptr,
region->start_addr - start,
IN_GLOBAL_DATA));
} else { // pass: no intersection
live_objects->push_back(AllocObject(i->ptr, i->size,
IN_GLOBAL_DATA));
}
}
// Move live_objects back into l->second
// for filtering by the next region.
live_objects->swap(l->second);
live_objects->clear();
}
}
// Now get and use live_objects from the final version of l->second:
if (VLOG_IS_ON(2)) {
for (LiveObjectsStack::const_iterator i = l->second.begin();
i != l->second.end(); ++i) {
RAW_VLOG(2, "Library live region at %p of %"PRIuS" bytes",
i->ptr, i->size);
}
}
live_objects->swap(l->second);
IgnoreLiveObjectsLocked("in globals of\n ", l->first.c_str());
}
if (have_null_region_callers) {
RAW_LOG(ERROR, "Have memory regions w/o callers: "
"might report false leaks");
}
Allocator::DeleteAndNull(&library_live_objects);
}
}
void HeapLeakChecker::IgnoreAllLiveObjectsLocked(const void* self_stack_top) {
RAW_CHECK(live_objects == NULL, "");
live_objects = new (Allocator::Allocate(sizeof(LiveObjectsStack)))
LiveObjectsStack;
stack_tops = new (Allocator::Allocate(sizeof(StackTopSet))) StackTopSet;
// Record global data as live:
if (FLAGS_heap_check_ignore_global_live) {
library_live_objects =
new (Allocator::Allocate(sizeof(LibraryLiveObjectsStacks)))
LibraryLiveObjectsStacks;
}
// Ignore all thread stacks:
thread_listing_status = CALLBACK_NOT_STARTED;
bool need_to_ignore_non_thread_objects = true;
self_thread_pid = getpid();
self_thread_stack_top = self_stack_top;
if (FLAGS_heap_check_ignore_thread_live) {
// We fully suspend the threads right here before any liveness checking
// and keep them suspended for the whole time of liveness checking
// inside of the IgnoreLiveThreads callback.
// (The threads can't (de)allocate due to lock on the delete hook but
// if not suspended they could still mess with the pointer
// graph while we walk it).
int r = ListAllProcessThreads(NULL, IgnoreLiveThreads);
need_to_ignore_non_thread_objects = r < 0;
if (r < 0) {
RAW_LOG(WARNING, "Thread finding failed with %d errno=%d", r, errno);
if (thread_listing_status == CALLBACK_COMPLETED) {
RAW_LOG(INFO, "Thread finding callback "
"finished ok; hopefully everything is fine");
need_to_ignore_non_thread_objects = false;
} else if (thread_listing_status == CALLBACK_STARTED) {
RAW_LOG(FATAL, "Thread finding callback was "
"interrupted or crashed; can't fix this");
} else { // CALLBACK_NOT_STARTED
RAW_LOG(ERROR, "Could not find thread stacks. "
"Will likely report false leak positives.");
}
} else if (r != 0) {
RAW_LOG(ERROR, "Thread stacks not found for %d threads. "
"Will likely report false leak positives.", r);
} else {
RAW_VLOG(2, "Thread stacks appear to be found for all threads");
}
} else {
RAW_LOG(WARNING, "Not looking for thread stacks; "
"objects reachable only from there "
"will be reported as leaks");
}
// Do all other live data ignoring here if we did not do it
// within thread listing callback with all threads stopped.
if (need_to_ignore_non_thread_objects) {
if (FLAGS_heap_check_ignore_global_live) {
UseProcMapsLocked(RECORD_GLOBAL_DATA);
}
IgnoreNonThreadLiveObjectsLocked();
}
if (live_objects_total) {
RAW_VLOG(0, "Ignoring %"PRId64" reachable objects of %"PRId64" bytes",
live_objects_total, live_bytes_total);
}
// Free these: we made them here and heap_profile never saw them
Allocator::DeleteAndNull(&live_objects);
Allocator::DeleteAndNull(&stack_tops);
}
// Alignment at which we should consider pointer positions
// in IgnoreLiveObjectsLocked. Use 1 if any alignment is ok.
static size_t pointer_alignment = sizeof(void*);
// Global lock for HeapLeakChecker::DoNoLeaks to protect pointer_alignment.
static SpinLock alignment_checker_lock(SpinLock::LINKER_INITIALIZED);
// This function does not change heap_profile's state:
// we only record live objects to be skipped into profile_adjust_objects
// instead of modifying the heap_profile itself.
void HeapLeakChecker::IgnoreLiveObjectsLocked(const char* name,
const char* name2) {
int64 live_object_count = 0;
int64 live_byte_count = 0;
while (!live_objects->empty()) {
const void* object = live_objects->back().ptr;
size_t size = live_objects->back().size;
const ObjectPlacement place = live_objects->back().place;
live_objects->pop_back();
size_t object_size;
if (place == MUST_BE_ON_HEAP &&
HaveOnHeapLocked(&object, &object_size) &&
profile_adjust_objects->insert(object).second) {
live_object_count += 1;
live_byte_count += size;
}
RAW_VLOG(4, "Looking for heap pointers in %p of %"PRIuS" bytes",
object, size);
// Try interpretting any byte sequence in object,size as a heap pointer:
const size_t remainder =
reinterpret_cast<uintptr_t>(object) % pointer_alignment;
if (remainder) {
object = (reinterpret_cast<const char*>(object) +
pointer_alignment - remainder);
if (size >= pointer_alignment - remainder) {
size -= pointer_alignment - remainder;
} else {
size = 0;
}
}
while (size >= sizeof(void*)) {
const void* ptr;
memcpy(&ptr, object, sizeof(ptr)); // size-independent UNALIGNED_LOAD
const void* current_object = object;
object = reinterpret_cast<const char*>(object) + pointer_alignment;
size -= pointer_alignment;
if (ptr == NULL) continue;
RAW_VLOG(8, "Trying pointer to %p at %p", ptr, current_object);
size_t object_size;
if (HaveOnHeapLocked(&ptr, &object_size) &&
profile_adjust_objects->insert(ptr).second) {
// We take the (hopefully low) risk here of encountering by accident
// a byte sequence in memory that matches an address of
// a heap object which is in fact leaked.
// I.e. in very rare and probably not repeatable/lasting cases
// we might miss some real heap memory leaks.
RAW_VLOG(5, "Found pointer to %p of %"PRIuS" bytes at %p",
ptr, object_size, current_object);
live_object_count += 1;
live_byte_count += object_size;
live_objects->push_back(AllocObject(ptr, object_size, IGNORED_ON_HEAP));
}
}
}
live_objects_total += live_object_count;
live_bytes_total += live_byte_count;
if (live_object_count) {
RAW_VLOG(1, "Removed %"PRId64" live heap objects of %"PRId64" bytes: %s%s",
live_object_count, live_byte_count, name, name2);
}
}
bool HeapLeakChecker::HeapProfileFilter(const void* ptr, size_t size) {
if (profile_adjust_objects->find(ptr) != profile_adjust_objects->end()) {
RAW_VLOG(4, "Ignoring object at %p of %"PRIuS" bytes", ptr, size);
// erase so we can later test that all adjust-objects got utilized
profile_adjust_objects->erase(ptr);
return true;
}
return false;
}
//----------------------------------------------------------------------
// HeapLeakChecker leak check disabling components
//----------------------------------------------------------------------
void HeapLeakChecker::DisableChecksUp(int stack_frames) {
if (!heap_checker_on) return;
RAW_CHECK(stack_frames >= 1, "");
void* stack[1];
if (GetStackTrace(stack, 1, stack_frames + 1) != 1) {
RAW_LOG(FATAL, "Can't get stack trace");
}
DisableChecksAt(stack[0]);
}
void HeapLeakChecker::DisableChecksAt(const void* address) {
if (!heap_checker_on) return;
heap_checker_lock.Lock();
DisableChecksAtLocked(address);
heap_checker_lock.Unlock();
}
bool HeapLeakChecker::HaveDisabledChecksUp(int stack_frames) {
if (!heap_checker_on) return false;
RAW_CHECK(stack_frames >= 1, "");
void* stack[1];
if (GetStackTrace(stack, 1, stack_frames + 1) != 1) {
RAW_LOG(FATAL, "Can't get stack trace");
}
return HaveDisabledChecksAt(stack[0]);
}
bool HeapLeakChecker::HaveDisabledChecksAt(const void* address) {
if (!heap_checker_on) return false;
heap_checker_lock.Lock();
bool result = disabled_addresses != NULL &&
disabled_addresses->
find(reinterpret_cast<uintptr_t>(address)) !=
disabled_addresses->end();
heap_checker_lock.Unlock();
return result;
}
void HeapLeakChecker::DisableChecksIn(const char* pattern) {
if (!heap_checker_on) return;
heap_checker_lock.Lock();
DisableChecksInLocked(pattern);
heap_checker_lock.Unlock();
}
void* HeapLeakChecker::GetDisableChecksStart() {
if (!heap_checker_on) return NULL;
void* start_address = NULL;
if (GetStackTrace(&start_address, 1, 1) != 1) {
RAW_LOG(FATAL, "Can't get stack trace");
}
return start_address;
}
void HeapLeakChecker::DisableChecksToHereFrom(const void* start_address) {
if (!heap_checker_on) return;
void* end_address_ptr = NULL;
if (GetStackTrace(&end_address_ptr, 1, 1) != 1) {
RAW_LOG(FATAL, "Can't get stack trace");
}
const void* end_address = end_address_ptr;
if (start_address > end_address) swap(start_address, end_address);
heap_checker_lock.Lock();
DisableChecksFromToLocked(start_address, end_address, 10000);
// practically no stack depth limit:
// our heap_profile keeps much shorter stack traces
heap_checker_lock.Unlock();
}
void HeapLeakChecker::IgnoreObject(const void* ptr) {
if (!heap_checker_on) return;
heap_checker_lock.Lock();
IgnoreObjectLocked(ptr);
heap_checker_lock.Unlock();
}
void HeapLeakChecker::IgnoreObjectLocked(const void* ptr) {
size_t object_size;
if (HaveOnHeapLocked(&ptr, &object_size)) {
RAW_VLOG(1, "Going to ignore live object at %p of %"PRIuS" bytes",
ptr, object_size);
if (ignored_objects == NULL) {
ignored_objects = new (Allocator::Allocate(sizeof(IgnoredObjectsMap)))
IgnoredObjectsMap;
}
if (!ignored_objects->insert(make_pair(reinterpret_cast<uintptr_t>(ptr),
object_size)).second) {
RAW_LOG(FATAL, "Object at %p is already being ignored", ptr);
}
}
}
void HeapLeakChecker::UnIgnoreObject(const void* ptr) {
if (!heap_checker_on) return;
heap_checker_lock.Lock();
size_t object_size;
bool ok = HaveOnHeapLocked(&ptr, &object_size);
if (ok) {
ok = false;
if (ignored_objects) {
IgnoredObjectsMap::iterator object =
ignored_objects->find(reinterpret_cast<uintptr_t>(ptr));
if (object != ignored_objects->end() && object_size == object->second) {
ignored_objects->erase(object);
ok = true;
RAW_VLOG(1, "Now not going to ignore live object "
"at %p of %"PRIuS" bytes", ptr, object_size);
}
}
}
heap_checker_lock.Unlock();
if (!ok) RAW_LOG(FATAL, "Object at %p has not been ignored", ptr);
}
//----------------------------------------------------------------------
// HeapLeakChecker non-static functions
//----------------------------------------------------------------------
void HeapLeakChecker::DumpProfileLocked(ProfileType profile_type,
const void* self_stack_top,
size_t* alloc_bytes,
size_t* alloc_objects) {
RAW_VLOG(0, "%s check \"%s\"%s",
(profile_type == START_PROFILE ? "Starting"
: "At an end point for"),
name_,
(pointer_alignment == 1 ? " w/o pointer alignment" : ""));
// Sanity check that nobody is messing with the hooks we need:
// Important to have it here: else we can misteriously SIGSEGV
// in IgnoreLiveObjectsLocked inside ListAllProcessThreads's callback
// by looking into a region that got unmapped w/o our knowledge.
MemoryRegionMap::CheckMallocHooks();
if (MallocHook::GetNewHook() != NewHook ||
MallocHook::GetDeleteHook() != DeleteHook) {
RAW_LOG(FATAL, "new/delete malloc hooks got changed");
}
// Make the heap profile, other threads are locked out.
RAW_CHECK(profile_adjust_objects == NULL, "");
const int alloc_count = Allocator::alloc_count();
profile_adjust_objects =
new (Allocator::Allocate(sizeof(ProfileAdjustObjectSet)))
ProfileAdjustObjectSet;
IgnoreAllLiveObjectsLocked(self_stack_top);
const int len = profile_prefix->size() + strlen(name_) + 10 + 2;
char* file_name = reinterpret_cast<char*>(Allocator::Allocate(len));
snprintf(file_name, len, "%s.%s%s%s",
profile_prefix->c_str(), name_,
profile_type == START_PROFILE ? "-beg" : "-end",
HeapProfileTable::kFileExt);
HeapProfileTable::Stats stats;
bool ok = heap_profile->DumpFilteredProfile(
file_name, HeapProfileFilter, FLAGS_heap_check_identify_leaks, &stats);
RAW_CHECK(ok, "No sense to continue");
*alloc_bytes = stats.alloc_size - stats.free_size;
*alloc_objects = stats.allocs - stats.frees;
Allocator::Free(file_name);
RAW_CHECK(profile_adjust_objects->empty(),
"Some objects to ignore are not on the heap");
Allocator::DeleteAndNull(&profile_adjust_objects);
// Check that we made no leaks ourselves:
if (Allocator::alloc_count() != alloc_count) {
RAW_LOG(FATAL, "Internal HeapChecker leak of %d objects",
Allocator::alloc_count() - alloc_count);
}
}
void HeapLeakChecker::Create(const char *name) {
name_ = NULL;
has_checked_ = false;
char* n = new char[strlen(name) + 1]; // do this before we lock
IgnoreObject(n); // otherwise it might be treated as live due to our stack
alignment_checker_lock.Lock();
heap_checker_lock.Lock();
// Heap activity in other threads is paused for this whole function.
MemoryRegionMap::Lock();
if (heap_checker_on) {
RAW_DCHECK(strchr(name, '/') == NULL, "must be a simple name");
name_ = n;
memcpy(name_, name, strlen(name) + 1);
// Use our stack ptr to make stack data live:
int a_local_var;
DumpProfileLocked(START_PROFILE, &a_local_var,
&start_inuse_bytes_, &start_inuse_allocs_);
RAW_VLOG(1, "Start check \"%s\" profile: %"PRIuS" bytes "
"in %"PRIuS" objects",
name_, start_inuse_bytes_, start_inuse_allocs_);
} else {
RAW_LOG(WARNING, "Heap checker is not active, "
"hence checker \"%s\" will do nothing!", name);
RAW_LOG(WARNING, "To activate set the HEAPCHECK environment variable.\n");
}
MemoryRegionMap::Unlock();
heap_checker_lock.Unlock();
alignment_checker_lock.Unlock();
if (name_ == NULL) {
UnIgnoreObject(n);
delete[] n; // must be done after we unlock
}
}
HeapLeakChecker::HeapLeakChecker(const char *name) {
RAW_DCHECK(strcmp(name, "_main_") != 0, "_main_ is reserved");
Create(name);
}
HeapLeakChecker::HeapLeakChecker() {
Create("_main_");
}
ssize_t HeapLeakChecker::BytesLeaked() const {
if (!has_checked_) {
RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call");
}
return inuse_bytes_increase_;
}
ssize_t HeapLeakChecker::ObjectsLeaked() const {
if (!has_checked_) {
RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call");
}
return inuse_allocs_increase_;
}
// Save pid of main thread for using in naming dump files
static int32 main_thread_pid = getpid();
#ifdef HAVE_PROGRAM_INVOCATION_NAME
extern char* program_invocation_name;
extern char* program_invocation_short_name;
static const char* invocation_name() { return program_invocation_short_name; }
static const char* invocation_path() { return program_invocation_name; }
#else
static const char* invocation_name() { return "<your binary>"; }
static const char* invocation_path() { return "<your binary>"; }
#endif
static void MakeCommand(const char* basename,
bool check_type_is_no_leaks,
bool use_initial_profile,
const string& prefix,
string* beg_profile,
string* end_profile,
string* command) {
string ignore_re;
if (disabled_regexp) {
ignore_re += " --ignore='^";
ignore_re += disabled_regexp->c_str();
ignore_re += "$'";
}
*command += *flags_heap_profile_pprof;
if (use_initial_profile) {
// compare against initial profile only if need to
*beg_profile = prefix + "." + basename +
"-beg" + HeapProfileTable::kFileExt;
*command += string(" --base=\"") + *beg_profile + "\"";
}
if (check_type_is_no_leaks) *command += string(" --drop_negative");
*end_profile = prefix + "." + basename + "-end" + HeapProfileTable::kFileExt;
*command += string(" ") +
invocation_path() +
" \"" + *end_profile + "\"" + ignore_re + " --inuse_objects";
if (!FLAGS_heap_check_identify_leaks) {
*command += " --lines"; // important to catch leaks when !see_leaks
} else {
*command += " --addresses"; // stronger than --lines and prints
// unresolvable object addresses
}
}
static int GetStatusOutput(const char* command, string* output) {
// We don't want the heapchecker to run in the child helper
// processes that we fork() as part of this process' heap check.
// setenv() can call realloc(), so we don't want to call it while
// the heap profiling is disabled. Instead just overwrite the final
// char of the env var name, so it has a different name and gets
// ignored in the child. We assume the env looks like 'VAR=VALUE\0VAR=VALUE'
char *env_heapcheck = getenv("HEAPCHECK");
char *env_ldpreload = getenv("LD_PRELOAD");
if (env_heapcheck) {
assert(env_heapcheck[-1] == '=');
env_heapcheck[-2] = '?';
}
if (env_ldpreload) {
assert(env_ldpreload[-1] == '=');
env_ldpreload[-2] = '?';
}
FILE* f = popen(command, "r");
if (f == NULL) {
fprintf(stderr, "popen(%s) failed!\n", command); // This shouldn't happen
exit(1);
}
if (env_heapcheck) env_heapcheck[-2] = 'K'; // last letter in heapchecK
if (env_ldpreload) env_heapcheck[-2] = 'D'; // last letter in ldpreloaD
const int kMaxOutputLine = 10000;
char line[kMaxOutputLine];
while (fgets(line, sizeof(line), f) != NULL) {
if (output)
*output += line;
}
return pclose(f);
}
// RAW_LOG 'str' line by line to prevent its truncation in RAW_LOG:
static void RawLogLines(const string& str) {
int p = 0;
while (1) {
int l = str.find('\n', p);
if (l == string::npos) {
if (str[p]) { // print last line if non empty
RAW_LOG(INFO, "%s", str.c_str() + p);
}
break;
}
const_cast<string&>(str)[l] = '\0'; // safe for our use case
RAW_LOG(INFO, "%s", str.c_str() + p);
const_cast<string&>(str)[l] = '\n';
p = l + 1;
}
}
bool HeapLeakChecker::DoNoLeaks(CheckType check_type,
CheckFullness fullness,
ReportMode report_mode) {
// The locking also helps us keep the messages
// for the two checks close together.
alignment_checker_lock.Lock();
bool result;
if (FLAGS_heap_check_test_pointer_alignment) {
pointer_alignment = 1;
bool result_wo_align = DoNoLeaksOnce(check_type, fullness, NO_REPORT);
pointer_alignment = sizeof(void*);
result = DoNoLeaksOnce(check_type, fullness, report_mode);
if (!result) {
if (result_wo_align) {
RAW_LOG(WARNING, "Found no leaks without pointer alignment: "
"something might be placing pointers at "
"unaligned addresses! This needs to be fixed.");
} else {
RAW_LOG(INFO, "Found leaks without pointer alignment as well: "
"unaligned pointers must not be the cause of leaks.");
RAW_LOG(INFO, "--heap_check_test_pointer_alignment did not help to "
"diagnose the leaks.");
}
}
} else {
result = DoNoLeaksOnce(check_type, fullness, report_mode);
if (!result) {
if (!FLAGS_heap_check_identify_leaks) {
RAW_LOG(INFO, "setenv HEAP_CHECK_IDENTIFY_LEAKS=1 and rerun to identify "
"the addresses of all leaked objects; "
"will be reported as fake immediate allocation callers");
}
RAW_LOG(INFO, "If you are totally puzzled about why the leaks are there, "
"try rerunning it with "
"setenv HEAP_CHECK_TEST_POINTER_ALIGNMENT=1");
}
}
alignment_checker_lock.Unlock();
return result;
}
bool HeapLeakChecker::DoNoLeaksOnce(CheckType check_type,
CheckFullness fullness,
ReportMode report_mode) {
// Heap activity in other threads is paused for this function
// until we got all profile difference info.
heap_checker_lock.Lock();
MemoryRegionMap::Lock();
if (heap_checker_on) {
if (name_ == NULL) {
RAW_LOG(FATAL, "Heap profiling must be not turned on "
"after construction of a HeapLeakChecker");
}
// Use our stack ptr to make stack data live:
int a_local_var;
size_t end_inuse_bytes;
size_t end_inuse_allocs;
DumpProfileLocked(END_PROFILE, &a_local_var,
&end_inuse_bytes, &end_inuse_allocs);
const bool use_initial_profile =
!(FLAGS_heap_check_before_constructors && this == main_heap_checker);
if (!use_initial_profile) { // compare against empty initial profile
start_inuse_bytes_ = 0;
start_inuse_allocs_ = 0;
}
RAW_VLOG(1, "End check \"%s\" profile: %"PRIuS" bytes in %"PRIuS" objects",
name_, end_inuse_bytes, end_inuse_allocs);
inuse_bytes_increase_ = static_cast<ssize_t>(end_inuse_bytes -
start_inuse_bytes_);
inuse_allocs_increase_ = static_cast<ssize_t>(end_inuse_allocs -
start_inuse_allocs_);
has_checked_ = true;
MemoryRegionMap::Unlock();
heap_checker_lock.Unlock();
bool see_leaks =
check_type == SAME_HEAP
? (inuse_bytes_increase_ != 0 || inuse_allocs_increase_ != 0)
: (inuse_bytes_increase_ > 0 || inuse_allocs_increase_ > 0);
if (see_leaks || fullness == USE_PPROF) {
const bool pprof_can_ignore = disabled_regexp != NULL;
string beg_profile;
string end_profile;
string base_command;
MakeCommand(name_, check_type == NO_LEAKS,
use_initial_profile, *profile_prefix,
&beg_profile, &end_profile, &base_command);
// Make the two command lines out of the base command, with
// appropriate mode options
string command = base_command + " --text";
string gv_command;
gv_command = base_command;
gv_command +=
" --edgefraction=1e-10 --nodefraction=1e-10 --heapcheck --gv";
if (see_leaks) {
RAW_LOG(ERROR, "Heap memory leaks of %"PRIdS" bytes and/or "
"%"PRIdS" allocations detected by check \"%s\".",
inuse_bytes_increase_, inuse_allocs_increase_, name_);
RAW_LOG(ERROR, "TO INVESTIGATE leaks RUN e.g. THIS shell command:\n"
"\n%s\n", gv_command.c_str());
}
string output;
bool checked_leaks = true;
if ((see_leaks && report_mode == PPROF_REPORT) ||
fullness == USE_PPROF) {
if (access(flags_heap_profile_pprof->c_str(), X_OK|R_OK) != 0) {
RAW_LOG(WARNING, "Skipping pprof check: could not run it at %s",
flags_heap_profile_pprof->c_str());
checked_leaks = false;
} else {
// We don't care about pprof's stderr as long as it
// succeeds with empty report:
checked_leaks = GetStatusOutput((command + " 2>/dev/null").c_str(),
&output) == 0;
}
if (see_leaks && pprof_can_ignore && output.empty() && checked_leaks) {
RAW_LOG(WARNING, "These must be leaks that we disabled"
" (pprof succeeded)! This check WILL FAIL"
" if the binary is strip'ped!");
see_leaks = false;
}
// do not fail the check just due to us being a stripped binary
if (!see_leaks && strstr(output.c_str(), "nm: ") != NULL &&
strstr(output.c_str(), ": no symbols") != NULL) output.clear();
}
// Make sure the profiles we created are still there.
// They can get deleted e.g. if the program forks/executes itself
// and FLAGS_cleanup_old_heap_profiles was kept as true.
if (access(end_profile.c_str(), R_OK) != 0 ||
(!beg_profile.empty() && access(beg_profile.c_str(), R_OK) != 0)) {
RAW_LOG(FATAL, "One of the heap profiles is gone: %s %s",
beg_profile.c_str(), end_profile.c_str());
}
if (!(see_leaks || checked_leaks)) {
// Crash if something went wrong with executing pprof
// and we rely on pprof to do its work:
RAW_LOG(FATAL, "The pprof command failed: %s", command.c_str());
}
if (see_leaks && use_initial_profile) {
RAW_LOG(WARNING, "CAVEAT: Some of the reported leaks might have "
"occurred before check \"%s\" was started!", name_);
}
bool tricky_leaks = !output.empty();
if (!see_leaks && tricky_leaks) {
RAW_LOG(WARNING, "Tricky heap memory leaks of"
" no bytes and no allocations "
"detected by check \"%s\".", name_);
RAW_LOG(WARNING, "TO INVESTIGATE leaks RUN e.g. THIS shell command:\n"
"\n%s\n", gv_command.c_str());
if (use_initial_profile) {
RAW_LOG(WARNING, "CAVEAT: Some of the reported leaks might have "
"occurred before check \"%s\" was started!", name_);
}
see_leaks = true;
}
if (see_leaks && report_mode == PPROF_REPORT) {
if (checked_leaks) {
RAW_LOG(INFO, "Below is (less informative) textual version "
"of this pprof command's output:");
RawLogLines(output);
} else {
RAW_LOG(ERROR, "The pprof command has failed");
}
}
} else {
RAW_VLOG(0, "No leaks found for check \"%s\" "
"(but no 100%% guarantee that there aren't any)", name_);
}
return !see_leaks;
} else {
if (name_ != NULL) {
RAW_LOG(FATAL, "Profiling must stay enabled during leak checking");
}
MemoryRegionMap::Unlock();
heap_checker_lock.Unlock();
return true;
}
}
HeapLeakChecker::~HeapLeakChecker() {
if (name_ != NULL) { // had leak checking enabled when created the checker
if (!has_checked_) {
RAW_LOG(FATAL, "Some *NoLeaks|SameHeap method"
" must be called on any created checker");
}
UnIgnoreObject(name_);
delete[] name_;
name_ = NULL;
}
}
//----------------------------------------------------------------------
// HeapLeakChecker overall heap check components
//----------------------------------------------------------------------
bool HeapLeakChecker::IsActive() {
return heap_checker_on;
}
vector<HeapCleaner::void_function>* HeapCleaner::heap_cleanups_ = NULL;
// When a HeapCleaner object is intialized, add its function to the static list
// of cleaners to be run before leaks checking.
HeapCleaner::HeapCleaner(void_function f) {
if (heap_cleanups_ == NULL)
heap_cleanups_ = new vector<HeapCleaner::void_function>;
heap_cleanups_->push_back(f);
}
// Run all of the cleanup functions and delete the vector.
void HeapCleaner::RunHeapCleanups() {
if (!heap_cleanups_)
return;
for (int i = 0; i < heap_cleanups_->size(); i++) {
void (*f)(void) = (*heap_cleanups_)[i];
f();
}
delete heap_cleanups_;
heap_cleanups_ = NULL;
}
// Program exit heap cleanup registered with atexit().
// Will not get executed when we crash on a signal.
void HeapLeakChecker::RunHeapCleanups() {
if (heap_checker_pid == getpid()) { // can get here (via forks?)
// with other pids
HeapCleaner::RunHeapCleanups();
if (!FLAGS_heap_check_after_destructors && do_main_heap_check) {
DoMainHeapCheck();
}
}
}
// defined below
static int GetCommandLineFrom(const char* file, char* cmdline, int size);
static bool internal_init_start_has_run = false;
// Called exactly once, before main() (but hopefully just before).
// This picks a good unique name for the dumped leak checking heap profiles.
void HeapLeakChecker::InternalInitStart() {
RAW_CHECK(!internal_init_start_has_run, "Only one call is expected");
internal_init_start_has_run = true;
if (FLAGS_heap_check.empty()) {
// turns out we do not need checking in the end; can stop profiling
TurnItselfOff();
return;
}
// Changing this to false can be useful when debugging heap-checker itself:
if (!FLAGS_heap_check_run_under_gdb) {
// See if heap checker should turn itself off because we are
// running under gdb (to avoid conflicts over ptrace-ing rights):
char name_buf[15+15];
snprintf(name_buf, sizeof(name_buf), "/proc/%d/cmdline", int(getppid()));
char cmdline[1024*8];
int size = GetCommandLineFrom(name_buf, cmdline, sizeof(cmdline)-1);
cmdline[size] = '\0';
// look for "gdb" in the executable's name:
const char* last = strrchr(cmdline, '/');
if (last) last += 1;
else last = cmdline;
if (strncmp(last, "gdb", 3) == 0) {
RAW_LOG(WARNING, "We seem to be running under gdb; will turn itself off");
TurnItselfOff();
return;
}
}
if (!constructor_heap_profiling) {
RAW_LOG(FATAL, "Can not start so late. You have to enable heap checking "
"with HEAPCHECK=<mode>.");
}
// make an indestructible copy for heap leak checking
// happening after global variable destruction
flags_heap_profile_pprof = new string(FLAGS_heap_profile_pprof);
// Set all flags
if (FLAGS_heap_check == "minimal") {
// The least we can check.
FLAGS_heap_check_before_constructors = false; // from after main
// (ignore more)
FLAGS_heap_check_after_destructors = false; // to after cleanup
// (most data is live)
FLAGS_heap_check_strict_check = false; // < profile check (ignore more)
FLAGS_heap_check_ignore_thread_live = true; // ignore all live
FLAGS_heap_check_ignore_global_live = true; // ignore all live
} else if (FLAGS_heap_check == "normal") {
// Faster than 'minimal' and not much stricter.
FLAGS_heap_check_before_constructors = true; // from no profile (fast)
FLAGS_heap_check_after_destructors = false; // to after cleanup
// (most data is live)
FLAGS_heap_check_strict_check = true; // == profile check (fast)
FLAGS_heap_check_ignore_thread_live = true; // ignore all live
FLAGS_heap_check_ignore_global_live = true; // ignore all live
} else if (FLAGS_heap_check == "strict") {
// A bit stricter than 'normal': global destructors must fully clean up
// after themselves if they are present.
FLAGS_heap_check_before_constructors = true; // from no profile (fast)
FLAGS_heap_check_after_destructors = true; // to after destructors
// (less data live)
FLAGS_heap_check_strict_check = true; // == profile check (fast)
FLAGS_heap_check_ignore_thread_live = true; // ignore all live
FLAGS_heap_check_ignore_global_live = true; // ignore all live
} else if (FLAGS_heap_check == "draconian") {
// Drop not very portable and not very exact live heap flooding.
FLAGS_heap_check_before_constructors = true; // from no profile (fast)
FLAGS_heap_check_after_destructors = true; // to after destructors
// (need them)
FLAGS_heap_check_strict_check = true; // == profile check (fast)
FLAGS_heap_check_ignore_thread_live = false; // no live flood (stricter)
FLAGS_heap_check_ignore_global_live = false; // no live flood (stricter)
} else if (FLAGS_heap_check == "as-is") {
// do nothing: use other flags as is
} else if (FLAGS_heap_check == "local") {
// do nothing
} else {
RAW_LOG(FATAL, "Unsupported heap_check flag: %s",
FLAGS_heap_check.c_str());
}
RAW_DCHECK(heap_checker_pid == getpid(), "");
heap_checker_on = true;
RAW_DCHECK(heap_profile, "");
heap_checker_lock.Lock();
ProcMapsResult pm_result = UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS);
// might neeed to do this more than once
// if one later dynamically loads libraries that we want disabled
heap_checker_lock.Unlock();
if (pm_result != PROC_MAPS_USED) { // can't function
TurnItselfOff();
return;
}
// make a good place and name for heap profile leak dumps
profile_prefix = new string(FLAGS_heap_check_dump_directory);
*profile_prefix += "/";
*profile_prefix += invocation_name();
HeapProfileTable::CleanupOldProfiles(profile_prefix->c_str());
// Finalize prefix for dumping leak checking profiles.
char pid_buf[15];
if (main_thread_pid == 0) // possible if we're called before constructors
main_thread_pid = getpid();
snprintf(pid_buf, sizeof(pid_buf), ".%d", main_thread_pid);
*profile_prefix += pid_buf;
// Make sure new/delete hooks are installed properly
// and heap profiler is indeed able to keep track
// of the objects being allocated.
// We test this to make sure we are indeed checking for leaks.
char* test_str = new char[5];
size_t size;
RAW_CHECK(heap_profile->FindAlloc(test_str, &size),
"our own new/delete not linked?");
delete[] test_str;
RAW_CHECK(!heap_profile->FindAlloc(test_str, &size),
"our own new/delete not linked?");
// If we crash in the above code, it probably means that
// "nm <this_binary> | grep new" will show that tcmalloc's new/delete
// implementation did not get linked-in into this binary
// (i.e. nm will list __builtin_new and __builtin_vec_new as undefined).
// If this happens, it is a BUILD bug to be fixed.
if (FLAGS_heap_check != "local") {
// Schedule registered heap cleanup
atexit(RunHeapCleanups);
RAW_DCHECK(main_heap_checker == NULL,
"Repeated creation of main_heap_checker");
main_heap_checker = new HeapLeakChecker();
do_main_heap_check = true;
}
RAW_CHECK(heap_checker_on && constructor_heap_profiling,
"Leak checking is expected to be fully turned on now");
}
// We want this to run early as well, but not so early as
// ::BeforeConstructors (we want flag assignments to have already
// happened, for instance). Initializer-registration does the trick.
REGISTER_MODULE_INITIALIZER(init_start, HeapLeakChecker::InternalInitStart());
void HeapLeakChecker::DoMainHeapCheck() {
RAW_DCHECK(heap_checker_pid == getpid() && do_main_heap_check, "");
if (!NoGlobalLeaks()) {
if (FLAGS_heap_check_identify_leaks) {
RAW_LOG(FATAL, "Whole-program memory leaks found.");
}
RAW_LOG(ERROR, "Exiting with error code (instead of crashing) "
"because of whole-program memory leaks");
_exit(1); // we don't want to call atexit() routines!
}
do_main_heap_check = false; // just did it
}
HeapLeakChecker* HeapLeakChecker::GlobalChecker() {
return main_heap_checker;
}
bool HeapLeakChecker::NoGlobalLeaks() {
bool result = true;
HeapLeakChecker* main_hc = main_heap_checker;
if (main_hc) {
CheckType check_type = FLAGS_heap_check_strict_check ? SAME_HEAP : NO_LEAKS;
if (FLAGS_heap_check_before_constructors) check_type = SAME_HEAP;
// NO_LEAKS here just would make it slower in this case
// (we don't use the starting profile anyway)
CheckFullness fullness = check_type == NO_LEAKS ? USE_PPROF : USE_COUNTS;
// use pprof if it can help ignore false leaks
ReportMode report_mode = FLAGS_heap_check_report ? PPROF_REPORT : NO_REPORT;
RAW_VLOG(0, "Checking for whole-program memory leaks");
result = main_hc->DoNoLeaks(check_type, fullness, report_mode);
}
return result;
}
void HeapLeakChecker::CancelGlobalCheck() {
if (do_main_heap_check) {
RAW_VLOG(0, "Canceling the automatic at-exit "
"whole-program memory leak check");
do_main_heap_check = false;
}
}
//----------------------------------------------------------------------
// HeapLeakChecker global constructor/destructor ordering components
//----------------------------------------------------------------------
static bool in_initial_malloc_hook = false;
#ifdef HAVE___ATTRIBUTE__ // we need __attribute__((weak)) for this to work
#define INSTALLED_INITIAL_MALLOC_HOOKS
void HeapLeakChecker_BeforeConstructors(); // below
// Helper for InitialMallocHook_* below
static inline void InitHeapLeakCheckerFromMallocHook() {
RAW_CHECK(!in_initial_malloc_hook,
"Something did not reset initial MallocHook-s");
in_initial_malloc_hook = true;
// Initialize heap checker on the very first allocation/mmap/sbrk call:
HeapLeakChecker_BeforeConstructors();
in_initial_malloc_hook = false;
}
// These will owerwrite the weak definitions in malloc_hook.cc:
// Important to have this to catch the first allocation call from the binary:
extern void InitialMallocHook_New(const void* ptr, size_t size) {
InitHeapLeakCheckerFromMallocHook();
// record this first allocation as well (if we need to):
MallocHook::InvokeNewHook(ptr, size);
}
// Important to have this to catch the first mmap call (say from tcmalloc):
extern void InitialMallocHook_MMap(const void* result,
const void* start,
size_t size,
int protection,
int flags,
int fd,
off_t offset) {
InitHeapLeakCheckerFromMallocHook();
// record this first mmap as well (if we need to):
MallocHook::InvokeMmapHook(
result, start, size, protection, flags, fd, offset);
}
// Important to have this to catch the first sbrk call (say from tcmalloc):
extern void InitialMallocHook_Sbrk(const void* result, ptrdiff_t increment) {
InitHeapLeakCheckerFromMallocHook();
// record this first sbrk as well (if we need to):
MallocHook::InvokeSbrkHook(result, increment);
}
#endif
// Optional silencing, it must be called shortly after leak checker activates
// in HeapLeakChecker::BeforeConstructors not to let logging messages through,
// but it can't be called when BeforeConstructors() is called from within
// the first mmap/sbrk/alloc call (something deadlocks in this case).
// Hence we arrange for this to be called from the first global c-tor
// that calls HeapLeakChecker_BeforeConstructors.
static void HeapLeakChecker_MaybeMakeSilent() {
#if 0 // TODO(csilvers): see if we can get something like this to work
if (!VLOG_IS_ON(1)) // not on a verbose setting
FLAGS_verbose = WARNING; // only log WARNING and ERROR and FATAL
#endif
}
void HeapLeakChecker::BeforeConstructors() {
RAW_CHECK(!constructor_heap_profiling,
"BeforeConstructors called multiple times");
// set hooks early to crash if 'new' gets called before we make heap_profile:
MallocHook::SetNewHook(NewHook);
MallocHook::SetDeleteHook(DeleteHook);
constructor_heap_profiling = true;
MemoryRegionMap::Init(); // set up MemoryRegionMap
// (important that it's done before HeapProfileTable creation below)
Allocator::Init();
RAW_CHECK(heap_profile == NULL, "");
heap_checker_lock.Lock(); // Allocator expects it
heap_profile = new (Allocator::Allocate(sizeof(HeapProfileTable)))
HeapProfileTable(&Allocator::Allocate, &Allocator::Free);
heap_checker_lock.Unlock();
RAW_VLOG(0, "Starting tracking the heap");
heap_checker_on = true;
// Run silencing if we are called from the first global c-tor,
// not from the first mmap/sbrk/alloc call:
if (!in_initial_malloc_hook) HeapLeakChecker_MaybeMakeSilent();
}
void HeapLeakChecker::TurnItselfOff() {
FLAGS_heap_check = ""; // for users who test for it
if (constructor_heap_profiling) {
RAW_CHECK(heap_checker_on, "");
RAW_LOG(INFO, "Turning heap leak checking off");
heap_checker_on = false;
MallocHook::SetNewHook(NULL);
MallocHook::SetDeleteHook(NULL);
heap_checker_lock.Lock(); // Allocator expects it
Allocator::DeleteAndNull(&heap_profile);
// free our optional global data:
Allocator::DeleteAndNullIfNot(&disabled_regexp);
Allocator::DeleteAndNullIfNot(&ignored_objects);
Allocator::DeleteAndNullIfNot(&disabled_addresses);
Allocator::DeleteAndNullIfNot(&disabled_ranges);
Allocator::DeleteAndNullIfNot(&global_region_caller_ranges);
heap_checker_lock.Unlock();
Allocator::Shutdown();
MemoryRegionMap::Shutdown();
}
RAW_CHECK(!heap_checker_on, "");
}
// Read in the command line from 'file' into 'cmdline' and return the size read
// 'size' is the space available in 'cmdline'
// We need this because we don't yet have argv/argc.
// CAVEAT: 'file' (some /proc/*/cmdline) might contain
// the command line truncated.
// Arguments in cmdline will be '\0'-terminated,
// the first one will be the binary's name.
static int GetCommandLineFrom(const char* file, char* cmdline, int size) {
// This routine is only used to check if we're running under gdb, so
// it's ok if this #if fails and the routine is a no-op.
#if defined(HAVE_SYS_SYSCALL_H)
// This function is called before memory allocation hooks are set up
// so we must not have any memory allocations in it. We use syscall
// versions of open/read/close here because we don't trust the non-syscall
// versions: they might 'accidentally' cause a memory allocation.
// Here's a real-life problem scenario we had:
// 1) A program LD_PRELOADed a library called list_file_used.a
// 2) list_file_used intercepted open/read/close and called dlsym()
// 3) dlsym() called pthread_setspecific() which called malloc().
// This malloced memory is 'hidden' from the heap-checker. By
// definition, this thread-local data is live, and everything it points
// to is live (not a memory leak) as well. But because this memory
// was hidden from the heap-checker, everything it points to was
// taken to be orphaned, and therefore, a memory leak.
int fd = syscall(SYS_open, file, O_RDONLY);
int result = 0;
if (fd >= 0) {
ssize_t r;
while ((r = syscall(SYS_read, fd, cmdline + result, size)) > 0) {
result += r;
size -= r;
}
syscall(SYS_close, fd);
}
return result;
#else // HAVE_SYS_SYSCALL_H
return 0;
#endif
}
extern bool heap_leak_checker_bcad_variable; // in heap-checker-bcad.cc
static bool has_called_BeforeConstructors = false;
void HeapLeakChecker_BeforeConstructors() {
// We can be called from several places: the first mmap/sbrk/alloc call
// or the first global c-tor from heap-checker-bcad.cc:
if (has_called_BeforeConstructors) {
// Make sure silencing is done when we are called from first global c-tor:
if (heap_checker_on) HeapLeakChecker_MaybeMakeSilent();
return; // do not re-execure initialization
}
has_called_BeforeConstructors = true;
heap_checker_pid = getpid(); // set it always
heap_leak_checker_bcad_variable = true;
// just to reference it, so that heap-checker-bcad.o is linked in
// This function can be called *very* early, before the normal
// global-constructor that sets FLAGS_verbose. Set it manually now,
// so the RAW_LOG messages here are controllable.
const char* verbose_str = GetenvBeforeMain("PERFTOOLS_VERBOSE");
if (verbose_str && atoi(verbose_str)) { // different than the default of 0?
FLAGS_verbose = atoi(verbose_str);
}
bool need_heap_check = true;
// The user indicates a desire for heap-checking via the HEAPCHECK
// environment variable. If it's not set, there's no way to do
// heap-checking.
if (!GetenvBeforeMain("HEAPCHECK")) {
need_heap_check = false;
}
#ifdef HAVE_GETEUID
if (need_heap_check && getuid() != geteuid()) {
// heap-checker writes out files. Thus, for security reasons, we don't
// recognize the env. var. to turn on heap-checking if we're setuid.
RAW_LOG(WARNING, ("HeapChecker: ignoring HEAPCHECK because "
"program seems to be setuid\n"));
need_heap_check = false;
}
#endif
if (need_heap_check) {
HeapLeakChecker::BeforeConstructors();
} else { // cancel our initial hooks
#ifdef INSTALLED_INITIAL_MALLOC_HOOKS
if (MallocHook::GetNewHook() == &InitialMallocHook_New)
MallocHook::SetNewHook(NULL);
if (MallocHook::GetMmapHook() == &InitialMallocHook_MMap)
MallocHook::SetMmapHook(NULL);
if (MallocHook::GetSbrkHook() == &InitialMallocHook_Sbrk)
MallocHook::SetSbrkHook(NULL);
#endif
}
}
// This function is executed after all global object destructors run.
void HeapLeakChecker_AfterDestructors() {
if (heap_checker_pid == getpid()) { // can get here (via forks?)
// with other pids
if (FLAGS_heap_check_after_destructors && do_main_heap_check) {
HeapLeakChecker::DoMainHeapCheck();
poll(NULL, 0, 500);
// Need this hack to wait for other pthreads to exit.
// Otherwise tcmalloc find errors
// on a free() call from pthreads.
}
RAW_CHECK(!do_main_heap_check, "should have done it");
}
}
//----------------------------------------------------------------------
// HeapLeakChecker disabling helpers
//----------------------------------------------------------------------
// These functions are at the end of the file to prevent their inlining:
void HeapLeakChecker::DisableChecksInLocked(const char* pattern) {
// make disabled_regexp
if (disabled_regexp == NULL) {
disabled_regexp = new (Allocator::Allocate(sizeof(HCL_string))) HCL_string;
}
RAW_VLOG(1, "Disabling leak checking in stack traces "
"under frames maching \"%s\"", pattern);
if (disabled_regexp->size()) *disabled_regexp += '|';
*disabled_regexp += pattern;
}
void HeapLeakChecker::DisableChecksFromToLocked(const void* start_address,
const void* end_address,
int max_depth) {
RAW_DCHECK(heap_checker_lock.IsHeld(), "");
RAW_DCHECK(start_address < end_address, "");
if (disabled_ranges == NULL) {
disabled_ranges = new (Allocator::Allocate(sizeof(DisabledRangeMap)))
DisabledRangeMap;
}
RangeValue value;
value.start_address = reinterpret_cast<uintptr_t>(start_address);
value.max_depth = max_depth;
if (disabled_ranges->
insert(make_pair(reinterpret_cast<uintptr_t>(end_address),
value)).second) {
RAW_VLOG(1, "Disabling leak checking in stack traces "
"under frame addresses between %p..%p",
start_address, end_address);
} else { // check that this is just a verbatim repetition
RangeValue const& val =
disabled_ranges->find(reinterpret_cast<uintptr_t>(end_address))->second;
if (val.max_depth != value.max_depth ||
val.start_address != value.start_address) {
RAW_LOG(FATAL, "Two DisableChecksToHereFrom calls conflict: "
"(%p, %p, %d) vs. (%p, %p, %d)",
(void*)value.start_address, end_address,
value.max_depth,
start_address, end_address, max_depth);
}
}
}
void HeapLeakChecker::DisableChecksAtLocked(const void* address) {
RAW_DCHECK(heap_checker_lock.IsHeld(), "");
if (disabled_addresses == NULL) {
disabled_addresses = new (Allocator::Allocate(sizeof(DisabledAddressSet)))
DisabledAddressSet;
}
// disable the requested address
if (disabled_addresses->insert(reinterpret_cast<uintptr_t>(address)).second) {
RAW_VLOG(1, "Disabling leak checking in stack traces "
"under frame address %p", address);
}
}
bool HeapLeakChecker::HaveOnHeapLocked(const void** ptr, size_t* object_size) {
RAW_DCHECK(heap_checker_lock.IsHeld(), "");
// Size of the C++ object array size integer
// (potentially compiler dependent; 4 on i386 and gcc; 8 on x86_64 and gcc)
const int kArraySizeOffset = sizeof(size_t);
// sizeof(basic_string<...>::_Rep) for C++ library of gcc 3.4
// (basically three integer counters;
// library/compiler dependent; 12 on i386 and gcc)
const int kStringOffset = sizeof(size_t) * 3;
// Size of refcount used by UnicodeString in third_party/icu.
const int kUnicodeStringOffset = sizeof(uint32);
// NOTE: One can add more similar offset cases below
// even when they do not happen for the used compiler/library;
// all that's impacted is
// - HeapLeakChecker's performace during live heap walking
// - and a slightly greater chance to mistake random memory bytes
// for a pointer and miss a leak in a particular run of a binary.
bool result = true;
if (heap_profile->FindAlloc(*ptr, object_size)) {
// done
} else if (heap_profile->FindAlloc(reinterpret_cast<const char*>(*ptr)
- kArraySizeOffset,
object_size) &&
*object_size > kArraySizeOffset) {
// this case is to account for the array size stored inside of
// the memory allocated by new FooClass[size] for classes with destructors
*ptr = reinterpret_cast<const char*>(*ptr) - kArraySizeOffset;
RAW_VLOG(7, "Got poiter into %p at +%d", ptr, kArraySizeOffset);
} else if (heap_profile->FindAlloc(reinterpret_cast<const char*>(*ptr)
- kStringOffset,
object_size) &&
*object_size > kStringOffset) {
// this case is to account for basic_string<> representation in
// newer C++ library versions when the kept pointer points to inside of
// the allocated region
*ptr = reinterpret_cast<const char*>(*ptr) - kStringOffset;
RAW_VLOG(7, "Got poiter into %p at +%d", ptr, kStringOffset);
} else if (kUnicodeStringOffset != kArraySizeOffset &&
heap_profile->FindAlloc(
reinterpret_cast<const char*>(*ptr) - kUnicodeStringOffset,
object_size) &&
*object_size > kUnicodeStringOffset) {
// this case is to account for third party UnicodeString.
// UnicodeString stores a 32-bit refcount (in both 32-bit and
// 64-bit binaries) as the first uint32 in the allocated memory
// and a pointer points into the second uint32 behind the refcount.
*ptr = reinterpret_cast<const char*>(*ptr) - kUnicodeStringOffset;
RAW_VLOG(7, "Got poiter into %p at +%d", ptr, kUnicodeStringOffset);
} else {
result = false;
}
return result;
}
const void* HeapLeakChecker::GetAllocCaller(void* ptr) {
// this is used only in unittest, so the heavy checks are fine
HeapProfileTable::AllocInfo info;
heap_checker_lock.Lock();
CHECK(heap_profile->FindAllocDetails(ptr, &info));
heap_checker_lock.Unlock();
CHECK(info.stack_depth >= 1);
return info.call_stack[0];
}
syntax highlighted by Code2HTML, v. 0.9.1