heap-checker.cc

// -*- Mode: C++; c-basic-offset: 2; indent-tabs-mode: nil -*-
// Copyright (c) 2005, Google Inc.
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
//     * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//     * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
//     * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// ---
// All Rights Reserved.
//
// Author: Maxim Lifantsev
//

#include "config.h"

#ifndef __linux__
#error we only support non-ancient Linux-es with native TLS support
#endif

#include <errno.h>
#include <fcntl.h>    // for O_RDONLY (we use syscall to do actual reads)
#include <pthread.h>
#include <stddef.h>
#include <string.h>
#include <sys/mman.h> // TODO: check if needed
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/types.h>
#include <sys/uio.h>
#include <time.h>
#include <unistd.h>

#include <sys/ptrace.h>
#include <sys/procfs.h>
#include <sys/user.h>
#include <elf.h> // NT_PRSTATUS

#include <algorithm>
#include <functional>
#include <map>
#include <set>
#include <string>
#include <utility>
#include <vector>

#include <gperftools/heap-checker.h>

#include <gperftools/malloc_extension.h>
#include <gperftools/malloc_hook.h>
#include <gperftools/stacktrace.h>
#include <gperftools/tcmalloc.h>

#include "base/basictypes.h"
#include "base/commandlineflags.h"
#include "base/googleinit.h"
#include "base/linuxthreads.h"
#include "base/logging.h"
#include "base/low_level_alloc.h"
#include "base/proc_maps_iterator.h"
#include "base/spinlock.h"
#include "base/stl_allocator.h"
#include "base/sysinfo.h"
#include "heap-profile-table.h"
#include "malloc_hook-inl.h"
#include "memory_region_map.h"
#include "safe_strerror.h"

// When dealing with ptrace-ed threads, we need to capture all thread
// registers (as potential live pointers), and we need to capture
// thread's stack pointer to scan stack. capture_regs function uses
// ptrace API to grab and scan registers and fetch stack pointer.
template <typename Body>
static std::pair<bool, uintptr_t> CaptureRegs(pid_t tid, const Body& body) {
  uintptr_t sp_offset;
#if defined(__aarch64__)
  sp_offset = offsetof(user_regs_struct, sp);
#elif defined(__arm__)
  sp_offset = 13 * sizeof(uintptr_t); // reg 13 is sp on legacy arms
#elif defined(__x86_64__)
  sp_offset = offsetof(user_regs_struct, rsp);
#elif defined(__i386__)
  sp_offset = offsetof(user_regs_struct, esp);
#elif defined(__riscv)
  sp_offset = 2 * sizeof(uintptr_t); // register #2 is SP on riscv-s
#elif defined(__PPC__)
  sp_offset = 1 * sizeof(uintptr_t);
#elif defined(__mips__)
  sp_offset = offsetof(struct user, regs[EF_REG29]);
#else
  // unsupported HW architecture. Single-threaded programs don't run
  // this code, so we still have chance to be useful on less supported
  // architectures.
  abort();
#endif

  elf_gregset_t regs;
  int rv;

  // PTRACE_GETREGSET is better interface, but manpage says it is
  // 2.6.34 or later. RHEL6's kernel is, sadly, older. Yet, I'd like
  // us to still support rhel6, so we handle this case too.
#ifdef PTRACE_GETREGSET
  iovec iov = {&regs, sizeof(regs)};
  rv = syscall(SYS_ptrace, PTRACE_GETREGSET, tid, NT_PRSTATUS, &iov);
  if (rv == 0) {
    if (iov.iov_len != sizeof(regs)) {
      abort(); // bug?!
    }
  }
#else
  errno = ENOSYS;
  rv = -1;
#endif

  // Some Linux architectures and glibc versions only have
  // PTRACE_GETREGSET, some only PTRACE_GETREGS and some both.
  //
  // But glibc tends to define PTRACE_XYZ constants as enums. Some
  // newer versions also do define, but older glibc (i.e. as shipped
  // by rhel 6) only define PT_GETREGS (to enum value
  // PTRACE_GETREGS). Bionic and musl do regular defines,
  // thankfully. So there seem to be no absolutely perfect way to
  // detect things.
  //
  // We do detect older interface detection by testing defines for
  // both PTRACE_GETREGS and PT_GETREGS. Which seems to work for range
  // of OS-es we try to support.
#if defined(PTRACE_GETREGS) || defined(PT_GETREGS)
  if (rv < 0 && errno == ENOSYS) {
    rv = syscall(SYS_ptrace, PTRACE_GETREGS, tid, nullptr, &regs);
  }
#endif

  if (rv < 0) {
    return {false, 0};
  }

  uintptr_t sp = *reinterpret_cast<uintptr_t*>(reinterpret_cast<uintptr_t>(&regs) + sp_offset);
  for (void** p = reinterpret_cast<void**>(&regs);
       p < reinterpret_cast<void**>(&regs + 1); ++p) {
    body(*p);
  }

  return {true, sp};
}

using std::string;
using std::basic_string;
using std::pair;
using std::map;
using std::set;
using std::vector;
using std::swap;
using std::make_pair;
using std::min;
using std::max;
using std::less;
using std::char_traits;

// If current process is being ptrace()d, 'TracerPid' in /proc/self/status
// will be non-zero.
static bool IsDebuggerAttached(void) {    // only works under linux, probably
  char buf[256];   // TracerPid comes relatively earlier in status output
  int fd = open("/proc/self/status", O_RDONLY);
  if (fd == -1) {
    return false;  // Can't tell for sure.
  }
  const int len = read(fd, buf, sizeof(buf));
  bool rc = false;
  if (len > 0) {
    const char *const kTracerPid = "TracerPid:\t";
    buf[len - 1] = '\0';
    const char *p = strstr(buf, kTracerPid);
    if (p != NULL) {
      rc = (strncmp(p + strlen(kTracerPid), "0\n", 2) != 0);
    }
  }
  close(fd);
  return rc;
}

// This is the default if you don't link in -lprofiler
extern "C" {
ATTRIBUTE_WEAK PERFTOOLS_DLL_DECL int ProfilingIsEnabledForAllThreads();
int ProfilingIsEnabledForAllThreads() { return false; }
}

//----------------------------------------------------------------------
// Flags that control heap-checking
//----------------------------------------------------------------------

DEFINE_string(heap_check,
              EnvToString("HEAPCHECK", ""),
              "The heap leak checking to be done over the whole executable: "
              "\"minimal\", \"normal\", \"strict\", "
              "\"draconian\", \"as-is\", and \"local\" "
              " or the empty string are the supported choices. "
              "(See HeapLeakChecker_InternalInitStart for details.)");

DEFINE_bool(heap_check_report, true, "Obsolete");

DEFINE_bool(heap_check_before_constructors,
            true,
            "deprecated; pretty much always true now");

DEFINE_bool(heap_check_after_destructors,
            EnvToBool("HEAP_CHECK_AFTER_DESTRUCTORS", false),
            "If overall heap check is to end after global destructors "
            "or right after all REGISTER_HEAPCHECK_CLEANUP's");

DEFINE_bool(heap_check_strict_check, true, "Obsolete");

DEFINE_bool(heap_check_ignore_global_live,
            EnvToBool("HEAP_CHECK_IGNORE_GLOBAL_LIVE", true),
            "If overall heap check is to ignore heap objects reachable "
            "from the global data");

DEFINE_bool(heap_check_identify_leaks,
            EnvToBool("HEAP_CHECK_IDENTIFY_LEAKS", false),
            "If heap check should generate the addresses of the leaked "
            "objects in the memory leak profiles.  This may be useful "
            "in tracking down leaks where only a small fraction of "
            "objects allocated at the same stack trace are leaked.");

DEFINE_bool(heap_check_ignore_thread_live,
            EnvToBool("HEAP_CHECK_IGNORE_THREAD_LIVE", true),
            "If set to true, objects reachable from thread stacks "
            "and registers are not reported as leaks");

DEFINE_bool(heap_check_test_pointer_alignment,
            EnvToBool("HEAP_CHECK_TEST_POINTER_ALIGNMENT", false),
            "Set to true to check if the found leak can be due to "
            "use of unaligned pointers");

// Alignment at which all pointers in memory are supposed to be located;
// use 1 if any alignment is ok.
// heap_check_test_pointer_alignment flag guides if we try the value of 1.
// The larger it can be, the lesser is the chance of missing real leaks.
static const size_t kPointerSourceAlignment = sizeof(void*);
DEFINE_int32(heap_check_pointer_source_alignment,
	     EnvToInt("HEAP_CHECK_POINTER_SOURCE_ALIGNMENT",
                      kPointerSourceAlignment),
             "Alignment at which all pointers in memory are supposed to be "
             "located.  Use 1 if any alignment is ok.");

// A reasonable default to handle pointers inside of typical class objects:
// Too low and we won't be able to traverse pointers to normally-used
// nested objects and base parts of multiple-inherited objects.
// Too high and it will both slow down leak checking (FindInsideAlloc
// in HaveOnHeapLocked will get slower when there are large on-heap objects)
// and make it probabilistically more likely to miss leaks
// of large-sized objects.
static const int64_t kHeapCheckMaxPointerOffset = 1024;
DEFINE_int64(heap_check_max_pointer_offset,
	     EnvToInt("HEAP_CHECK_MAX_POINTER_OFFSET",
                      kHeapCheckMaxPointerOffset),
             "Largest pointer offset for which we traverse "
             "pointers going inside of heap allocated objects. "
             "Set to -1 to use the actual largest heap object size.");

DEFINE_bool(heap_check_run_under_gdb,
            EnvToBool("HEAP_CHECK_RUN_UNDER_GDB", false),
            "If false, turns off heap-checking library when running under gdb "
            "(normally, set to 'true' only when debugging the heap-checker)");

DEFINE_int32(heap_check_delay_seconds, 0,
             "Number of seconds to delay on-exit heap checking."
             " If you set this flag,"
             " you may also want to set exit_timeout_seconds in order to"
             " avoid exit timeouts.\n"
             "NOTE: This flag is to be used only to help diagnose issues"
             " where it is suspected that the heap checker is reporting"
             " false leaks that will disappear if the heap checker delays"
             " its checks. Report any such issues to the heap-checker"
             " maintainer(s).");

//----------------------------------------------------------------------

DEFINE_string(heap_profile_pprof,
              EnvToString("PPROF_PATH", "pprof"),
              "OBSOLETE; not used");

DEFINE_string(heap_check_dump_directory,
              EnvToString("HEAP_CHECK_DUMP_DIRECTORY", "/tmp"),
              "Directory to put heap-checker leak dump information");


//----------------------------------------------------------------------
// HeapLeakChecker global data
//----------------------------------------------------------------------

// Global lock for all the global data of this module.
static SpinLock heap_checker_lock;

//----------------------------------------------------------------------

// Heap profile prefix for leak checking profiles.
// Gets assigned once when leak checking is turned on, then never modified.
static const string* profile_name_prefix = NULL;

// Whole-program heap leak checker.
// Gets assigned once when leak checking is turned on,
// then main_heap_checker is never deleted.
static HeapLeakChecker* main_heap_checker = NULL;

// Whether we will use main_heap_checker to do a check at program exit
// automatically. In any case user can ask for more checks on main_heap_checker
// via GlobalChecker().
static bool do_main_heap_check = false;

// The heap profile we use to collect info about the heap.
// This is created in HeapLeakChecker::BeforeConstructorsLocked
// together with setting heap_checker_on (below) to true
// and registering our new/delete malloc hooks;
// similarly all are unset in HeapLeakChecker::TurnItselfOffLocked.
static HeapProfileTable* heap_profile = NULL;

// If we are doing (or going to do) any kind of heap-checking.
static bool heap_checker_on = false;

// pid of the process that does whole-program heap leak checking
static pid_t heap_checker_pid = 0;

// If we did heap profiling during global constructors execution
static bool constructor_heap_profiling = false;

// RAW_VLOG level we dump key INFO messages at.  If you want to turn
// off these messages, set the environment variable PERFTOOLS_VERBOSE=-1.
static const int heap_checker_info_level = 0;

//----------------------------------------------------------------------
// HeapLeakChecker's own memory allocator that is
// independent of the normal program allocator.
//----------------------------------------------------------------------

// Wrapper of LowLevelAlloc for STL_Allocator and direct use.
// We always access this class under held heap_checker_lock,
// this allows us to in particular protect the period when threads are stopped
// at random spots with TCMalloc_ListAllProcessThreads by heap_checker_lock,
// w/o worrying about the lock in LowLevelAlloc::Arena.
// We rely on the fact that we use an own arena with an own lock here.
class HeapLeakChecker::Allocator {
 public:
  static void Init() {
    RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    RAW_DCHECK(arena_ == NULL, "");
    arena_ = LowLevelAlloc::NewArena(nullptr);
  }
  static void Shutdown() {
    RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    if (!LowLevelAlloc::DeleteArena(arena_)  ||  alloc_count_ != 0) {
      RAW_LOG(FATAL, "Internal heap checker leak of %d objects", alloc_count_);
    }
  }
  static int alloc_count() {
    RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    return alloc_count_;
  }
  static void* Allocate(size_t n) {
    RAW_DCHECK(arena_  &&  heap_checker_lock.IsHeld(), "");
    void* p = LowLevelAlloc::AllocWithArena(n, arena_);
    if (p) alloc_count_ += 1;
    return p;
  }
  static void Free(void* p) {
    RAW_DCHECK(heap_checker_lock.IsHeld(), "");
    if (p) alloc_count_ -= 1;
    LowLevelAlloc::Free(p);
  }
  static void Free(void* p, size_t /* n */) {
    Free(p);
  }
  // destruct, free, and make *p to be NULL
  template<typename T> static void DeleteAndNull(T** p) {
    (*p)->~T();
    Free(*p);
    *p = NULL;
  }
  template<typename T> static void DeleteAndNullIfNot(T** p) {
    if (*p != NULL) DeleteAndNull(p);
  }
 private:
  static LowLevelAlloc::Arena* arena_;
  static int alloc_count_;
};

LowLevelAlloc::Arena* HeapLeakChecker::Allocator::arena_ = NULL;
int HeapLeakChecker::Allocator::alloc_count_ = 0;

//----------------------------------------------------------------------
// HeapLeakChecker live object tracking components
//----------------------------------------------------------------------

// Cases of live object placement we distinguish
enum ObjectPlacement {
  MUST_BE_ON_HEAP,   // Must point to a live object of the matching size in the
                     // heap_profile map of the heap when we get to it
  IGNORED_ON_HEAP,   // Is a live (ignored) object on heap
  MAYBE_LIVE,        // Is a piece of writable memory from /proc/self/maps
  IN_GLOBAL_DATA,    // Is part of global data region of the executable
  THREAD_DATA,       // Part of a thread stack and a thread descriptor with TLS
  THREAD_REGISTERS,  // Values in registers of some thread
};

// Information about an allocated object
struct AllocObject {
  const void* ptr;        // the object
  uintptr_t size;         // its size
  ObjectPlacement place;  // where ptr points to

  AllocObject(const void* p, size_t s, ObjectPlacement l)
    : ptr(p), size(s), place(l) { }
};

// All objects (memory ranges) ignored via HeapLeakChecker::IgnoreObject
// Key is the object's address; value is its size.
typedef map<uintptr_t, size_t, less<uintptr_t>,
            STL_Allocator<pair<const uintptr_t, size_t>,
                          HeapLeakChecker::Allocator>
           > IgnoredObjectsMap;
static IgnoredObjectsMap* ignored_objects = NULL;

// All objects (memory ranges) that we consider to be the sources of pointers
// to live (not leaked) objects.
// At different times this holds (what can be reached from) global data regions
// and the objects we've been told to ignore.
// For any AllocObject::ptr "live_objects" is supposed to contain at most one
// record at any time. We maintain this by checking with the heap_profile map
// of the heap and removing the live heap objects we've handled from it.
// This vector is maintained as a stack and the frontier of reachable
// live heap objects in our flood traversal of them.
typedef vector<AllocObject,
               STL_Allocator<AllocObject, HeapLeakChecker::Allocator>
              > LiveObjectsStack;
static LiveObjectsStack* live_objects = NULL;

// A special string type that uses my allocator
typedef basic_string<char, char_traits<char>,
                     STL_Allocator<char, HeapLeakChecker::Allocator>
                    > HCL_string;

// A placeholder to fill-in the starting values for live_objects
// for each library so we can keep the library-name association for logging.
typedef map<HCL_string, LiveObjectsStack, less<HCL_string>,
            STL_Allocator<pair<const HCL_string, LiveObjectsStack>,
                          HeapLeakChecker::Allocator>
           > LibraryLiveObjectsStacks;
static LibraryLiveObjectsStacks* library_live_objects = NULL;

// Value stored in the map of disabled address ranges;
// its key is the end of the address range.
// We'll ignore allocations with a return address in a disabled range
// if the address occurs at 'max_depth' or less in the stack trace.
struct HeapLeakChecker::RangeValue {
  uintptr_t start_address;  // the start of the range
  int       max_depth;      // the maximal stack depth to disable at
};
typedef map<uintptr_t, HeapLeakChecker::RangeValue, less<uintptr_t>,
            STL_Allocator<pair<const uintptr_t, HeapLeakChecker::RangeValue>,
                          HeapLeakChecker::Allocator>
           > DisabledRangeMap;
// The disabled program counter address ranges for profile dumping
// that are registered with HeapLeakChecker::DisableChecksFromToLocked.
static DisabledRangeMap* disabled_ranges = NULL;

// Set of stack tops.
// These are used to consider live only appropriate chunks of the memory areas
// that are used for stacks (and maybe thread-specific data as well)
// so that we do not treat pointers from outdated stack frames as live.
typedef set<uintptr_t, less<uintptr_t>,
            STL_Allocator<uintptr_t, HeapLeakChecker::Allocator>
           > StackTopSet;
static StackTopSet* stack_tops = NULL;

// A map of ranges of code addresses for the system libraries
// that can mmap/mremap/sbrk-allocate memory regions for stacks
// and thread-local storage that we want to consider as live global data.
// Maps from the end address to the start address.
typedef map<uintptr_t, uintptr_t, less<uintptr_t>,
            STL_Allocator<pair<const uintptr_t, uintptr_t>,
                          HeapLeakChecker::Allocator>
           > GlobalRegionCallerRangeMap;
static GlobalRegionCallerRangeMap* global_region_caller_ranges = NULL;

// TODO(maxim): make our big data structs into own modules

// Disabler is implemented by keeping track of a per-thread count
// of active Disabler objects.  Any objects allocated while the
// count > 0 are not reported.

static __thread int thread_disable_counter ATTR_INITIAL_EXEC;

inline int get_thread_disable_counter() {
  return thread_disable_counter;
}
inline void set_thread_disable_counter(int value) {
  thread_disable_counter = value;
}

HeapLeakChecker::Disabler::Disabler() {
  // It is faster to unconditionally increment the thread-local
  // counter than to check whether or not heap-checking is on
  // in a thread-safe manner.
  int counter = get_thread_disable_counter();
  set_thread_disable_counter(counter + 1);
  RAW_VLOG(10, "Increasing thread disable counter to %d", counter + 1);
}

HeapLeakChecker::Disabler::~Disabler() {
  int counter = get_thread_disable_counter();
  RAW_DCHECK(counter > 0, "");
  if (counter > 0) {
    set_thread_disable_counter(counter - 1);
    RAW_VLOG(10, "Decreasing thread disable counter to %d", counter);
  } else {
    RAW_VLOG(0, "Thread disable counter underflow : %d", counter);
  }
}

//----------------------------------------------------------------------

// The size of the largest heap object allocated so far.
static size_t max_heap_object_size = 0;
// The possible range of addresses that can point
// into one of the elements of heap_objects.
static uintptr_t min_heap_address = uintptr_t(-1LL);
static uintptr_t max_heap_address = 0;

//----------------------------------------------------------------------

// Simple casting helpers for uintptr_t and void*:
template<typename T>
inline static const void* AsPtr(T addr) {
  return reinterpret_cast<void*>(addr);
}
inline static uintptr_t AsInt(const void* ptr) {
  return reinterpret_cast<uintptr_t>(ptr);
}

//----------------------------------------------------------------------

// We've seen reports that strstr causes heap-checker crashes in some
// libc's (?):
//    https://github.com/gperftools/gperftools/issues/265
// It's simple enough to use our own.  This is not in time-critical code.
static const char* hc_strstr(const char* s1, const char* s2) {
  const size_t len = strlen(s2);
  RAW_CHECK(len > 0, "Unexpected empty string passed to strstr()");
  for (const char* p = strchr(s1, *s2); p != NULL; p = strchr(p+1, *s2)) {
    if (strncmp(p, s2, len) == 0) {
      return p;
    }
  }
  return NULL;
}

//----------------------------------------------------------------------

// Our hooks for MallocHook
static void NewHook(const void* ptr, size_t size) {
  if (ptr != NULL) {
    const int counter = get_thread_disable_counter();
    const bool ignore = (counter > 0);
    RAW_VLOG(16, "Recording Alloc: %p of %zu; %d", ptr, size,
             int(counter));

    // Fetch the caller's stack trace before acquiring heap_checker_lock.
    void* stack[HeapProfileTable::kMaxStackDepth];
    int depth = HeapProfileTable::GetCallerStackTrace(0, stack);

    { SpinLockHolder l(&heap_checker_lock);
      if (size > max_heap_object_size) max_heap_object_size = size;
      uintptr_t addr = AsInt(ptr);
      if (addr < min_heap_address) min_heap_address = addr;
      addr += size;
      if (addr > max_heap_address) max_heap_address = addr;
      if (heap_checker_on) {
        heap_profile->RecordAlloc(ptr, size, depth, stack);
        if (ignore) {
          heap_profile->MarkAsIgnored(ptr);
        }
      }
    }
    RAW_VLOG(17, "Alloc Recorded: %p of %zu", ptr, size);
  }
}

static void DeleteHook(const void* ptr) {
  if (ptr != NULL) {
    RAW_VLOG(16, "Recording Free %p", ptr);
    { SpinLockHolder l(&heap_checker_lock);
      if (heap_checker_on) heap_profile->RecordFree(ptr);
    }
    RAW_VLOG(17, "Free Recorded: %p", ptr);
  }
}

//----------------------------------------------------------------------

enum StackDirection {
  GROWS_TOWARDS_HIGH_ADDRESSES,
  GROWS_TOWARDS_LOW_ADDRESSES,
  UNKNOWN_DIRECTION
};

// Determine which way the stack grows:

static StackDirection ATTRIBUTE_NOINLINE GetStackDirection(
    const uintptr_t *const ptr) {
  uintptr_t x;
  if (&x < ptr)
    return GROWS_TOWARDS_LOW_ADDRESSES;
  if (ptr < &x)
    return GROWS_TOWARDS_HIGH_ADDRESSES;

  RAW_CHECK(0, "");  // Couldn't determine the stack direction.

  return UNKNOWN_DIRECTION;
}

// Direction of stack growth (will initialize via GetStackDirection())
static StackDirection stack_direction = UNKNOWN_DIRECTION;

// This routine is called for every thread stack we know about to register it.
static void RegisterStackLocked(const void* top_ptr) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
  RAW_VLOG(10, "Thread stack at %p", top_ptr);
  uintptr_t top = AsInt(top_ptr);
  stack_tops->insert(top);  // add for later use

  // make sure stack_direction is initialized
  if (stack_direction == UNKNOWN_DIRECTION) {
    stack_direction = GetStackDirection(&top);
  }

  // Find memory region with this stack
  MemoryRegionMap::Region region;
  if (MemoryRegionMap::FindAndMarkStackRegion(top, &region)) {
    // Make the proper portion of the stack live:
    if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) {
      RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes",
                  top_ptr, region.end_addr - top);
      live_objects->push_back(AllocObject(top_ptr, region.end_addr - top,
                                          THREAD_DATA));
    } else {  // GROWS_TOWARDS_HIGH_ADDRESSES
      RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes",
                  AsPtr(region.start_addr),
                  top - region.start_addr);
      live_objects->push_back(AllocObject(AsPtr(region.start_addr),
                                          top - region.start_addr,
                                          THREAD_DATA));
    }
  // not in MemoryRegionMap, look in library_live_objects:
  } else if (FLAGS_heap_check_ignore_global_live) {
    for (LibraryLiveObjectsStacks::iterator lib = library_live_objects->begin();
         lib != library_live_objects->end(); ++lib) {
      for (LiveObjectsStack::iterator span = lib->second.begin();
           span != lib->second.end(); ++span) {
        uintptr_t start = AsInt(span->ptr);
        uintptr_t end = start + span->size;
        if (start <= top  &&  top < end) {
          RAW_VLOG(11, "Stack at %p is inside /proc/self/maps chunk %p..%p",
                      top_ptr, AsPtr(start), AsPtr(end));
          // Shrink start..end region by chopping away the memory regions in
          // MemoryRegionMap that land in it to undo merging of regions
          // in /proc/self/maps, so that we correctly identify what portion
          // of start..end is actually the stack region.
          uintptr_t stack_start = start;
          uintptr_t stack_end = end;
          // can optimize-away this loop, but it does not run often
          RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
          for (MemoryRegionMap::RegionIterator r =
                 MemoryRegionMap::BeginRegionLocked();
               r != MemoryRegionMap::EndRegionLocked(); ++r) {
            if (top < r->start_addr  &&  r->start_addr < stack_end) {
              stack_end = r->start_addr;
            }
            if (stack_start < r->end_addr  &&  r->end_addr <= top) {
              stack_start = r->end_addr;
            }
          }
          if (stack_start != start  ||  stack_end != end) {
            RAW_VLOG(11, "Stack at %p is actually inside memory chunk %p..%p",
                        top_ptr, AsPtr(stack_start), AsPtr(stack_end));
          }
          // Make the proper portion of the stack live:
          if (stack_direction == GROWS_TOWARDS_LOW_ADDRESSES) {
            RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes",
                        top_ptr, stack_end - top);
            live_objects->push_back(
              AllocObject(top_ptr, stack_end - top, THREAD_DATA));
          } else {  // GROWS_TOWARDS_HIGH_ADDRESSES
            RAW_VLOG(11, "Live stack at %p of %" PRIuPTR " bytes",
                        AsPtr(stack_start), top - stack_start);
            live_objects->push_back(
              AllocObject(AsPtr(stack_start), top - stack_start, THREAD_DATA));
          }
          lib->second.erase(span);  // kill the rest of the region
          // Put the non-stack part(s) of the region back:
          if (stack_start != start) {
            lib->second.push_back(AllocObject(AsPtr(start), stack_start - start,
                                  MAYBE_LIVE));
          }
          if (stack_end != end) {
            lib->second.push_back(AllocObject(AsPtr(stack_end), end - stack_end,
                                  MAYBE_LIVE));
          }
          return;
        }
      }
    }
    RAW_LOG(ERROR, "Memory region for stack at %p not found. "
                   "Will likely report false leak positives.", top_ptr);
  }
}

// Iterator for heap allocation map data to make ignored objects "live"
// (i.e., treated as roots for the mark-and-sweep phase)
static void MakeIgnoredObjectsLiveCallbackLocked(
    const void* ptr, const HeapProfileTable::AllocInfo& info) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  if (info.ignored) {
    live_objects->push_back(AllocObject(ptr, info.object_size,
                                        MUST_BE_ON_HEAP));
  }
}

// Iterator for heap allocation map data to make objects allocated from
// disabled regions of code to be live.
static void MakeDisabledLiveCallbackLocked(
    const void* ptr, const HeapProfileTable::AllocInfo& info) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  bool stack_disable = false;
  bool range_disable = false;
  for (int depth = 0; depth < info.stack_depth; depth++) {
    uintptr_t addr = AsInt(info.call_stack[depth]);
    if (disabled_ranges) {
      DisabledRangeMap::const_iterator iter
        = disabled_ranges->upper_bound(addr);
      if (iter != disabled_ranges->end()) {
        RAW_DCHECK(iter->first > addr, "");
        if (iter->second.start_address < addr  &&
            iter->second.max_depth > depth) {
          range_disable = true;  // in range; dropping
          break;
        }
      }
    }
  }
  if (stack_disable || range_disable) {
    uintptr_t start_address = AsInt(ptr);
    uintptr_t end_address = start_address + info.object_size;
    StackTopSet::const_iterator iter
      = stack_tops->lower_bound(start_address);
    if (iter != stack_tops->end()) {
      RAW_DCHECK(*iter >= start_address, "");
      if (*iter < end_address) {
        // We do not disable (treat as live) whole allocated regions
        // if they are used to hold thread call stacks
        // (i.e. when we find a stack inside).
        // The reason is that we'll treat as live the currently used
        // stack portions anyway (see RegisterStackLocked),
        // and the rest of the region where the stack lives can well
        // contain outdated stack variables which are not live anymore,
        // hence should not be treated as such.
        RAW_VLOG(11, "Not %s-disabling %zu bytes at %p"
                    ": have stack inside: %p",
                    (stack_disable ? "stack" : "range"),
                    info.object_size, ptr, AsPtr(*iter));
        return;
      }
    }
    RAW_VLOG(11, "%s-disabling %zu bytes at %p",
                (stack_disable ? "Stack" : "Range"), info.object_size, ptr);
    live_objects->push_back(AllocObject(ptr, info.object_size,
                                        MUST_BE_ON_HEAP));
  }
}

static const char kUnnamedProcSelfMapEntry[] = "UNNAMED";

// This function takes some fields from a /proc/self/maps line:
//
//   start_address  start address of a memory region.
//   end_address    end address of a memory region
//   permissions    rwx + private/shared bit
//   filename       filename of the mapped file
//
// If the region is not writeable, then it cannot have any heap
// pointers in it, otherwise we record it as a candidate live region
// to get filtered later.
static void RecordGlobalDataLocked(uintptr_t start_address,
                                   uintptr_t end_address,
                                   const char* permissions,
                                   const char* filename) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  // Ignore non-writeable regions.
  if (strchr(permissions, 'w') == NULL) return;
  if (filename == NULL  ||  *filename == '\0') {
    filename = kUnnamedProcSelfMapEntry;
  }
  RAW_VLOG(11, "Looking into %s: 0x%" PRIxPTR "..0x%" PRIxPTR,
              filename, start_address, end_address);
  (*library_live_objects)[filename].
    push_back(AllocObject(AsPtr(start_address),
                          end_address - start_address,
                          MAYBE_LIVE));
}

// See if 'library' from /proc/self/maps has base name 'library_base'
// i.e. contains it and has '.' or '-' after it.
static bool IsLibraryNamed(const char* library, const char* library_base) {
  const char* p = hc_strstr(library, library_base);
  size_t sz = strlen(library_base);
  return p != NULL  &&  (p[sz] == '.'  ||  p[sz] == '-');
}

// static
void HeapLeakChecker::DisableLibraryAllocsLocked(const char* library,
                                                 uintptr_t start_address,
                                                 uintptr_t end_address) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  int depth = 0;
  // TODO(maxim): maybe this should be extended to also use objdump
  //              and pick the text portion of the library more precisely.
  if (IsLibraryNamed(library, "/libpthread")  ||
        // libpthread has a lot of small "system" leaks we don't care about.
        // In particular it allocates memory to store data supplied via
        // tcmalloc::SetTlsValue (which can be the only pointer to a heap object).
      IsLibraryNamed(library, "/libdl")  ||
        // library loaders leak some "system" heap that we don't care about
      IsLibraryNamed(library, "/libcrypto")  ||
        // Sometimes libcrypto of OpenSSH is compiled with -fomit-frame-pointer
        // (any library can be, of course, but this one often is because speed
        // is so important for making crypto usable).  We ignore all its
        // allocations because we can't see the call stacks.  We'd prefer
        // to ignore allocations done in files/symbols that match
        // "default_malloc_ex|default_realloc_ex"
        // but that doesn't work when the end-result binary is stripped.
      IsLibraryNamed(library, "/libjvm")  ||
        // JVM has a lot of leaks we don't care about.
      IsLibraryNamed(library, "/libzip")
        // The JVM leaks java.util.zip.Inflater after loading classes.
     ) {
    depth = 1;  // only disable allocation calls directly from the library code
  } else if (IsLibraryNamed(library, "/ld")
               // library loader leaks some "system" heap
               // (e.g. thread-local storage) that we don't care about
            ) {
    depth = 2;  // disable allocation calls directly from the library code
                // and at depth 2 from it.
    // We need depth 2 here solely because of a libc bug that
    // forces us to jump through __memalign_hook and MemalignOverride hoops
    // in tcmalloc.cc.
    // Those buggy __libc_memalign() calls are in ld-linux.so and happen for
    // thread-local storage allocations that we want to ignore here.
    // We go with the depth-2 hack as a workaround for this libc bug:
    // otherwise we'd need to extend MallocHook interface
    // so that correct stack depth adjustment can be propagated from
    // the exceptional case of MemalignOverride.
    // Using depth 2 here should not mask real leaks because ld-linux.so
    // does not call user code.
  }
  if (depth) {
    RAW_VLOG(10, "Disabling allocations from %s at depth %d:", library, depth);
    DisableChecksFromToLocked(AsPtr(start_address), AsPtr(end_address), depth);
    if (IsLibraryNamed(library, "/libpthread")  ||
        IsLibraryNamed(library, "/libdl")  ||
        IsLibraryNamed(library, "/ld")) {
      RAW_VLOG(10, "Global memory regions made by %s will be live data",
                  library);
      if (global_region_caller_ranges == NULL) {
        global_region_caller_ranges =
          new(Allocator::Allocate(sizeof(GlobalRegionCallerRangeMap)))
            GlobalRegionCallerRangeMap;
      }
      global_region_caller_ranges
        ->insert(make_pair(end_address, start_address));
    }
  }
}

// static
HeapLeakChecker::ProcMapsResult HeapLeakChecker::UseProcMapsLocked(
                                  ProcMapsTask proc_maps_task) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  bool saw_shared_lib = false;
  bool saw_nonzero_inode = false;
  bool saw_shared_lib_with_nonzero_inode = false;

  bool ok = tcmalloc::ForEachProcMapping([&] (const tcmalloc::ProcMapping& mapping) {
    const uint64_t start_address = mapping.start, end_address = mapping.end;
    const char *filename = mapping.filename;

    if (start_address >= end_address) {
      // Warn if a line we can be interested in is ill-formed:
      if (mapping.inode != 0) {
        RAW_LOG(ERROR, "Errors reading /proc/self/maps. "
                       "Some global memory regions will not "
                       "be handled correctly.");
      }
      // Silently skip other ill-formed lines: some are possible
      // probably due to the interplay of how /proc/self/maps is updated
      // while we read it in chunks in ProcMapsIterator and
      // do things in this loop.
      return;
    }
    // Determine if any shared libraries are present (this is the same
    // list of extensions as is found in pprof).  We want to ignore
    // 'fake' libraries with inode 0 when determining.  However, some
    // systems don't share inodes via /proc, so we turn off this check
    // if we don't see any evidence that we're getting inode info.
    if (mapping.inode != 0) {
      saw_nonzero_inode = true;
    }
    if ((hc_strstr(filename, "lib") && hc_strstr(filename, ".so")) ||
        hc_strstr(filename, ".dll") ||
        // not all .dylib filenames start with lib. .dylib is big enough
        // that we are unlikely to get false matches just checking that.
        hc_strstr(filename, ".dylib") || hc_strstr(filename, ".bundle")) {
      saw_shared_lib = true;
      if (mapping.inode != 0) {
        saw_shared_lib_with_nonzero_inode = true;
      }
    }

    switch (proc_maps_task) {
      case DISABLE_LIBRARY_ALLOCS:
        // All lines starting like
        // "401dc000-4030f000 r??p 00132000 03:01 13991972  lib/bin"
        // identify a data and code sections of a shared library or our binary
        if (mapping.inode != 0 && strncmp(mapping.flags, "r-xp", 4) == 0) {
          DisableLibraryAllocsLocked(filename, start_address, end_address);
        }
        break;
      case RECORD_GLOBAL_DATA:
        RecordGlobalDataLocked(start_address, end_address,
                               mapping.flags, filename);
        break;
      default:
        RAW_CHECK(0, "");
    }
  });

  if (!ok) {
    int errsv = errno;
    RAW_LOG(ERROR, "Could not open /proc/self/maps: errno=%d. "
            "Libraries will not be handled correctly.", errsv);
    return CANT_OPEN_PROC_MAPS;
  }

  // If /proc/self/maps is reporting inodes properly (we saw a
  // non-zero inode), then we only say we saw a shared lib if we saw a
  // 'real' one, with a non-zero inode.
  if (saw_nonzero_inode) {
    saw_shared_lib = saw_shared_lib_with_nonzero_inode;
  }
  if (!saw_shared_lib) {
    RAW_LOG(ERROR, "No shared libs detected. Will likely report false leak "
                   "positives for statically linked executables.");
    return NO_SHARED_LIBS_IN_PROC_MAPS;
  }
  return PROC_MAPS_USED;
}

// Total number and size of live objects dropped from the profile;
// (re)initialized in IgnoreAllLiveObjectsLocked.
static int64_t live_objects_total;
static int64_t live_bytes_total;

// pid of the thread that is doing the current leak check
// (protected by our lock; IgnoreAllLiveObjectsLocked sets it)
static pid_t self_thread_pid = 0;

// Status of our thread listing callback execution
// (protected by our lock; used from within IgnoreAllLiveObjectsLocked)
static enum {
  CALLBACK_NOT_STARTED,
  CALLBACK_STARTED,
  CALLBACK_COMPLETED,
} thread_listing_status = CALLBACK_NOT_STARTED;

// Ideally to avoid deadlocks this function should not result in any libc
// or other function calls that might need to lock a mutex:
// It is called when all threads of a process are stopped
// at arbitrary points thus potentially holding those locks.
//
// In practice we are calling some simple i/o and sprintf-type library functions
// for logging messages, but use only our own LowLevelAlloc::Arena allocator.
//
// This is known to be buggy: the library i/o function calls are able to cause
// deadlocks when they request a lock that a stopped thread happens to hold.
// This issue as far as we know have so far not resulted in any deadlocks
// in practice, so for now we are taking our chance that the deadlocks
// have insignificant frequency.
//
// If such deadlocks become a problem we should make the i/o calls
// into appropriately direct system calls (or eliminate them),
// in particular write() is not safe and vsnprintf() is potentially dangerous
// due to reliance on locale functions (these are called through RAW_LOG
// and in other ways).
//

/*static*/ int HeapLeakChecker::IgnoreLiveThreadsLocked(void* parameter,
                                                        int num_threads,
                                                        pid_t* thread_pids,
                                                        va_list /*ap*/) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  thread_listing_status = CALLBACK_STARTED;
  RAW_VLOG(11, "Found %d threads (from pid %d)", num_threads, getpid());

  if (FLAGS_heap_check_ignore_global_live) {
    UseProcMapsLocked(RECORD_GLOBAL_DATA);
  }

  // We put the registers from other threads here
  // to make pointers stored in them live.
  vector<void*, STL_Allocator<void*, Allocator> > thread_registers;

  int failures = 0;
  for (int i = 0; i < num_threads; ++i) {
    // the leak checking thread itself is handled
    // specially via self_thread_stack, not here:
    if (thread_pids[i] == self_thread_pid) {
      continue;
    }
    RAW_VLOG(11, "Handling thread with pid %d", thread_pids[i]);
    auto add_reg = [&thread_registers] (void *reg) {
      RAW_VLOG(12, "Thread register %p", reg);
      thread_registers.push_back(reg);
    };
    uintptr_t sp;
    bool ok;
    std::tie(ok, sp) = CaptureRegs(thread_pids[i], add_reg);
    if (ok) {
      RegisterStackLocked(reinterpret_cast<void*>(sp));
    } else {
      failures += 1;
    }
  }
  // Use all the collected thread (stack) liveness sources:
  IgnoreLiveObjectsLocked("threads stack data", "");
  if (thread_registers.size()) {
    // Make thread registers be live heap data sources.
    // we rely here on the fact that vector is in one memory chunk:
    RAW_VLOG(11, "Live registers at %p of %zu bytes",
                &thread_registers[0], thread_registers.size() * sizeof(void*));
    live_objects->push_back(AllocObject(&thread_registers[0],
                                        thread_registers.size() * sizeof(void*),
                                        THREAD_REGISTERS));
    IgnoreLiveObjectsLocked("threads register data", "");
  }
  // Do all other liveness walking while all threads are stopped:
  IgnoreNonThreadLiveObjectsLocked();
  // Can now resume the threads:
  TCMalloc_ResumeAllProcessThreads(num_threads, thread_pids);
  thread_listing_status = CALLBACK_COMPLETED;
  return failures;
}

// Stack top of the thread that is doing the current leak check
// (protected by our lock; IgnoreAllLiveObjectsLocked sets it)
static const void* self_thread_stack_top;

// static
void HeapLeakChecker::IgnoreNonThreadLiveObjectsLocked() {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
  RAW_VLOG(11, "Handling self thread with pid %d", self_thread_pid);
  // Register our own stack:

  // Important that all stack ranges (including the one here)
  // are known before we start looking at them
  // in MakeDisabledLiveCallbackLocked:
  RegisterStackLocked(self_thread_stack_top);
  IgnoreLiveObjectsLocked("stack data", "");

  // Make objects we were told to ignore live:
  if (ignored_objects) {
    for (IgnoredObjectsMap::const_iterator object = ignored_objects->begin();
         object != ignored_objects->end(); ++object) {
      const void* ptr = AsPtr(object->first);
      RAW_VLOG(11, "Ignored live object at %p of %zu bytes",
                  ptr, object->second);
      live_objects->
        push_back(AllocObject(ptr, object->second, MUST_BE_ON_HEAP));
      // we do this liveness check for ignored_objects before doing any
      // live heap walking to make sure it does not fail needlessly:
      size_t object_size;
      if (!(heap_profile->FindAlloc(ptr, &object_size)  &&
            object->second == object_size)) {
        RAW_LOG(FATAL, "Object at %p of %zu bytes from an"
                       " IgnoreObject() has disappeared", ptr, object->second);
      }
    }
    IgnoreLiveObjectsLocked("ignored objects", "");
  }

  // Treat objects that were allocated when a Disabler was live as
  // roots.  I.e., if X was allocated while a Disabler was active,
  // and Y is reachable from X, arrange that neither X nor Y are
  // treated as leaks.
  heap_profile->IterateAllocs(MakeIgnoredObjectsLiveCallbackLocked);
  IgnoreLiveObjectsLocked("disabled objects", "");

  // Make code-address-disabled objects live and ignored:
  // This in particular makes all thread-specific data live
  // because the basic data structure to hold pointers to thread-specific data
  // is allocated from libpthreads and we have range-disabled that
  // library code with UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS);
  // so now we declare all thread-specific data reachable from there as live.
  heap_profile->IterateAllocs(MakeDisabledLiveCallbackLocked);
  IgnoreLiveObjectsLocked("disabled code", "");

  // Actually make global data live:
  if (FLAGS_heap_check_ignore_global_live) {
    bool have_null_region_callers = false;
    for (LibraryLiveObjectsStacks::iterator l = library_live_objects->begin();
         l != library_live_objects->end(); ++l) {
      RAW_CHECK(live_objects->empty(), "");
      // Process library_live_objects in l->second
      // filtering them by MemoryRegionMap:
      // It's safe to iterate over MemoryRegionMap
      // w/o locks here as we are inside MemoryRegionMap::Lock():
      RAW_DCHECK(MemoryRegionMap::LockIsHeld(), "");
      // The only change to MemoryRegionMap possible in this loop
      // is region addition as a result of allocating more memory
      // for live_objects. This won't invalidate the RegionIterator
      // or the intent of the loop.
      // --see the comment by MemoryRegionMap::BeginRegionLocked().
      for (MemoryRegionMap::RegionIterator region =
             MemoryRegionMap::BeginRegionLocked();
           region != MemoryRegionMap::EndRegionLocked(); ++region) {
        // "region" from MemoryRegionMap is to be subtracted from
        // (tentatively live) regions in l->second
        // if it has a stack inside or it was allocated by
        // a non-special caller (not one covered by a range
        // in global_region_caller_ranges).
        // This will in particular exclude all memory chunks used
        // by the heap itself as well as what's been allocated with
        // any allocator on top of mmap.
        bool subtract = true;
        if (!region->is_stack  &&  global_region_caller_ranges) {
          if (region->caller() == 0) {
            have_null_region_callers = true;
          } else {
            GlobalRegionCallerRangeMap::const_iterator iter
              = global_region_caller_ranges->upper_bound(region->caller());
            if (iter != global_region_caller_ranges->end()) {
              RAW_DCHECK(iter->first > region->caller(), "");
              if (iter->second < region->caller()) {  // in special region
                subtract = false;
              }
            }
          }
        }
        if (subtract) {
          // The loop puts the result of filtering l->second into live_objects:
          for (LiveObjectsStack::const_iterator i = l->second.begin();
               i != l->second.end(); ++i) {
            // subtract *region from *i
            uintptr_t start = AsInt(i->ptr);
            uintptr_t end = start + i->size;
            if (region->start_addr <= start  &&  end <= region->end_addr) {
              // full deletion due to subsumption
            } else if (start < region->start_addr  &&
                       region->end_addr < end) {  // cutting-out split
              live_objects->push_back(AllocObject(i->ptr,
                                                  region->start_addr - start,
                                                  IN_GLOBAL_DATA));
              live_objects->push_back(AllocObject(AsPtr(region->end_addr),
                                                  end - region->end_addr,
                                                  IN_GLOBAL_DATA));
            } else if (region->end_addr > start  &&
                       region->start_addr <= start) {  // cut from start
              live_objects->push_back(AllocObject(AsPtr(region->end_addr),
                                                  end - region->end_addr,
                                                  IN_GLOBAL_DATA));
            } else if (region->start_addr > start  &&
                       region->start_addr < end) {  // cut from end
              live_objects->push_back(AllocObject(i->ptr,
                                                  region->start_addr - start,
                                                  IN_GLOBAL_DATA));
            } else {  // pass: no intersection
              live_objects->push_back(AllocObject(i->ptr, i->size,
                                                  IN_GLOBAL_DATA));
            }
          }
          // Move live_objects back into l->second
          // for filtering by the next region.
          live_objects->swap(l->second);
          live_objects->clear();
        }
      }
      // Now get and use live_objects from the final version of l->second:
      if (VLOG_IS_ON(11)) {
        for (LiveObjectsStack::const_iterator i = l->second.begin();
             i != l->second.end(); ++i) {
          RAW_VLOG(11, "Library live region at %p of %" PRIuPTR " bytes",
                      i->ptr, i->size);
        }
      }
      live_objects->swap(l->second);
      IgnoreLiveObjectsLocked("in globals of\n  ", l->first.c_str());
    }
    if (have_null_region_callers) {
      RAW_LOG(ERROR, "Have memory regions w/o callers: "
                     "might report false leaks");
    }
    Allocator::DeleteAndNull(&library_live_objects);
  }
}

// Callback for TCMalloc_ListAllProcessThreads in IgnoreAllLiveObjectsLocked below
// to test/verify that we have just the one main thread, in which case
// we can do everything in that main thread,
// so that CPU profiler can collect all its samples.
// Returns the number of threads in the process.
static int IsOneThread(void* parameter, int num_threads,
                       pid_t* thread_pids, va_list ap) {
  if (num_threads != 1) {
    RAW_LOG(WARNING, "Have threads: Won't CPU-profile the bulk of leak "
                     "checking work happening in IgnoreLiveThreadsLocked!");
  }
  TCMalloc_ResumeAllProcessThreads(num_threads, thread_pids);
  return num_threads;
}

// Dummy for IgnoreAllLiveObjectsLocked below.
// Making it global helps with compiler warnings.
static va_list dummy_ap;

// static
void HeapLeakChecker::IgnoreAllLiveObjectsLocked(const void* self_stack_top) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_CHECK(live_objects == NULL, "");
  live_objects = new(Allocator::Allocate(sizeof(LiveObjectsStack)))
                   LiveObjectsStack;
  stack_tops = new(Allocator::Allocate(sizeof(StackTopSet))) StackTopSet;
  // reset the counts
  live_objects_total = 0;
  live_bytes_total = 0;
  // Reduce max_heap_object_size to FLAGS_heap_check_max_pointer_offset
  // for the time of leak check.
  // FLAGS_heap_check_max_pointer_offset caps max_heap_object_size
  // to manage reasonably low chances of random bytes
  // appearing to be pointing into large actually leaked heap objects.
  const size_t old_max_heap_object_size = max_heap_object_size;
  max_heap_object_size = (
    FLAGS_heap_check_max_pointer_offset != -1
    ? min(size_t(FLAGS_heap_check_max_pointer_offset), max_heap_object_size)
    : max_heap_object_size);
  // Record global data as live:
  if (FLAGS_heap_check_ignore_global_live) {
    library_live_objects =
      new(Allocator::Allocate(sizeof(LibraryLiveObjectsStacks)))
        LibraryLiveObjectsStacks;
  }
  // Ignore all thread stacks:
  thread_listing_status = CALLBACK_NOT_STARTED;
  bool need_to_ignore_non_thread_objects = true;
  self_thread_pid = getpid();
  self_thread_stack_top = self_stack_top;
  if (FLAGS_heap_check_ignore_thread_live) {
    // In case we are doing CPU profiling we'd like to do all the work
    // in the main thread, not in the special thread created by
    // TCMalloc_ListAllProcessThreads, so that CPU profiler can
    // collect all its samples.  The machinery of
    // TCMalloc_ListAllProcessThreads conflicts with the CPU profiler
    // by also relying on signals and ::sigaction.  We can do this
    // (run everything in the main thread) safely only if there's just
    // the main thread itself in our process.  This variable reflects
    // these two conditions:
    bool want_and_can_run_in_main_thread =
      ProfilingIsEnabledForAllThreads()  &&
      TCMalloc_ListAllProcessThreads(NULL, IsOneThread) == 1;
    // When the normal path of TCMalloc_ListAllProcessThreads below is taken,
    // we fully suspend the threads right here before any liveness checking
    // and keep them suspended for the whole time of liveness checking
    // inside of the IgnoreLiveThreadsLocked callback.
    // (The threads can't (de)allocate due to lock on the delete hook but
    //  if not suspended they could still mess with the pointer
    //  graph while we walk it).
    int r = want_and_can_run_in_main_thread
            ? IgnoreLiveThreadsLocked(NULL, 1, &self_thread_pid, dummy_ap)
            : TCMalloc_ListAllProcessThreads(NULL, IgnoreLiveThreadsLocked);
    need_to_ignore_non_thread_objects = r < 0;
    if (r < 0) {
      RAW_LOG(WARNING, "Thread finding failed with %d errno=%d", r, errno);
      if (thread_listing_status == CALLBACK_COMPLETED) {
        RAW_LOG(INFO, "Thread finding callback "
                      "finished ok; hopefully everything is fine");
        need_to_ignore_non_thread_objects = false;
      } else if (thread_listing_status == CALLBACK_STARTED) {
        RAW_LOG(FATAL, "Thread finding callback was "
                       "interrupted or crashed; can't fix this");
      } else {  // CALLBACK_NOT_STARTED
        RAW_LOG(ERROR, "Could not find thread stacks. "
                       "Will likely report false leak positives.");
      }
    } else if (r != 0) {
      RAW_LOG(ERROR, "Thread stacks not found for %d threads. "
                     "Will likely report false leak positives.", r);
    } else {
      RAW_VLOG(11, "Thread stacks appear to be found for all threads");
    }
  } else {
    RAW_LOG(WARNING, "Not looking for thread stacks; "
                     "objects reachable only from there "
                     "will be reported as leaks");
  }
  // Do all other live data ignoring here if we did not do it
  // within thread listing callback with all threads stopped.
  if (need_to_ignore_non_thread_objects) {
    if (FLAGS_heap_check_ignore_global_live) {
      UseProcMapsLocked(RECORD_GLOBAL_DATA);
    }
    IgnoreNonThreadLiveObjectsLocked();
  }
  if (live_objects_total) {
    RAW_VLOG(10, "Ignoring %" PRId64 " reachable objects of %" PRId64 " bytes",
                live_objects_total, live_bytes_total);
  }
  // Free these: we made them here and heap_profile never saw them
  Allocator::DeleteAndNull(&live_objects);
  Allocator::DeleteAndNull(&stack_tops);
  max_heap_object_size = old_max_heap_object_size;  // reset this var
}

// Alignment at which we should consider pointer positions
// in IgnoreLiveObjectsLocked. Will normally use the value of
// FLAGS_heap_check_pointer_source_alignment.
static size_t pointer_source_alignment = kPointerSourceAlignment;
// Global lock for HeapLeakChecker::DoNoLeaks
// to protect pointer_source_alignment.
static SpinLock alignment_checker_lock;

// This function changes the live bits in the heap_profile-table's state:
// we only record the live objects to be skipped.
//
// When checking if a byte sequence points to a heap object we use
// HeapProfileTable::FindInsideAlloc to handle both pointers to
// the start and inside of heap-allocated objects.
// The "inside" case needs to be checked to support
// at least the following relatively common cases:
// - C++ arrays allocated with new FooClass[size] for classes
//   with destructors have their size recorded in a sizeof(int) field
//   before the place normal pointers point to.
// - basic_string<>-s for e.g. the C++ library of gcc 3.4
//   have the meta-info in basic_string<...>::_Rep recorded
//   before the place normal pointers point to.
// - Multiple-inherited objects have their pointers when cast to
//   different base classes pointing inside of the actually
//   allocated object.
// - Sometimes reachability pointers point to member objects of heap objects,
//   and then those member objects point to the full heap object.
// - Third party UnicodeString: it stores a 32-bit refcount
//   (in both 32-bit and 64-bit binaries) as the first uint32
//   in the allocated memory and a normal pointer points at
//   the second uint32 behind the refcount.
// By finding these additional objects here
// we slightly increase the chance to mistake random memory bytes
// for a pointer and miss a leak in a particular run of a binary.
//
/*static*/ void HeapLeakChecker::IgnoreLiveObjectsLocked(const char* name,
                                                         const char* name2) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  int64_t live_object_count = 0;
  int64_t live_byte_count = 0;
  while (!live_objects->empty()) {
    const char* object =
      reinterpret_cast<const char*>(live_objects->back().ptr);
    size_t size = live_objects->back().size;
    const ObjectPlacement place = live_objects->back().place;
    live_objects->pop_back();
    if (place == MUST_BE_ON_HEAP  &&  heap_profile->MarkAsLive(object)) {
      live_object_count += 1;
      live_byte_count += size;
    }
    RAW_VLOG(13, "Looking for heap pointers in %p of %zu bytes",
                object, size);
    const char* const whole_object = object;
    size_t const whole_size = size;
    // Try interpretting any byte sequence in object,size as a heap pointer:
    const size_t remainder = AsInt(object) % pointer_source_alignment;
    if (remainder) {
      object += pointer_source_alignment - remainder;
      if (size >= pointer_source_alignment - remainder) {
        size -= pointer_source_alignment - remainder;
      } else {
        size = 0;
      }
    }
    if (size < sizeof(void*)) continue;

    // Frame pointer omission requires us to use libunwind, which uses direct
    // mmap and munmap system calls, and that needs special handling.
    if (name2 == kUnnamedProcSelfMapEntry) {
      static const uintptr_t page_mask = ~(getpagesize() - 1);
      const uintptr_t addr = reinterpret_cast<uintptr_t>(object);
      if ((addr & page_mask) == 0 && (size & page_mask) == 0) {
        // This is an object we slurped from /proc/self/maps.
        // It may or may not be readable at this point.
        //
        // In case all the above conditions made a mistake, and the object is
        // not related to libunwind, we also verify that it's not readable
        // before ignoring it.
        if (msync(const_cast<char*>(object), size, MS_ASYNC) != 0) {
          // Skip unreadable object, so we don't crash trying to sweep it.
          RAW_VLOG(0, "Ignoring inaccessible object [%p, %p) "
                   "(msync error %d (%s))",
                   object, object + size, errno, tcmalloc::SafeStrError(errno).c_str());
          continue;
        }
      }
    }

    const char* const max_object = object + size - sizeof(void*);
    while (object <= max_object) {
      // potentially unaligned load:
      const uintptr_t addr = *reinterpret_cast<const uintptr_t*>(object);
      // Do fast check before the more expensive HaveOnHeapLocked lookup:
      // this code runs for all memory words that are potentially pointers:
      const bool can_be_on_heap =
        // Order tests by the likelyhood of the test failing in 64/32 bit modes.
        // Yes, this matters: we either lose 5..6% speed in 32 bit mode
        // (which is already slower) or by a factor of 1.5..1.91 in 64 bit mode.
        // After the alignment test got dropped the above performance figures
        // must have changed; might need to revisit this.
#if defined(__x86_64__)
        addr <= max_heap_address  &&  // <= is for 0-sized object with max addr
        min_heap_address <= addr;
#else
        min_heap_address <= addr  &&
        addr <= max_heap_address;  // <= is for 0-sized object with max addr
#endif
      if (can_be_on_heap) {
        const void* ptr = reinterpret_cast<const void*>(addr);
        // Too expensive (inner loop): manually uncomment when debugging:
        // RAW_VLOG(17, "Trying pointer to %p at %p", ptr, object);
        size_t object_size;
        if (HaveOnHeapLocked(&ptr, &object_size)  &&
            heap_profile->MarkAsLive(ptr)) {
          // We take the (hopefully low) risk here of encountering by accident
          // a byte sequence in memory that matches an address of
          // a heap object which is in fact leaked.
          // I.e. in very rare and probably not repeatable/lasting cases
          // we might miss some real heap memory leaks.
          RAW_VLOG(14, "Found pointer to %p of %zu bytes at %p "
                      "inside %p of size %zu",
                      ptr, object_size, object, whole_object, whole_size);
          if (VLOG_IS_ON(15)) {
            // log call stacks to help debug how come something is not a leak
            HeapProfileTable::AllocInfo alloc;
            if (!heap_profile->FindAllocDetails(ptr, &alloc)) {
              RAW_LOG(FATAL, "FindAllocDetails failed on ptr %p", ptr);
            }
            RAW_LOG(INFO, "New live %p object's alloc stack:", ptr);
            for (int i = 0; i < alloc.stack_depth; ++i) {
              RAW_LOG(INFO, "  @ %p", alloc.call_stack[i]);
            }
          }
          live_object_count += 1;
          live_byte_count += object_size;
          live_objects->push_back(AllocObject(ptr, object_size,
                                              IGNORED_ON_HEAP));
        }
      }
      object += pointer_source_alignment;
    }
  }
  live_objects_total += live_object_count;
  live_bytes_total += live_byte_count;
  if (live_object_count) {
    RAW_VLOG(10, "Removed %" PRId64 " live heap objects of %" PRId64 " bytes: %s%s",
                live_object_count, live_byte_count, name, name2);
  }
}

//----------------------------------------------------------------------
// HeapLeakChecker leak check disabling components
//----------------------------------------------------------------------

// static
void HeapLeakChecker::DisableChecksIn(const char* pattern) {
  RAW_LOG(WARNING, "DisableChecksIn(%s) is ignored", pattern);
}

// static
void HeapLeakChecker::DoIgnoreObject(const void* ptr) {
  SpinLockHolder l(&heap_checker_lock);
  if (!heap_checker_on) return;
  size_t object_size;
  if (!HaveOnHeapLocked(&ptr, &object_size)) {
    RAW_LOG(ERROR, "No live heap object at %p to ignore", ptr);
  } else {
    RAW_VLOG(10, "Going to ignore live object at %p of %zu bytes",
                ptr, object_size);
    if (ignored_objects == NULL)  {
      ignored_objects = new(Allocator::Allocate(sizeof(IgnoredObjectsMap)))
                          IgnoredObjectsMap;
    }
    if (!ignored_objects->insert(make_pair(AsInt(ptr), object_size)).second) {
      RAW_LOG(WARNING, "Object at %p is already being ignored", ptr);
    }
  }
}

// static
void HeapLeakChecker::UnIgnoreObject(const void* ptr) {
  SpinLockHolder l(&heap_checker_lock);
  if (!heap_checker_on) return;
  size_t object_size;
  if (!HaveOnHeapLocked(&ptr, &object_size)) {
    RAW_LOG(FATAL, "No live heap object at %p to un-ignore", ptr);
  } else {
    bool found = false;
    if (ignored_objects) {
      IgnoredObjectsMap::iterator object = ignored_objects->find(AsInt(ptr));
      if (object != ignored_objects->end()  &&  object_size == object->second) {
        ignored_objects->erase(object);
        found = true;
        RAW_VLOG(10, "Now not going to ignore live object "
                    "at %p of %zu bytes", ptr, object_size);
      }
    }
    if (!found)  RAW_LOG(FATAL, "Object at %p has not been ignored", ptr);
  }
}

//----------------------------------------------------------------------
// HeapLeakChecker non-static functions
//----------------------------------------------------------------------

char* HeapLeakChecker::MakeProfileNameLocked() {
  RAW_DCHECK(lock_->IsHeld(), "");
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  const int len = profile_name_prefix->size() + strlen(name_) + 5 +
                  strlen(HeapProfileTable::kFileExt) + 1;
  char* file_name = reinterpret_cast<char*>(Allocator::Allocate(len));
  snprintf(file_name, len, "%s.%s-end%s",
           profile_name_prefix->c_str(), name_,
           HeapProfileTable::kFileExt);
  return file_name;
}

void HeapLeakChecker::Create(const char *name, bool make_start_snapshot) {
  SpinLockHolder l(lock_);
  name_ = NULL;  // checker is inactive
  start_snapshot_ = NULL;
  has_checked_ = false;
  inuse_bytes_increase_ = 0;
  inuse_allocs_increase_ = 0;
  keep_profiles_ = false;
  char* n = new char[strlen(name) + 1];   // do this before we lock
  IgnoreObject(n);  // otherwise it might be treated as live due to our stack
  { // Heap activity in other threads is paused for this whole scope.
    SpinLockHolder al(&alignment_checker_lock);
    SpinLockHolder hl(&heap_checker_lock);
    MemoryRegionMap::LockHolder ml;
    if (heap_checker_on  &&  profile_name_prefix != NULL) {
      RAW_DCHECK(strchr(name, '/') == NULL, "must be a simple name");
      memcpy(n, name, strlen(name) + 1);
      name_ = n;  // checker is active
      if (make_start_snapshot) {
        start_snapshot_ = heap_profile->TakeSnapshot();
      }

      const HeapProfileTable::Stats& t = heap_profile->total();
      const int64_t start_inuse_bytes = t.alloc_size - t.free_size;
      const int64_t start_inuse_allocs = t.allocs - t.frees;
      RAW_VLOG(10, "Start check \"%s\" profile: %" PRId64 " bytes "
               "in %" PRId64 " objects",
               name_, start_inuse_bytes, start_inuse_allocs);
    } else {
      RAW_LOG(WARNING, "Heap checker is not active, "
                       "hence checker \"%s\" will do nothing!", name);
    RAW_LOG(WARNING, "To activate set the HEAPCHECK environment variable.\n");
    }
  }
  if (name_ == NULL) {
    UnIgnoreObject(n);
    delete[] n;  // must be done after we unlock
  }
}

HeapLeakChecker::HeapLeakChecker(const char *name) : lock_(new SpinLock) {
  RAW_DCHECK(strcmp(name, "_main_") != 0, "_main_ is reserved");
  Create(name, true/*create start_snapshot_*/);
}

HeapLeakChecker::HeapLeakChecker() : lock_(new SpinLock) {
  if (FLAGS_heap_check_before_constructors) {
    // We want to check for leaks of objects allocated during global
    // constructors (i.e., objects allocated already).  So we do not
    // create a baseline snapshot and hence check for leaks of objects
    // that may have already been created.
    Create("_main_", false);
  } else {
    // We want to ignore leaks of objects allocated during global
    // constructors (i.e., objects allocated already).  So we snapshot
    // the current heap contents and use them as a baseline that is
    // not reported by the leak checker.
    Create("_main_", true);
  }
}

ssize_t HeapLeakChecker::BytesLeaked() const {
  SpinLockHolder l(lock_);
  if (!has_checked_) {
    RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call");
  }
  return inuse_bytes_increase_;
}

ssize_t HeapLeakChecker::ObjectsLeaked() const {
  SpinLockHolder l(lock_);
  if (!has_checked_) {
    RAW_LOG(FATAL, "*NoLeaks|SameHeap must execute before this call");
  }
  return inuse_allocs_increase_;
}

// Save pid of main thread for using in naming dump files
static int32_t main_thread_pid = getpid();
#ifdef HAVE_PROGRAM_INVOCATION_NAME
#ifdef __UCLIBC__
extern const char* program_invocation_name;
extern const char* program_invocation_short_name;
#else
extern char* program_invocation_name;
extern char* program_invocation_short_name;
#endif
static const char* invocation_name() { return program_invocation_short_name; }
static string invocation_path() { return program_invocation_name; }
#else
static const char* invocation_name() { return "<your binary>"; }
static string invocation_path() { return "<your binary>"; }
#endif

// Prints commands that users can run to get more information
// about the reported leaks.
static void SuggestPprofCommand(const char* pprof_file_arg) {
  // Extra help information to print for the user when the test is
  // being run in a way where the straightforward pprof command will
  // not suffice.
  string extra_help;

  // Common header info to print for remote runs
  const string remote_header =
      "This program is being executed remotely and therefore the pprof\n"
      "command printed above will not work.  Either run this program\n"
      "locally, or adjust the pprof command as follows to allow it to\n"
      "work on your local machine:\n";

  // Extra command for fetching remote data
  string fetch_cmd;

  RAW_LOG(WARNING,
          "\n\n"
          "If the preceding stack traces are not enough to find "
          "the leaks, try running THIS shell command:\n\n"
          "%s%s %s \"%s\" --inuse_objects --lines --heapcheck "
          " --edgefraction=1e-10 --nodefraction=1e-10 --gv\n"
          "\n"
          "%s"
          "If you are still puzzled about why the leaks are "
          "there, try rerunning this program with "
          "HEAP_CHECK_TEST_POINTER_ALIGNMENT=1 and/or with "
          "HEAP_CHECK_MAX_POINTER_OFFSET=-1\n"
          "If the leak report occurs in a small fraction of runs, "
          "try running with TCMALLOC_MAX_FREE_QUEUE_SIZE of few hundred MB "
          "or with TCMALLOC_RECLAIM_MEMORY=false, "  // only works for debugalloc
          "it might help find leaks more repeatably\n",
          fetch_cmd.c_str(),
          "pprof",           // works as long as pprof is on your path
          invocation_path().c_str(),
          pprof_file_arg,
          extra_help.c_str()
          );
}

bool HeapLeakChecker::DoNoLeaks(ShouldSymbolize should_symbolize) {
  SpinLockHolder l(lock_);
  // The locking also helps us keep the messages
  // for the two checks close together.
  SpinLockHolder al(&alignment_checker_lock);

  // thread-safe: protected by alignment_checker_lock
  static bool have_disabled_hooks_for_symbolize = false;
  // Once we've checked for leaks and symbolized the results once, it's
  // not safe to do it again.  This is because in order to symbolize
  // safely, we had to disable all the malloc hooks here, so we no
  // longer can be confident we've collected all the data we need.
  if (have_disabled_hooks_for_symbolize) {
    RAW_LOG(FATAL, "Must not call heap leak checker manually after "
            " program-exit's automatic check.");
  }

  HeapProfileTable::Snapshot* leaks = NULL;
  char* pprof_file = NULL;

  {
    // Heap activity in other threads is paused during this function
    // (i.e. until we got all profile difference info).
    SpinLockHolder hl(&heap_checker_lock);
    if (heap_checker_on == false) {
      if (name_ != NULL) {  // leak checking enabled when created the checker
        RAW_LOG(WARNING, "Heap leak checker got turned off after checker "
                "\"%s\" has been created, no leak check is being done for it!",
                name_);
      }
      return true;
    }

    // Update global_region_caller_ranges. They may need to change since
    // e.g. initialization because shared libraries might have been loaded or
    // unloaded.
    Allocator::DeleteAndNullIfNot(&global_region_caller_ranges);
    ProcMapsResult pm_result = UseProcMapsLocked(DISABLE_LIBRARY_ALLOCS);
    RAW_CHECK(pm_result == PROC_MAPS_USED, "");

    // Keep track of number of internally allocated objects so we
    // can detect leaks in the heap-leak-checket itself
    const int initial_allocs = Allocator::alloc_count();

    if (name_ == NULL) {
      RAW_LOG(FATAL, "Heap leak checker must not be turned on "
              "after construction of a HeapLeakChecker");
    }

    MemoryRegionMap::LockHolder ml;
    int a_local_var;  // Use our stack ptr to make stack data live:

    // Make the heap profile, other threads are locked out.
    HeapProfileTable::Snapshot* base =
        reinterpret_cast<HeapProfileTable::Snapshot*>(start_snapshot_);
    RAW_DCHECK(FLAGS_heap_check_pointer_source_alignment > 0, "");
    pointer_source_alignment = FLAGS_heap_check_pointer_source_alignment;
    IgnoreAllLiveObjectsLocked(&a_local_var);
    leaks = heap_profile->NonLiveSnapshot(base);

    inuse_bytes_increase_ = static_cast<ssize_t>(leaks->total().alloc_size);
    inuse_allocs_increase_ = static_cast<ssize_t>(leaks->total().allocs);
    if (leaks->Empty()) {
      heap_profile->ReleaseSnapshot(leaks);
      leaks = NULL;

      // We can only check for internal leaks along the no-user-leak
      // path since in the leak path we temporarily release
      // heap_checker_lock and another thread can come in and disturb
      // allocation counts.
      if (Allocator::alloc_count() != initial_allocs) {
        RAW_LOG(FATAL, "Internal HeapChecker leak of %d objects ; %d -> %d",
                Allocator::alloc_count() - initial_allocs,
                initial_allocs, Allocator::alloc_count());
      }
    } else if (FLAGS_heap_check_test_pointer_alignment) {
      if (pointer_source_alignment == 1) {
        RAW_LOG(WARNING, "--heap_check_test_pointer_alignment has no effect: "
                "--heap_check_pointer_source_alignment was already set to 1");
      } else {
        // Try with reduced pointer aligment
        pointer_source_alignment = 1;
        IgnoreAllLiveObjectsLocked(&a_local_var);
        HeapProfileTable::Snapshot* leaks_wo_align =
            heap_profile->NonLiveSnapshot(base);
        pointer_source_alignment = FLAGS_heap_check_pointer_source_alignment;
        if (leaks_wo_align->Empty()) {
          RAW_LOG(WARNING, "Found no leaks without pointer alignment: "
                  "something might be placing pointers at "
                  "unaligned addresses! This needs to be fixed.");
        } else {
          RAW_LOG(INFO, "Found leaks without pointer alignment as well: "
                  "unaligned pointers must not be the cause of leaks.");
          RAW_LOG(INFO, "--heap_check_test_pointer_alignment did not help "
                  "to diagnose the leaks.");
        }
        heap_profile->ReleaseSnapshot(leaks_wo_align);
      }
    }

    if (leaks != NULL) {
      pprof_file = MakeProfileNameLocked();
    }
  }

  has_checked_ = true;
  if (leaks == NULL) {
    if (FLAGS_heap_check_max_pointer_offset == -1) {
      RAW_LOG(WARNING,
              "Found no leaks without max_pointer_offset restriction: "
              "it's possible that the default value of "
              "heap_check_max_pointer_offset flag is too low. "
              "Do you use pointers with larger than that offsets "
              "pointing in the middle of heap-allocated objects?");
    }
    const HeapProfileTable::Stats& stats = heap_profile->total();
    RAW_VLOG(heap_checker_info_level,
             "No leaks found for check \"%s\" "
             "(but no 100%% guarantee that there aren't any): "
             "found %" PRId64 " reachable heap objects of %" PRId64 " bytes",
             name_,
             stats.allocs - stats.frees,
             stats.alloc_size - stats.free_size);
  } else {
    if (should_symbolize == SYMBOLIZE) {
      // To turn addresses into symbols, we need to fork, which is a
      // problem if both parent and child end up trying to call the
      // same malloc-hooks we've set up, at the same time.  To avoid
      // trouble, we turn off the hooks before symbolizing.  Note that
      // this makes it unsafe to ever leak-report again!  Luckily, we
      // typically only want to report once in a program's run, at the
      // very end.
      if (MallocHook::GetNewHook() == NewHook)
        MallocHook::SetNewHook(NULL);
      if (MallocHook::GetDeleteHook() == DeleteHook)
        MallocHook::SetDeleteHook(NULL);
      MemoryRegionMap::Shutdown();
      // Make sure all the hooks really got unset:
      RAW_CHECK(MallocHook::GetNewHook() == NULL, "");
      RAW_CHECK(MallocHook::GetDeleteHook() == NULL, "");
      have_disabled_hooks_for_symbolize = true;
      leaks->ReportLeaks(name_, pprof_file, true);  // true = should_symbolize
    } else {
      leaks->ReportLeaks(name_, pprof_file, false);
    }
    if (FLAGS_heap_check_identify_leaks) {
      leaks->ReportIndividualObjects();
    }

    SuggestPprofCommand(pprof_file);

    {
      SpinLockHolder hl(&heap_checker_lock);
      heap_profile->ReleaseSnapshot(leaks);
      Allocator::Free(pprof_file);
    }
  }

  return (leaks == NULL);
}

HeapLeakChecker::~HeapLeakChecker() {
  if (name_ != NULL) {  // had leak checking enabled when created the checker
    if (!has_checked_) {
      RAW_LOG(FATAL, "Some *NoLeaks|SameHeap method"
                     " must be called on any created HeapLeakChecker");
    }

    // Deallocate any snapshot taken at start
    if (start_snapshot_ != NULL) {
      SpinLockHolder l(&heap_checker_lock);
      heap_profile->ReleaseSnapshot(
          reinterpret_cast<HeapProfileTable::Snapshot*>(start_snapshot_));
    }

    UnIgnoreObject(name_);
    delete[] name_;
    name_ = NULL;
  }
  delete lock_;
}

//----------------------------------------------------------------------
// HeapLeakChecker overall heap check components
//----------------------------------------------------------------------

// static
bool HeapLeakChecker::IsActive() {
  SpinLockHolder l(&heap_checker_lock);
  return heap_checker_on;
}

vector<HeapCleaner::void_function>* HeapCleaner::heap_cleanups_ = NULL;

// When a HeapCleaner object is intialized, add its function to the static list
// of cleaners to be run before leaks checking.
HeapCleaner::HeapCleaner(void_function f) {
  if (heap_cleanups_ == NULL)
    heap_cleanups_ = new vector<HeapCleaner::void_function>;
  heap_cleanups_->push_back(f);
}

// Run all of the cleanup functions and delete the vector.
void HeapCleaner::RunHeapCleanups() {
  if (!heap_cleanups_)
    return;
  for (int i = 0; i < heap_cleanups_->size(); i++) {
    void (*f)(void) = (*heap_cleanups_)[i];
    f();
  }
  delete heap_cleanups_;
  heap_cleanups_ = NULL;
}

// Program exit heap cleanup registered as a module object destructor.
// Will not get executed when we crash on a signal.
//
void HeapLeakChecker_RunHeapCleanups() {
  if (FLAGS_heap_check == "local")   // don't check heap in this mode
    return;
  { SpinLockHolder l(&heap_checker_lock);
    // can get here (via forks?) with other pids
    if (heap_checker_pid != getpid()) return;
  }
  HeapCleaner::RunHeapCleanups();
  if (!FLAGS_heap_check_after_destructors) HeapLeakChecker::DoMainHeapCheck();
}

static bool internal_init_start_has_run = false;

// Called exactly once, before main() (but hopefully just before).
// This picks a good unique name for the dumped leak checking heap profiles.
//
// Because we crash when InternalInitStart is called more than once,
// it's fine that we hold heap_checker_lock only around pieces of
// this function: this is still enough for thread-safety w.r.t. other functions
// of this module.
// We can't hold heap_checker_lock throughout because it would deadlock
// on a memory allocation since our new/delete hooks can be on.
//
void HeapLeakChecker_InternalInitStart() {
  { SpinLockHolder l(&heap_checker_lock);
    RAW_CHECK(!internal_init_start_has_run,
              "Heap-check constructor called twice.  Perhaps you both linked"
              " in the heap checker, and also used LD_PRELOAD to load it?");
    internal_init_start_has_run = true;

#ifdef ADDRESS_SANITIZER
    // AddressSanitizer's custom malloc conflicts with HeapChecker.
    FLAGS_heap_check = "";
#endif

    if (FLAGS_heap_check.empty()) {
      // turns out we do not need checking in the end; can stop profiling
      HeapLeakChecker::TurnItselfOffLocked();
      return;
    } else if (RunningOnValgrind()) {
      // There is no point in trying -- we'll just fail.
      RAW_LOG(WARNING, "Can't run under Valgrind; will turn itself off");
      HeapLeakChecker::TurnItselfOffLocked();
      return;
    }
  }

  // Changing this to false can be useful when debugging heap-checker itself:
  if (!FLAGS_heap_check_run_under_gdb && IsDebuggerAttached()) {
    RAW_LOG(WARNING, "Someone is ptrace()ing us; will turn itself off");
    SpinLockHolder l(&heap_checker_lock);
    HeapLeakChecker::TurnItselfOffLocked();
    return;
  }

  { SpinLockHolder l(&heap_checker_lock);
    if (!constructor_heap_profiling) {
      RAW_LOG(FATAL, "Can not start so late. You have to enable heap checking "
	             "with HEAPCHECK=<mode>.");
    }
  }

  // Set all flags
  RAW_DCHECK(FLAGS_heap_check_pointer_source_alignment > 0, "");
  if (FLAGS_heap_check == "minimal") {
    // The least we can check.
    FLAGS_heap_check_before_constructors = false;  // from after main
                                                   // (ignore more)
    FLAGS_heap_check_after_destructors = false;  // to after cleanup
                                                 // (most data is live)
    FLAGS_heap_check_ignore_thread_live = true;  // ignore all live
    FLAGS_heap_check_ignore_global_live = true;  // ignore all live
  } else if (FLAGS_heap_check == "normal") {
    // Faster than 'minimal' and not much stricter.
    FLAGS_heap_check_before_constructors = true;  // from no profile (fast)
    FLAGS_heap_check_after_destructors = false;  // to after cleanup
                                                 // (most data is live)
    FLAGS_heap_check_ignore_thread_live = true;  // ignore all live
    FLAGS_heap_check_ignore_global_live = true;  // ignore all live
  } else if (FLAGS_heap_check == "strict") {
    // A bit stricter than 'normal': global destructors must fully clean up
    // after themselves if they are present.
    FLAGS_heap_check_before_constructors = true;  // from no profile (fast)
    FLAGS_heap_check_after_destructors = true;  // to after destructors
                                                // (less data live)
    FLAGS_heap_check_ignore_thread_live = true;  // ignore all live
    FLAGS_heap_check_ignore_global_live = true;  // ignore all live
  } else if (FLAGS_heap_check == "draconian") {
    // Drop not very portable and not very exact live heap flooding.
    FLAGS_heap_check_before_constructors = true;  // from no profile (fast)
    FLAGS_heap_check_after_destructors = true;  // to after destructors
                                                // (need them)
    FLAGS_heap_check_ignore_thread_live = false;  // no live flood (stricter)
    FLAGS_heap_check_ignore_global_live = false;  // no live flood (stricter)
  } else if (FLAGS_heap_check == "as-is") {
    // do nothing: use other flags as is
  } else if (FLAGS_heap_check == "local") {
    // do nothing
  } else {
    RAW_LOG(FATAL, "Unsupported heap_check flag: %s",
                   FLAGS_heap_check.c_str());
  }
  // FreeBSD doesn't seem to honor atexit execution order:
  //    https://github.com/gperftools/gperftools/issues/378
  // Since heap-checking before destructors depends on atexit running
  // at the right time, on FreeBSD we always check after, even in the
  // less strict modes.  This just means FreeBSD is always a bit
  // stricter in its checking than other OSes.
  // This now appears to be the case in other OSes as well;
  // so always check afterwards.
  FLAGS_heap_check_after_destructors = true;

  { SpinLockHolder l(&heap_checker_lock);
    RAW_DCHECK(heap_checker_pid == getpid(), "");
    heap_checker_on = true;
    RAW_DCHECK(heap_profile, "");
    HeapLeakChecker::ProcMapsResult pm_result = HeapLeakChecker::UseProcMapsLocked(HeapLeakChecker::DISABLE_LIBRARY_ALLOCS);
      // might neeed to do this more than once
      // if one later dynamically loads libraries that we want disabled
    if (pm_result != HeapLeakChecker::PROC_MAPS_USED) {  // can't function
      HeapLeakChecker::TurnItselfOffLocked();
      return;
    }
  }

  // make a good place and name for heap profile leak dumps
  string* profile_prefix =
    new string(FLAGS_heap_check_dump_directory + "/" + invocation_name());

  // Finalize prefix for dumping leak checking profiles.
  const int32_t our_pid = getpid();   // safest to call getpid() outside lock
  { SpinLockHolder l(&heap_checker_lock);
    // main_thread_pid might still be 0 if this function is being called before
    // global constructors.  In that case, our pid *is* the main pid.
    if (main_thread_pid == 0)
      main_thread_pid = our_pid;
  }
  char pid_buf[15];
  snprintf(pid_buf, sizeof(pid_buf), ".%d", main_thread_pid);
  *profile_prefix += pid_buf;
  { SpinLockHolder l(&heap_checker_lock);
    RAW_DCHECK(profile_name_prefix == NULL, "");
    profile_name_prefix = profile_prefix;
  }

  // Make sure new/delete hooks are installed properly
  // and heap profiler is indeed able to keep track
  // of the objects being allocated.
  // We test this to make sure we are indeed checking for leaks.
  char* test_str = new (tc_newarray(5)) char[5];
  size_t size;
  { SpinLockHolder l(&heap_checker_lock);
    RAW_CHECK(heap_profile->FindAlloc(test_str, &size),
              "our own new/delete not linked?");
  }

  tc_deletearray(test_str);

  { SpinLockHolder l(&heap_checker_lock);
    // This check can fail when it should not if another thread allocates
    // into this same spot right this moment,
    // which is unlikely since this code runs in InitGoogle.
    RAW_CHECK(!heap_profile->FindAlloc(test_str, &size),
              "our own new/delete not linked?");
  }
  // If we crash in the above code, it probably means that
  // "nm <this_binary> | grep new" will show that tcmalloc's new/delete
  // implementation did not get linked-in into this binary
  // (i.e. nm will list __builtin_new and __builtin_vec_new as undefined).
  // If this happens, it is a BUILD bug to be fixed.

  RAW_VLOG(heap_checker_info_level,
           "WARNING: Perftools heap leak checker is active "
           "-- Performance may suffer");

  if (FLAGS_heap_check != "local") {
    HeapLeakChecker* main_hc = new HeapLeakChecker();
    SpinLockHolder l(&heap_checker_lock);
    RAW_DCHECK(main_heap_checker == NULL,
               "Repeated creation of main_heap_checker");
    main_heap_checker = main_hc;
    do_main_heap_check = true;
  }

  { SpinLockHolder l(&heap_checker_lock);
    RAW_CHECK(heap_checker_on  &&  constructor_heap_profiling,
              "Leak checking is expected to be fully turned on now");
  }

  // For binaries built in debug mode, this will set release queue of
  // debugallocation.cc to 100M to make it less likely for real leaks to
  // be hidden due to reuse of heap memory object addresses.
  // Running a test with --malloc_reclaim_memory=0 would help find leaks even
  // better, but the test might run out of memory as a result.
  // The scenario is that a heap object at address X is allocated and freed,
  // but some other data-structure still retains a pointer to X.
  // Then the same heap memory is used for another object, which is leaked,
  // but the leak is not noticed due to the pointer to the original object at X.
  // TODO(csilvers): support this in some manner.
#if 0
  SetCommandLineOptionWithMode("max_free_queue_size", "104857600",  // 100M
                               SET_FLAG_IF_DEFAULT);
#endif
}

// We want this to run early as well, but not so early as
// ::BeforeConstructors (we want flag assignments to have already
// happened, for instance).  Initializer-registration does the trick.
REGISTER_MODULE_INITIALIZER(init_start, HeapLeakChecker_InternalInitStart());
REGISTER_MODULE_DESTRUCTOR(init_start, HeapLeakChecker_RunHeapCleanups());

// static
bool HeapLeakChecker::NoGlobalLeaksMaybeSymbolize(
    ShouldSymbolize should_symbolize) {
  // we never delete or change main_heap_checker once it's set:
  HeapLeakChecker* main_hc = GlobalChecker();
  if (main_hc) {
    RAW_VLOG(10, "Checking for whole-program memory leaks");
    return main_hc->DoNoLeaks(should_symbolize);
  }
  return true;
}

// static
bool HeapLeakChecker::DoMainHeapCheck() {
  if (FLAGS_heap_check_delay_seconds > 0) {
    sleep(FLAGS_heap_check_delay_seconds);
  }
  { SpinLockHolder l(&heap_checker_lock);
    if (!do_main_heap_check) return false;
    RAW_DCHECK(heap_checker_pid == getpid(), "");
    do_main_heap_check = false;  // will do it now; no need to do it more
  }

  // The program is over, so it's safe to symbolize addresses (which
  // requires a fork) because no serious work is expected to be done
  // after this.  Symbolizing is really useful -- knowing what
  // function has a leak is better than knowing just an address --
  // and while we can only safely symbolize once in a program run,
  // now is the time (after all, there's no "later" that would be better).
  if (!NoGlobalLeaksMaybeSymbolize(SYMBOLIZE)) {
    if (FLAGS_heap_check_identify_leaks) {
      RAW_LOG(FATAL, "Whole-program memory leaks found.");
    }
    RAW_LOG(ERROR, "Exiting with error code (instead of crashing) "
                   "because of whole-program memory leaks");
    _exit(1);    // we don't want to call atexit() routines!
  }
  return true;
}

// static
HeapLeakChecker* HeapLeakChecker::GlobalChecker() {
  SpinLockHolder l(&heap_checker_lock);
  return main_heap_checker;
}

// static
bool HeapLeakChecker::NoGlobalLeaks() {
  // symbolizing requires a fork, which isn't safe to do in general.
  return NoGlobalLeaksMaybeSymbolize(DO_NOT_SYMBOLIZE);
}

// static
void HeapLeakChecker::CancelGlobalCheck() {
  SpinLockHolder l(&heap_checker_lock);
  if (do_main_heap_check) {
    RAW_VLOG(heap_checker_info_level,
             "Canceling the automatic at-exit whole-program memory leak check");
    do_main_heap_check = false;
  }
}

// static
void HeapLeakChecker::BeforeConstructorsLocked() {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_CHECK(!constructor_heap_profiling,
            "BeforeConstructorsLocked called multiple times");
#ifdef ADDRESS_SANITIZER
  // AddressSanitizer's custom malloc conflicts with HeapChecker.
  return;
#endif
  // Set hooks early to crash if 'new' gets called before we make heap_profile,
  // and make sure no other hooks existed:
  RAW_CHECK(MallocHook::AddNewHook(&NewHook), "");
  RAW_CHECK(MallocHook::AddDeleteHook(&DeleteHook), "");
  constructor_heap_profiling = true;
  MemoryRegionMap::Init(1, /* use_buckets */ false);
    // Set up MemoryRegionMap with (at least) one caller stack frame to record
    // (important that it's done before HeapProfileTable creation below).
  Allocator::Init();
  RAW_CHECK(heap_profile == NULL, "");
  heap_profile = new(Allocator::Allocate(sizeof(HeapProfileTable)))
      HeapProfileTable(&Allocator::Allocate, &Allocator::Free,
                       /* profile_mmap */ false);
  RAW_VLOG(10, "Starting tracking the heap");
  heap_checker_on = true;
}

// static
void HeapLeakChecker::TurnItselfOffLocked() {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  // Set FLAGS_heap_check to "", for users who test for it
  if (!FLAGS_heap_check.empty())  // be a noop in the common case
    FLAGS_heap_check.clear();     // because clear() could allocate memory
  if (constructor_heap_profiling) {
    RAW_CHECK(heap_checker_on, "");
    RAW_VLOG(heap_checker_info_level, "Turning perftools heap leak checking off");
    heap_checker_on = false;
    // Unset our hooks checking they were set:
    RAW_CHECK(MallocHook::RemoveNewHook(&NewHook), "");
    RAW_CHECK(MallocHook::RemoveDeleteHook(&DeleteHook), "");
    Allocator::DeleteAndNull(&heap_profile);
    // free our optional global data:
    Allocator::DeleteAndNullIfNot(&ignored_objects);
    Allocator::DeleteAndNullIfNot(&disabled_ranges);
    Allocator::DeleteAndNullIfNot(&global_region_caller_ranges);
    Allocator::Shutdown();
    MemoryRegionMap::Shutdown();
  }
  RAW_CHECK(!heap_checker_on, "");
}

static bool has_called_before_constructors = false;

// TODO(maxim): inline this function with
// MallocHook_InitAtFirstAllocation_HeapLeakChecker, and also rename
// HeapLeakChecker::BeforeConstructorsLocked.
void HeapLeakChecker_BeforeConstructors() {
  SpinLockHolder l(&heap_checker_lock);
  // We can be called from several places: the first mmap/sbrk/alloc call
  // or the first global c-tor from heap-checker-bcad.cc:
  // Do not re-execute initialization:
  if (has_called_before_constructors) return;
  has_called_before_constructors = true;

  heap_checker_pid = getpid();  // set it always
  // just to reference it, so that heap-checker-bcad.o is linked in

  // This function can be called *very* early, before the normal
  // global-constructor that sets FLAGS_verbose.  Set it manually now,
  // so the RAW_LOG messages here are controllable.
  const char* verbose_str = GetenvBeforeMain("PERFTOOLS_VERBOSE");
  if (verbose_str && atoi(verbose_str)) {  // different than the default of 0?
    FLAGS_verbose = atoi(verbose_str);
  }

  bool need_heap_check = true;
  // The user indicates a desire for heap-checking via the HEAPCHECK
  // environment variable.  If it's not set, there's no way to do
  // heap-checking.
  if (!GetenvBeforeMain("HEAPCHECK")) {
    need_heap_check = false;
  }
#ifdef HAVE_GETEUID
  if (need_heap_check && getuid() != geteuid()) {
    // heap-checker writes out files.  Thus, for security reasons, we don't
    // recognize the env. var. to turn on heap-checking if we're setuid.
    RAW_LOG(WARNING, ("HeapChecker: ignoring HEAPCHECK because "
                      "program seems to be setuid\n"));
    need_heap_check = false;
  }
#endif
  if (need_heap_check) {
    HeapLeakChecker::BeforeConstructorsLocked();
  }
}

// This function overrides the weak function defined in malloc_hook.cc and
// called by one of the initial malloc hooks (malloc_hook.cc) when the very
// first memory allocation or an mmap/sbrk happens.  This ensures that
// HeapLeakChecker is initialized and installs all its hooks early enough to
// track absolutely all memory allocations and all memory region acquisitions
// via mmap and sbrk.
extern "C" int MallocHook_InitAtFirstAllocation_HeapLeakChecker() {
  static tcmalloc::TrivialOnce once;

  return once.RunOnce(&HeapLeakChecker_BeforeConstructors);
}

// This function is executed after all global object destructors run.
//
// Note, heap checker is Linux-only. Which implies we're dealing with
// gcc-compatible compiler, so we can do __attribute__ thingy. In
// practice destructor functions end up being called after c++
// destructors (those are set up via __cxa_atexit thingy), so we get
// the right ordering.
__attribute__((destructor))
void HeapLeakChecker_AfterDestructors() {
  { SpinLockHolder l(&heap_checker_lock);
    // can get here (via forks?) with other pids
    if (heap_checker_pid != getpid()) return;
  }
  if (FLAGS_heap_check_after_destructors) {
    if (HeapLeakChecker::DoMainHeapCheck()) {
      const struct timespec sleep_time = { 0, 500000000 };  // 500 ms
      nanosleep(&sleep_time, NULL);
        // Need this hack to wait for other pthreads to exit.
        // Otherwise tcmalloc find errors
        // on a free() call from pthreads.
    }
  }
  SpinLockHolder l(&heap_checker_lock);
  RAW_CHECK(!do_main_heap_check, "should have done it");
}

//----------------------------------------------------------------------
// HeapLeakChecker disabling helpers
//----------------------------------------------------------------------

// These functions are at the end of the file to prevent their inlining:

// static
void HeapLeakChecker::DisableChecksFromToLocked(const void* start_address,
                                                const void* end_address,
                                                int max_depth) {
  RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  RAW_DCHECK(start_address < end_address, "");
  if (disabled_ranges == NULL) {
    disabled_ranges = new(Allocator::Allocate(sizeof(DisabledRangeMap)))
                        DisabledRangeMap;
  }
  RangeValue value;
  value.start_address = AsInt(start_address);
  value.max_depth = max_depth;
  if (disabled_ranges->insert(make_pair(AsInt(end_address), value)).second) {
    RAW_VLOG(10, "Disabling leak checking in stack traces "
                "under frame addresses between %p..%p",
                start_address, end_address);
  } else {  // check that this is just a verbatim repetition
    RangeValue const& val = disabled_ranges->find(AsInt(end_address))->second;
    if (val.max_depth != value.max_depth  ||
        val.start_address != value.start_address) {
      RAW_LOG(FATAL, "Two DisableChecksToHereFrom calls conflict: "
                     "(%p, %p, %d) vs. (%p, %p, %d)",
                     AsPtr(val.start_address), end_address, val.max_depth,
                     start_address, end_address, max_depth);
    }
  }
}

// static
inline bool HeapLeakChecker::HaveOnHeapLocked(const void** ptr,
                                              size_t* object_size) {
  // Commented-out because HaveOnHeapLocked is very performance-critical:
  // RAW_DCHECK(heap_checker_lock.IsHeld(), "");
  const uintptr_t addr = AsInt(*ptr);
  if (heap_profile->FindInsideAlloc(
        *ptr, max_heap_object_size, ptr, object_size)) {
    RAW_VLOG(16, "Got pointer into %p at +%" PRIuPTR " offset",
             *ptr, addr - AsInt(*ptr));
    return true;
  }
  return false;
}

// static
const void* HeapLeakChecker::GetAllocCaller(void* ptr) {
  // this is used only in the unittest, so the heavy checks are fine
  HeapProfileTable::AllocInfo info;
  { SpinLockHolder l(&heap_checker_lock);
    RAW_CHECK(heap_profile->FindAllocDetails(ptr, &info), "");
  }
  RAW_CHECK(info.stack_depth >= 1, "");
  return info.call_stack[0];
}

namespace tcmalloc {

ATTRIBUTE_HIDDEN
void DoIterateMemoryRegionMap(tcmalloc::FunctionRef<void(const void*)> callback) {
  { MemoryRegionMap::LockHolder l;
    for (MemoryRegionMap::RegionIterator
           i = MemoryRegionMap::BeginRegionLocked();
           i != MemoryRegionMap::EndRegionLocked(); ++i) {
      callback(&*i);
    }
  }
}

}  // namespace tcmalloc