This source file includes following definitions.
- InitialNewHook
- InitialPreMMapHook
- InitialPreSbrkHook
- RemoveInitialHooksAndCallInitializers
- MallocHook_InitAtFirstAllocation_HeapLeakChecker
- Exchange
- CompareAndSwap
- Add
- Remove
- Traverse
- MallocHook_AddNewHook
- MallocHook_RemoveNewHook
- MallocHook_AddDeleteHook
- MallocHook_RemoveDeleteHook
- MallocHook_AddPreMmapHook
- MallocHook_RemovePreMmapHook
- MallocHook_SetMmapReplacement
- MallocHook_RemoveMmapReplacement
- MallocHook_AddMmapHook
- MallocHook_RemoveMmapHook
- MallocHook_AddMunmapHook
- MallocHook_RemoveMunmapHook
- MallocHook_SetMunmapReplacement
- MallocHook_RemoveMunmapReplacement
- MallocHook_AddMremapHook
- MallocHook_RemoveMremapHook
- MallocHook_AddPreSbrkHook
- MallocHook_RemovePreSbrkHook
- MallocHook_AddSbrkHook
- MallocHook_RemoveSbrkHook
- MallocHook_SetNewHook
- MallocHook_SetDeleteHook
- MallocHook_SetPreMmapHook
- MallocHook_SetMmapHook
- MallocHook_SetMunmapHook
- MallocHook_SetMremapHook
- MallocHook_SetPreSbrkHook
- MallocHook_SetSbrkHook
- InvokeNewHookSlow
- InvokeDeleteHookSlow
- InvokePreMmapHookSlow
- InvokeMmapHookSlow
- InvokeMmapReplacementSlow
- InvokeMunmapHookSlow
- InvokeMunmapReplacementSlow
- InvokeMremapHookSlow
- InvokePreSbrkHookSlow
- InvokeSbrkHookSlow
- CheckInHookCaller
- MallocHook_GetCallerStackTrace
- UnhookedMMap
- UnhookedMUnmap
#include <config.h>
#ifdef HAVE_MMAP
# define mremap glibc_mremap
# include <sys/mman.h>
# undef mremap
#endif
#include <stddef.h>
#ifdef HAVE_STDINT_H
#include <stdint.h>
#endif
#include <algorithm>
#include "base/logging.h"
#include "base/spinlock.h"
#include "maybe_threads.h"
#include "malloc_hook-inl.h"
#include <gperftools/malloc_hook.h>
#ifdef NO_TCMALLOC_SAMPLES
# define GetStackTrace(stack, depth, skip) (0)
#else
# include <gperftools/stacktrace.h>
#endif
#ifndef __THROW
# define __THROW
#endif
using std::copy;
extern "C" {
ATTRIBUTE_WEAK void MallocHook_InitAtFirstAllocation_HeapLeakChecker();
}
namespace {
void RemoveInitialHooksAndCallInitializers();
pthread_once_t once = PTHREAD_ONCE_INIT;
void InitialNewHook(const void* ptr, size_t size) {
perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers);
MallocHook::InvokeNewHook(ptr, size);
}
void InitialPreMMapHook(const void* start,
size_t size,
int protection,
int flags,
int fd,
off_t offset) {
perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers);
MallocHook::InvokePreMmapHook(start, size, protection, flags, fd, offset);
}
void InitialPreSbrkHook(ptrdiff_t increment) {
perftools_pthread_once(&once, &RemoveInitialHooksAndCallInitializers);
MallocHook::InvokePreSbrkHook(increment);
}
void RemoveInitialHooksAndCallInitializers() {
RAW_CHECK(MallocHook::RemoveNewHook(&InitialNewHook), "");
RAW_CHECK(MallocHook::RemovePreMmapHook(&InitialPreMMapHook), "");
RAW_CHECK(MallocHook::RemovePreSbrkHook(&InitialPreSbrkHook), "");
MallocHook_InitAtFirstAllocation_HeapLeakChecker();
}
}
extern "C" void MallocHook_InitAtFirstAllocation_HeapLeakChecker() {
}
namespace base { namespace internal {
template<typename PtrT>
PtrT AtomicPtr<PtrT>::Exchange(PtrT new_val) {
base::subtle::MemoryBarrier();
PtrT old_val = reinterpret_cast<PtrT>(static_cast<AtomicWord>(
base::subtle::NoBarrier_AtomicExchange(
&data_,
reinterpret_cast<AtomicWord>(new_val))));
base::subtle::MemoryBarrier();
return old_val;
}
template<typename PtrT>
PtrT AtomicPtr<PtrT>::CompareAndSwap(PtrT old_val, PtrT new_val) {
base::subtle::MemoryBarrier();
PtrT retval = reinterpret_cast<PtrT>(static_cast<AtomicWord>(
base::subtle::NoBarrier_CompareAndSwap(
&data_,
reinterpret_cast<AtomicWord>(old_val),
reinterpret_cast<AtomicWord>(new_val))));
base::subtle::MemoryBarrier();
return retval;
}
AtomicPtr<MallocHook::NewHook> new_hook_ = { 0 };
AtomicPtr<MallocHook::DeleteHook> delete_hook_ = { 0 };
AtomicPtr<MallocHook::PreMmapHook> premmap_hook_ = { 0 };
AtomicPtr<MallocHook::MmapHook> mmap_hook_ = { 0 };
AtomicPtr<MallocHook::MunmapHook> munmap_hook_ = { 0 };
AtomicPtr<MallocHook::MremapHook> mremap_hook_ = { 0 };
AtomicPtr<MallocHook::PreSbrkHook> presbrk_hook_ = { 0 };
AtomicPtr<MallocHook::SbrkHook> sbrk_hook_ = { 0 };
static SpinLock hooklist_spinlock(base::LINKER_INITIALIZED);
template <typename T>
bool HookList<T>::Add(T value_as_t) {
AtomicWord value = bit_cast<AtomicWord>(value_as_t);
if (value == 0) {
return false;
}
SpinLockHolder l(&hooklist_spinlock);
int index = 0;
while ((index < kHookListMaxValues) &&
(base::subtle::NoBarrier_Load(&priv_data[index]) != 0)) {
++index;
}
if (index == kHookListMaxValues) {
return false;
}
AtomicWord prev_num_hooks = base::subtle::Acquire_Load(&priv_end);
base::subtle::Release_Store(&priv_data[index], value);
if (prev_num_hooks <= index) {
base::subtle::Release_Store(&priv_end, index + 1);
}
return true;
}
template <typename T>
bool HookList<T>::Remove(T value_as_t) {
if (value_as_t == 0) {
return false;
}
SpinLockHolder l(&hooklist_spinlock);
AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end);
int index = 0;
while (index < hooks_end && value_as_t != bit_cast<T>(
base::subtle::Acquire_Load(&priv_data[index]))) {
++index;
}
if (index == hooks_end) {
return false;
}
base::subtle::Release_Store(&priv_data[index], 0);
if (hooks_end == index + 1) {
hooks_end = index;
while ((hooks_end > 0) &&
(base::subtle::Acquire_Load(&priv_data[hooks_end - 1]) == 0)) {
--hooks_end;
}
base::subtle::Release_Store(&priv_end, hooks_end);
}
return true;
}
template <typename T>
int HookList<T>::Traverse(T* output_array, int n) const {
AtomicWord hooks_end = base::subtle::Acquire_Load(&priv_end);
int actual_hooks_end = 0;
for (int i = 0; i < hooks_end && n > 0; ++i) {
AtomicWord data = base::subtle::Acquire_Load(&priv_data[i]);
if (data != 0) {
*output_array++ = bit_cast<T>(data);
++actual_hooks_end;
--n;
}
}
return actual_hooks_end;
}
#define INIT_HOOK_LIST { 0 }
#define INIT_HOOK_LIST_WITH_VALUE(initial_value) \
{ 1, { reinterpret_cast<AtomicWord>(initial_value) } }
template struct HookList<MallocHook::NewHook>;
HookList<MallocHook::NewHook> new_hooks_ =
INIT_HOOK_LIST_WITH_VALUE(&InitialNewHook);
HookList<MallocHook::DeleteHook> delete_hooks_ = INIT_HOOK_LIST;
HookList<MallocHook::PreMmapHook> premmap_hooks_ =
INIT_HOOK_LIST_WITH_VALUE(&InitialPreMMapHook);
HookList<MallocHook::MmapHook> mmap_hooks_ = INIT_HOOK_LIST;
HookList<MallocHook::MunmapHook> munmap_hooks_ = INIT_HOOK_LIST;
HookList<MallocHook::MremapHook> mremap_hooks_ = INIT_HOOK_LIST;
HookList<MallocHook::PreSbrkHook> presbrk_hooks_ =
INIT_HOOK_LIST_WITH_VALUE(InitialPreSbrkHook);
HookList<MallocHook::SbrkHook> sbrk_hooks_ = INIT_HOOK_LIST;
HookList<MallocHook::MmapReplacement> mmap_replacement_ = { 0 };
HookList<MallocHook::MunmapReplacement> munmap_replacement_ = { 0 };
#undef INIT_HOOK_LIST_WITH_VALUE
#undef INIT_HOOK_LIST
} }
using base::internal::new_hook_;
using base::internal::delete_hook_;
using base::internal::premmap_hook_;
using base::internal::mmap_hook_;
using base::internal::munmap_hook_;
using base::internal::mremap_hook_;
using base::internal::presbrk_hook_;
using base::internal::sbrk_hook_;
using base::internal::kHookListMaxValues;
using base::internal::new_hooks_;
using base::internal::delete_hooks_;
using base::internal::premmap_hooks_;
using base::internal::mmap_hooks_;
using base::internal::mmap_replacement_;
using base::internal::munmap_hooks_;
using base::internal::munmap_replacement_;
using base::internal::mremap_hooks_;
using base::internal::presbrk_hooks_;
using base::internal::sbrk_hooks_;
extern "C"
int MallocHook_AddNewHook(MallocHook_NewHook hook) {
RAW_VLOG(10, "AddNewHook(%p)", hook);
return new_hooks_.Add(hook);
}
extern "C"
int MallocHook_RemoveNewHook(MallocHook_NewHook hook) {
RAW_VLOG(10, "RemoveNewHook(%p)", hook);
return new_hooks_.Remove(hook);
}
extern "C"
int MallocHook_AddDeleteHook(MallocHook_DeleteHook hook) {
RAW_VLOG(10, "AddDeleteHook(%p)", hook);
return delete_hooks_.Add(hook);
}
extern "C"
int MallocHook_RemoveDeleteHook(MallocHook_DeleteHook hook) {
RAW_VLOG(10, "RemoveDeleteHook(%p)", hook);
return delete_hooks_.Remove(hook);
}
extern "C"
int MallocHook_AddPreMmapHook(MallocHook_PreMmapHook hook) {
RAW_VLOG(10, "AddPreMmapHook(%p)", hook);
return premmap_hooks_.Add(hook);
}
extern "C"
int MallocHook_RemovePreMmapHook(MallocHook_PreMmapHook hook) {
RAW_VLOG(10, "RemovePreMmapHook(%p)", hook);
return premmap_hooks_.Remove(hook);
}
extern "C"
int MallocHook_SetMmapReplacement(MallocHook_MmapReplacement hook) {
RAW_VLOG(10, "SetMmapReplacement(%p)", hook);
RAW_CHECK(mmap_replacement_.empty(), "Only one MMapReplacement is allowed.");
return mmap_replacement_.Add(hook);
}
extern "C"
int MallocHook_RemoveMmapReplacement(MallocHook_MmapReplacement hook) {
RAW_VLOG(10, "RemoveMmapReplacement(%p)", hook);
return mmap_replacement_.Remove(hook);
}
extern "C"
int MallocHook_AddMmapHook(MallocHook_MmapHook hook) {
RAW_VLOG(10, "AddMmapHook(%p)", hook);
return mmap_hooks_.Add(hook);
}
extern "C"
int MallocHook_RemoveMmapHook(MallocHook_MmapHook hook) {
RAW_VLOG(10, "RemoveMmapHook(%p)", hook);
return mmap_hooks_.Remove(hook);
}
extern "C"
int MallocHook_AddMunmapHook(MallocHook_MunmapHook hook) {
RAW_VLOG(10, "AddMunmapHook(%p)", hook);
return munmap_hooks_.Add(hook);
}
extern "C"
int MallocHook_RemoveMunmapHook(MallocHook_MunmapHook hook) {
RAW_VLOG(10, "RemoveMunmapHook(%p)", hook);
return munmap_hooks_.Remove(hook);
}
extern "C"
int MallocHook_SetMunmapReplacement(MallocHook_MunmapReplacement hook) {
RAW_VLOG(10, "SetMunmapReplacement(%p)", hook);
RAW_CHECK(munmap_replacement_.empty(),
"Only one MunmapReplacement is allowed.");
return munmap_replacement_.Add(hook);
}
extern "C"
int MallocHook_RemoveMunmapReplacement(MallocHook_MunmapReplacement hook) {
RAW_VLOG(10, "RemoveMunmapReplacement(%p)", hook);
return munmap_replacement_.Remove(hook);
}
extern "C"
int MallocHook_AddMremapHook(MallocHook_MremapHook hook) {
RAW_VLOG(10, "AddMremapHook(%p)", hook);
return mremap_hooks_.Add(hook);
}
extern "C"
int MallocHook_RemoveMremapHook(MallocHook_MremapHook hook) {
RAW_VLOG(10, "RemoveMremapHook(%p)", hook);
return mremap_hooks_.Remove(hook);
}
extern "C"
int MallocHook_AddPreSbrkHook(MallocHook_PreSbrkHook hook) {
RAW_VLOG(10, "AddPreSbrkHook(%p)", hook);
return presbrk_hooks_.Add(hook);
}
extern "C"
int MallocHook_RemovePreSbrkHook(MallocHook_PreSbrkHook hook) {
RAW_VLOG(10, "RemovePreSbrkHook(%p)", hook);
return presbrk_hooks_.Remove(hook);
}
extern "C"
int MallocHook_AddSbrkHook(MallocHook_SbrkHook hook) {
RAW_VLOG(10, "AddSbrkHook(%p)", hook);
return sbrk_hooks_.Add(hook);
}
extern "C"
int MallocHook_RemoveSbrkHook(MallocHook_SbrkHook hook) {
RAW_VLOG(10, "RemoveSbrkHook(%p)", hook);
return sbrk_hooks_.Remove(hook);
}
extern "C"
MallocHook_NewHook MallocHook_SetNewHook(MallocHook_NewHook hook) {
RAW_VLOG(10, "SetNewHook(%p)", hook);
return new_hook_.Exchange(hook);
}
extern "C"
MallocHook_DeleteHook MallocHook_SetDeleteHook(MallocHook_DeleteHook hook) {
RAW_VLOG(10, "SetDeleteHook(%p)", hook);
return delete_hook_.Exchange(hook);
}
extern "C"
MallocHook_PreMmapHook MallocHook_SetPreMmapHook(MallocHook_PreMmapHook hook) {
RAW_VLOG(10, "SetPreMmapHook(%p)", hook);
return premmap_hook_.Exchange(hook);
}
extern "C"
MallocHook_MmapHook MallocHook_SetMmapHook(MallocHook_MmapHook hook) {
RAW_VLOG(10, "SetMmapHook(%p)", hook);
return mmap_hook_.Exchange(hook);
}
extern "C"
MallocHook_MunmapHook MallocHook_SetMunmapHook(MallocHook_MunmapHook hook) {
RAW_VLOG(10, "SetMunmapHook(%p)", hook);
return munmap_hook_.Exchange(hook);
}
extern "C"
MallocHook_MremapHook MallocHook_SetMremapHook(MallocHook_MremapHook hook) {
RAW_VLOG(10, "SetMremapHook(%p)", hook);
return mremap_hook_.Exchange(hook);
}
extern "C"
MallocHook_PreSbrkHook MallocHook_SetPreSbrkHook(MallocHook_PreSbrkHook hook) {
RAW_VLOG(10, "SetPreSbrkHook(%p)", hook);
return presbrk_hook_.Exchange(hook);
}
extern "C"
MallocHook_SbrkHook MallocHook_SetSbrkHook(MallocHook_SbrkHook hook) {
RAW_VLOG(10, "SetSbrkHook(%p)", hook);
return sbrk_hook_.Exchange(hook);
}
#define INVOKE_HOOKS(HookType, hook_list, args) do { \
HookType hooks[kHookListMaxValues]; \
int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \
for (int i = 0; i < num_hooks; ++i) { \
(*hooks[i])args; \
} \
} while (0)
#define INVOKE_REPLACEMENT(HookType, hook_list, args) do { \
HookType hooks[kHookListMaxValues]; \
int num_hooks = hook_list.Traverse(hooks, kHookListMaxValues); \
return (num_hooks > 0 && (*hooks[0])args); \
} while (0)
void MallocHook::InvokeNewHookSlow(const void* p, size_t s) {
INVOKE_HOOKS(NewHook, new_hooks_, (p, s));
}
void MallocHook::InvokeDeleteHookSlow(const void* p) {
INVOKE_HOOKS(DeleteHook, delete_hooks_, (p));
}
void MallocHook::InvokePreMmapHookSlow(const void* start,
size_t size,
int protection,
int flags,
int fd,
off_t offset) {
INVOKE_HOOKS(PreMmapHook, premmap_hooks_, (start, size, protection, flags, fd,
offset));
}
void MallocHook::InvokeMmapHookSlow(const void* result,
const void* start,
size_t size,
int protection,
int flags,
int fd,
off_t offset) {
INVOKE_HOOKS(MmapHook, mmap_hooks_, (result, start, size, protection, flags,
fd, offset));
}
bool MallocHook::InvokeMmapReplacementSlow(const void* start,
size_t size,
int protection,
int flags,
int fd,
off_t offset,
void** result) {
INVOKE_REPLACEMENT(MmapReplacement, mmap_replacement_,
(start, size, protection, flags, fd, offset, result));
}
void MallocHook::InvokeMunmapHookSlow(const void* p, size_t s) {
INVOKE_HOOKS(MunmapHook, munmap_hooks_, (p, s));
}
bool MallocHook::InvokeMunmapReplacementSlow(const void* p,
size_t s,
int* result) {
INVOKE_REPLACEMENT(MunmapReplacement, munmap_replacement_, (p, s, result));
}
void MallocHook::InvokeMremapHookSlow(const void* result,
const void* old_addr,
size_t old_size,
size_t new_size,
int flags,
const void* new_addr) {
INVOKE_HOOKS(MremapHook, mremap_hooks_, (result, old_addr, old_size, new_size,
flags, new_addr));
}
void MallocHook::InvokePreSbrkHookSlow(ptrdiff_t increment) {
INVOKE_HOOKS(PreSbrkHook, presbrk_hooks_, (increment));
}
void MallocHook::InvokeSbrkHookSlow(const void* result, ptrdiff_t increment) {
INVOKE_HOOKS(SbrkHook, sbrk_hooks_, (result, increment));
}
#undef INVOKE_HOOKS
DEFINE_ATTRIBUTE_SECTION_VARS(google_malloc);
DECLARE_ATTRIBUTE_SECTION_VARS(google_malloc);
DEFINE_ATTRIBUTE_SECTION_VARS(malloc_hook);
DECLARE_ATTRIBUTE_SECTION_VARS(malloc_hook);
#define ADDR_IN_ATTRIBUTE_SECTION(addr, name) \
(reinterpret_cast<uintptr_t>(ATTRIBUTE_SECTION_START(name)) <= \
reinterpret_cast<uintptr_t>(addr) && \
reinterpret_cast<uintptr_t>(addr) < \
reinterpret_cast<uintptr_t>(ATTRIBUTE_SECTION_STOP(name)))
static inline bool InHookCaller(const void* caller) {
return ADDR_IN_ATTRIBUTE_SECTION(caller, google_malloc) ||
ADDR_IN_ATTRIBUTE_SECTION(caller, malloc_hook);
}
#undef ADDR_IN_ATTRIBUTE_SECTION
static bool checked_sections = false;
static inline void CheckInHookCaller() {
if (!checked_sections) {
INIT_ATTRIBUTE_SECTION_VARS(google_malloc);
if (ATTRIBUTE_SECTION_START(google_malloc) ==
ATTRIBUTE_SECTION_STOP(google_malloc)) {
RAW_LOG(ERROR, "google_malloc section is missing, "
"thus InHookCaller is broken!");
}
INIT_ATTRIBUTE_SECTION_VARS(malloc_hook);
if (ATTRIBUTE_SECTION_START(malloc_hook) ==
ATTRIBUTE_SECTION_STOP(malloc_hook)) {
RAW_LOG(ERROR, "malloc_hook section is missing, "
"thus InHookCaller is broken!");
}
checked_sections = true;
}
}
extern "C" int MallocHook_GetCallerStackTrace(void** result, int max_depth,
int skip_count) {
#if defined(NO_TCMALLOC_SAMPLES)
return 0;
#elif !defined(HAVE_ATTRIBUTE_SECTION_START)
return GetStackTrace(result, max_depth, skip_count + int(DEBUG_MODE));
#else
CheckInHookCaller();
static const int kMaxSkip = 32 + 6 + 3;
static const int kStackSize = kMaxSkip + 1;
void* stack[kStackSize];
int depth = GetStackTrace(stack, kStackSize, 1);
if (depth == 0)
return 0;
for (int i = 0; i < depth; ++i) {
if (InHookCaller(stack[i])) {
RAW_VLOG(10, "Found hooked allocator at %d: %p <- %p",
i, stack[i], stack[i+1]);
i += 1;
depth -= i;
if (depth > max_depth) depth = max_depth;
copy(stack + i, stack + i + depth, result);
if (depth < max_depth && depth + i == kStackSize) {
depth +=
GetStackTrace(result + depth, max_depth - depth, 1 + kStackSize);
}
return depth;
}
}
RAW_LOG(WARNING, "Hooked allocator frame not found, returning empty trace");
return 0;
#endif
}
#if defined(__linux)
# include "malloc_hook_mmap_linux.h"
#elif defined(__FreeBSD__)
# include "malloc_hook_mmap_freebsd.h"
#else
void* MallocHook::UnhookedMMap(void *start, size_t length, int prot,
int flags, int fd, off_t offset) {
void* result;
if (!MallocHook::InvokeMmapReplacement(
start, length, prot, flags, fd, offset, &result)) {
result = mmap(start, length, prot, flags, fd, offset);
}
return result;
}
int MallocHook::UnhookedMUnmap(void *start, size_t length) {
int result;
if (!MallocHook::InvokeMunmapReplacement(start, length, &result)) {
result = munmap(start, length);
}
return result;
}
#endif