This source file includes following definitions.
- q_back_
- Full
- Push
- Pop
- AllocName
- DeallocName
- IsMMapped
- IsValidMagicValue
- real_malloced_size
- real_mmapped_size
- real_size
- size2_addr
- size2_addr
- magic2_addr
- magic2_addr
- Initialize
- CheckAndClear
- CheckLocked
- data_addr
- data_addr
- data_offset
- data_size
- set_offset
- Allocate
- Deallocate
- FreeQueueSize
- ProcessFreeQueue
- InitDeletedBuffer
- CheckForDanglingWrites
- CheckForCorruptedBuffer
- FromRawPointer
- FromRawPointer
- Check
- CheckEverything
- MemoryStats
- CheckCallback
- StatsCallback
- DanglingWriteChecker
- TracePrintf
- TraceFd
- TraceStack
- __malloctrace_write
- DebugAllocate
- DebugDeallocate
- GetNumericProperty
- VerifyNewMemory
- VerifyArrayNewMemory
- VerifyMallocMemory
- VerifyAllMemory
- MallocMemoryStats
- GetEstimatedAllocatedSize
- GetAllocatedSize
- GetOwnership
- GetFreeListSizes
- debug_cpp_alloc
- do_debug_malloc_or_debug_cpp_alloc
- tc_malloc
- tc_free
- tc_calloc
- tc_cfree
- tc_realloc
- tc_new
- tc_new_nothrow
- tc_delete
- tc_delete_nothrow
- tc_newarray
- tc_newarray_nothrow
- tc_deletearray
- tc_deletearray_nothrow
- RoundUp
- do_debug_memalign
- debug_cpp_memalign
- do_debug_memalign_or_debug_cpp_memalign
- tc_memalign
- tc_posix_memalign
- tc_valloc
- tc_pvalloc
- tc_malloc_stats
- tc_mallopt
- tc_mallinfo
- tc_malloc_size
#include "config.h"
#include <errno.h>
#ifdef HAVE_FCNTL_H
#include <fcntl.h>
#endif
#ifdef HAVE_INTTYPES_H
#include <inttypes.h>
#endif
#ifdef HAVE_STRUCT_MALLINFO
# if defined(HAVE_MALLOC_H)
# include <malloc.h>
# elif defined(HAVE_MALLOC_MALLOC_H)
# include <malloc/malloc.h>
# elif defined(HAVE_SYS_MALLOC_H)
# include <sys/malloc.h>
# endif
#endif
#ifdef HAVE_PTHREAD
#include <pthread.h>
#endif
#include <stdarg.h>
#include <stdio.h>
#include <string.h>
#ifdef HAVE_MMAP
#include <sys/mman.h>
#endif
#include <sys/stat.h>
#include <sys/types.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#include <gperftools/malloc_extension.h>
#include <gperftools/malloc_hook.h>
#include <gperftools/stacktrace.h>
#include "addressmap-inl.h"
#include "base/commandlineflags.h"
#include "base/googleinit.h"
#include "base/logging.h"
#include "base/spinlock.h"
#include "malloc_hook-inl.h"
#include "symbolize.h"
#define TCMALLOC_USING_DEBUGALLOCATION
#include "tcmalloc.cc"
#ifndef __THROW
# define __THROW
#endif
#ifndef MAP_ANONYMOUS
# define MAP_ANONYMOUS MAP_ANON
#endif
DEFINE_bool(malloctrace,
EnvToBool("TCMALLOC_TRACE", false),
"Enables memory (de)allocation tracing to /tmp/google.alloc.");
#ifdef HAVE_MMAP
DEFINE_bool(malloc_page_fence,
EnvToBool("TCMALLOC_PAGE_FENCE", false),
"Enables putting of memory allocations at page boundaries "
"with a guard page following the allocation (to catch buffer "
"overruns right when they happen).");
DEFINE_bool(malloc_page_fence_never_reclaim,
EnvToBool("TCMALLOC_PAGE_FRANCE_NEVER_RECLAIM", false),
"Enables making the virtual address space inaccessible "
"upon a deallocation instead of returning it and reusing later.");
#else
DEFINE_bool(malloc_page_fence, false, "Not usable (requires mmap)");
DEFINE_bool(malloc_page_fence_never_reclaim, false, "Not usable (required mmap)");
#endif
DEFINE_bool(malloc_reclaim_memory,
EnvToBool("TCMALLOC_RECLAIM_MEMORY", true),
"If set to false, we never return memory to malloc "
"when an object is deallocated. This ensures that all "
"heap object addresses are unique.");
DEFINE_int32(max_free_queue_size,
EnvToInt("TCMALLOC_MAX_FREE_QUEUE_SIZE", 10*1024*1024),
"If greater than 0, keep freed blocks in a queue instead of "
"releasing them to the allocator immediately. Release them when "
"the total size of all blocks in the queue would otherwise exceed "
"this limit.");
DEFINE_bool(symbolize_stacktrace,
EnvToBool("TCMALLOC_SYMBOLIZE_STACKTRACE", true),
"Symbolize the stack trace when provided (on some error exits)");
extern "C" int pthread_once(pthread_once_t *, void (*)(void))
ATTRIBUTE_WEAK;
static void TracePrintf(int fd, const char *fmt, ...)
__attribute__ ((__format__ (__printf__, 2, 3)));
#define BASE_MALLOC_NEW do_malloc
#define BASE_MALLOC do_malloc
#define BASE_FREE do_free
#define BASE_MALLOC_STATS do_malloc_stats
#define BASE_MALLOPT do_mallopt
#define BASE_MALLINFO do_mallinfo
class MallocBlock;
template <typename QueueEntry>
class FreeQueue {
public:
FreeQueue() : q_front_(0), q_back_(0) {}
bool Full() {
return (q_front_ + 1) % kFreeQueueSize == q_back_;
}
void Push(const QueueEntry& block) {
q_[q_front_] = block;
q_front_ = (q_front_ + 1) % kFreeQueueSize;
}
QueueEntry Pop() {
RAW_CHECK(q_back_ != q_front_, "Queue is empty");
const QueueEntry& ret = q_[q_back_];
q_back_ = (q_back_ + 1) % kFreeQueueSize;
return ret;
}
size_t size() const {
return (q_front_ - q_back_ + kFreeQueueSize) % kFreeQueueSize;
}
private:
static const int kFreeQueueSize = 1024;
QueueEntry q_[kFreeQueueSize];
int q_front_;
int q_back_;
};
struct MallocBlockQueueEntry {
MallocBlockQueueEntry() : block(NULL), size(0),
num_deleter_pcs(0), deleter_threadid(0) {}
MallocBlockQueueEntry(MallocBlock* b, size_t s) : block(b), size(s) {
if (FLAGS_max_free_queue_size != 0 && b != NULL) {
num_deleter_pcs =
GetStackTrace(deleter_pcs,
sizeof(deleter_pcs) / sizeof(deleter_pcs[0]),
4);
deleter_threadid = pthread_self();
} else {
num_deleter_pcs = 0;
deleter_threadid = 0;
}
}
MallocBlock* block;
size_t size;
void* deleter_pcs[16];
int num_deleter_pcs;
pthread_t deleter_threadid;
};
class MallocBlock {
public:
static const int kMallocType = 0xEFCDAB90;
static const int kNewType = 0xFEBADC81;
static const int kArrayNewType = 0xBCEADF72;
private:
static const int kAllocTypeMask = 0x3;
static const int kDeallocatedTypeBit = 0x4;
static const int kMagicUninitializedByte = 0xAB;
static const int kMagicDeletedByte = 0xCD;
static const size_t kMagicDeletedSizeT =
0xCDCDCDCD | (((size_t)0xCDCDCDCD << 16) << 16);
static const int kMagicMalloc = 0xDEADBEEF;
static const int kMagicMMap = 0xABCDEFAB;
static unsigned char kMagicDeletedBuffer[1024];
static pthread_once_t deleted_buffer_initialized_;
static bool deleted_buffer_initialized_no_pthreads_;
private:
size_t size1_;
size_t offset_;
size_t magic1_;
size_t alloc_type_;
size_t size2_;
int magic2_;
private:
typedef AddressMap<int> AllocMap;
static AllocMap* alloc_map_;
static SpinLock alloc_map_lock_;
static FreeQueue<MallocBlockQueueEntry>* free_queue_;
static size_t free_queue_size_;
static SpinLock free_queue_lock_;
static const char* const kAllocName[];
static const char* const kDeallocName[];
static const char* AllocName(int type) {
return kAllocName[type & kAllocTypeMask];
}
static const char* DeallocName(int type) {
return kDeallocName[type & kAllocTypeMask];
}
private:
bool IsMMapped() const { return kMagicMMap == magic1_; }
bool IsValidMagicValue(int value) const {
return kMagicMMap == value || kMagicMalloc == value;
}
static size_t real_malloced_size(size_t size) {
return size + sizeof(MallocBlock);
}
static size_t real_mmapped_size(size_t size) {
return size + MallocBlock::data_offset();
}
size_t real_size() {
return IsMMapped() ? real_mmapped_size(size1_) : real_malloced_size(size1_);
}
size_t* size2_addr() { return (size_t*)((char*)&size2_ + size1_); }
const size_t* size2_addr() const {
return (const size_t*)((char*)&size2_ + size1_);
}
int* magic2_addr() { return (int*)(size2_addr() + 1); }
const int* magic2_addr() const { return (const int*)(size2_addr() + 1); }
private:
void Initialize(size_t size, int type) {
RAW_CHECK(IsValidMagicValue(magic1_), "");
alloc_map_lock_.Lock();
if (!alloc_map_) {
void* p = BASE_MALLOC(sizeof(AllocMap));
alloc_map_ = new(p) AllocMap(BASE_MALLOC, BASE_FREE);
}
alloc_map_->Insert(data_addr(), type);
size1_ = size;
offset_ = 0;
alloc_type_ = type;
if (!IsMMapped()) {
*magic2_addr() = magic1_;
*size2_addr() = size;
}
alloc_map_lock_.Unlock();
memset(data_addr(), kMagicUninitializedByte, size);
if (!IsMMapped()) {
RAW_CHECK(size1_ == *size2_addr(), "should hold");
RAW_CHECK(magic1_ == *magic2_addr(), "should hold");
}
}
size_t CheckAndClear(int type) {
alloc_map_lock_.Lock();
CheckLocked(type);
if (!IsMMapped()) {
RAW_CHECK(size1_ == *size2_addr(), "should hold");
}
alloc_map_->Insert(data_addr(), type | kDeallocatedTypeBit);
alloc_map_lock_.Unlock();
const size_t size = real_size();
memset(this, kMagicDeletedByte, size);
return size;
}
void CheckLocked(int type) const {
int map_type = 0;
const int* found_type =
alloc_map_ != NULL ? alloc_map_->Find(data_addr()) : NULL;
if (found_type == NULL) {
RAW_LOG(FATAL, "memory allocation bug: object at %p "
"has never been allocated", data_addr());
} else {
map_type = *found_type;
}
if ((map_type & kDeallocatedTypeBit) != 0) {
RAW_LOG(FATAL, "memory allocation bug: object at %p "
"has been already deallocated (it was allocated with %s)",
data_addr(), AllocName(map_type & ~kDeallocatedTypeBit));
}
if (alloc_type_ == kMagicDeletedSizeT) {
RAW_LOG(FATAL, "memory stomping bug: a word before object at %p "
"has been corrupted; or else the object has been already "
"deallocated and our memory map has been corrupted",
data_addr());
}
if (!IsValidMagicValue(magic1_)) {
RAW_LOG(FATAL, "memory stomping bug: a word before object at %p "
"has been corrupted; "
"or else our memory map has been corrupted and this is a "
"deallocation for not (currently) heap-allocated object",
data_addr());
}
if (!IsMMapped()) {
if (size1_ != *size2_addr()) {
RAW_LOG(FATAL, "memory stomping bug: a word after object at %p "
"has been corrupted", data_addr());
}
if (!IsValidMagicValue(*magic2_addr())) {
RAW_LOG(FATAL, "memory stomping bug: a word after object at %p "
"has been corrupted", data_addr());
}
}
if (alloc_type_ != type) {
if ((alloc_type_ != MallocBlock::kMallocType) &&
(alloc_type_ != MallocBlock::kNewType) &&
(alloc_type_ != MallocBlock::kArrayNewType)) {
RAW_LOG(FATAL, "memory stomping bug: a word before object at %p "
"has been corrupted", data_addr());
}
RAW_LOG(FATAL, "memory allocation/deallocation mismatch at %p: "
"allocated with %s being deallocated with %s",
data_addr(), AllocName(alloc_type_), DeallocName(type));
}
if (alloc_type_ != map_type) {
RAW_LOG(FATAL, "memory stomping bug: our memory map has been corrupted : "
"allocation at %p made with %s "
"is recorded in the map to be made with %s",
data_addr(), AllocName(alloc_type_), AllocName(map_type));
}
}
public:
void* data_addr() { return (void*)&size2_; }
const void* data_addr() const { return (const void*)&size2_; }
static size_t data_offset() { return OFFSETOF_MEMBER(MallocBlock, size2_); }
size_t data_size() const { return size1_; }
void set_offset(int offset) { this->offset_ = offset; }
public:
static MallocBlock* Allocate(size_t size, int type) {
static size_t max_size_t = ~0;
if (size > max_size_t - sizeof(MallocBlock)) {
RAW_LOG(ERROR, "Massive size passed to malloc: %"PRIuS"", size);
return NULL;
}
MallocBlock* b = NULL;
const bool use_malloc_page_fence = FLAGS_malloc_page_fence;
#ifdef HAVE_MMAP
if (use_malloc_page_fence) {
size_t sz = real_mmapped_size(size);
int pagesize = getpagesize();
int num_pages = (sz + pagesize - 1) / pagesize + 1;
char* p = (char*) mmap(NULL, num_pages * pagesize, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_ANONYMOUS, -1, 0);
if (p == MAP_FAILED) {
RAW_LOG(FATAL, "Out of memory: possibly due to page fence overhead: %s",
strerror(errno));
}
if (mprotect(p + (num_pages - 1) * pagesize, pagesize, PROT_NONE)) {
RAW_LOG(FATAL, "Guard page setup failed: %s", strerror(errno));
}
b = (MallocBlock*) (p + (num_pages - 1) * pagesize - sz);
} else {
b = (MallocBlock*) (type == kMallocType ?
BASE_MALLOC(real_malloced_size(size)) :
BASE_MALLOC_NEW(real_malloced_size(size)));
}
#else
b = (MallocBlock*) (type == kMallocType ?
BASE_MALLOC(real_malloced_size(size)) :
BASE_MALLOC_NEW(real_malloced_size(size)));
#endif
if (b != NULL) {
b->magic1_ = use_malloc_page_fence ? kMagicMMap : kMagicMalloc;
b->Initialize(size, type);
}
return b;
}
void Deallocate(int type) {
if (IsMMapped()) {
#ifdef HAVE_MMAP
int size = CheckAndClear(type);
int pagesize = getpagesize();
int num_pages = (size + pagesize - 1) / pagesize + 1;
char* p = (char*) this;
if (FLAGS_malloc_page_fence_never_reclaim ||
!FLAGS_malloc_reclaim_memory) {
mprotect(p - (num_pages - 1) * pagesize + size,
num_pages * pagesize, PROT_NONE);
} else {
munmap(p - (num_pages - 1) * pagesize + size, num_pages * pagesize);
}
#endif
} else {
const size_t size = CheckAndClear(type);
if (FLAGS_malloc_reclaim_memory) {
ProcessFreeQueue(this, size, FLAGS_max_free_queue_size);
}
}
}
static size_t FreeQueueSize() {
SpinLockHolder l(&free_queue_lock_);
return free_queue_size_;
}
static void ProcessFreeQueue(MallocBlock* b, size_t size,
int max_free_queue_size) {
MallocBlockQueueEntry entries[4];
int num_entries = 0;
MallocBlockQueueEntry new_entry(b, size);
free_queue_lock_.Lock();
if (free_queue_ == NULL)
free_queue_ = new FreeQueue<MallocBlockQueueEntry>;
RAW_CHECK(!free_queue_->Full(), "Free queue mustn't be full!");
if (b != NULL) {
free_queue_size_ += size + sizeof(MallocBlockQueueEntry);
free_queue_->Push(new_entry);
}
while (free_queue_size_ > max_free_queue_size || free_queue_->Full()) {
RAW_CHECK(num_entries < arraysize(entries), "entries array overflow");
entries[num_entries] = free_queue_->Pop();
free_queue_size_ -=
entries[num_entries].size + sizeof(MallocBlockQueueEntry);
num_entries++;
if (num_entries == arraysize(entries)) {
free_queue_lock_.Unlock();
for (int i = 0; i < num_entries; i++) {
CheckForDanglingWrites(entries[i]);
BASE_FREE(entries[i].block);
}
num_entries = 0;
free_queue_lock_.Lock();
}
}
RAW_CHECK(free_queue_size_ >= 0, "Free queue size went negative!");
free_queue_lock_.Unlock();
for (int i = 0; i < num_entries; i++) {
CheckForDanglingWrites(entries[i]);
BASE_FREE(entries[i].block);
}
}
static void InitDeletedBuffer() {
memset(kMagicDeletedBuffer, kMagicDeletedByte, sizeof(kMagicDeletedBuffer));
deleted_buffer_initialized_no_pthreads_ = true;
}
static void CheckForDanglingWrites(const MallocBlockQueueEntry& queue_entry) {
if (pthread_once)
pthread_once(&deleted_buffer_initialized_, &InitDeletedBuffer);
if (!deleted_buffer_initialized_no_pthreads_) {
InitDeletedBuffer();
}
const unsigned char* p =
reinterpret_cast<unsigned char*>(queue_entry.block);
static const size_t size_of_buffer = sizeof(kMagicDeletedBuffer);
const size_t size = queue_entry.size;
const size_t buffers = size / size_of_buffer;
const size_t remainder = size % size_of_buffer;
size_t buffer_idx;
for (buffer_idx = 0; buffer_idx < buffers; ++buffer_idx) {
CheckForCorruptedBuffer(queue_entry, buffer_idx, p, size_of_buffer);
p += size_of_buffer;
}
CheckForCorruptedBuffer(queue_entry, buffer_idx, p, remainder);
}
static void CheckForCorruptedBuffer(const MallocBlockQueueEntry& queue_entry,
size_t buffer_idx,
const unsigned char* buffer,
size_t size_of_buffer) {
if (memcmp(buffer, kMagicDeletedBuffer, size_of_buffer) == 0) {
return;
}
RAW_LOG(ERROR,
"Found a corrupted memory buffer in MallocBlock (may be offset "
"from user ptr): buffer index: %zd, buffer ptr: %p, size of "
"buffer: %zd", buffer_idx, buffer, size_of_buffer);
if (size_of_buffer <= 1024) {
for (int i = 0; i < size_of_buffer; ++i) {
if (buffer[i] != kMagicDeletedByte) {
RAW_LOG(ERROR, "Buffer byte %d is 0x%02x (should be 0x%02x).",
i, buffer[i], kMagicDeletedByte);
}
}
} else {
RAW_LOG(ERROR, "Buffer too large to print corruption.");
}
const MallocBlock* b = queue_entry.block;
const size_t size = queue_entry.size;
if (queue_entry.num_deleter_pcs > 0) {
TracePrintf(STDERR_FILENO, "Deleted by thread %p\n",
reinterpret_cast<void*>(
PRINTABLE_PTHREAD(queue_entry.deleter_threadid)));
union { void* alignment; char buf[sizeof(SymbolTable)]; } tablebuf;
SymbolTable* symbolization_table = new (tablebuf.buf) SymbolTable;
for (int i = 0; i < queue_entry.num_deleter_pcs; i++) {
char *pc = reinterpret_cast<char*>(queue_entry.deleter_pcs[i]);
symbolization_table->Add(pc - 1);
}
if (FLAGS_symbolize_stacktrace)
symbolization_table->Symbolize();
for (int i = 0; i < queue_entry.num_deleter_pcs; i++) {
char *pc = reinterpret_cast<char*>(queue_entry.deleter_pcs[i]);
TracePrintf(STDERR_FILENO, " @ %p %s\n",
pc, symbolization_table->GetSymbol(pc - 1));
}
} else {
RAW_LOG(ERROR,
"Skipping the printing of the deleter's stack! Its stack was "
"not found; either the corruption occurred too early in "
"execution to obtain a stack trace or --max_free_queue_size was "
"set to 0.");
}
RAW_LOG(FATAL,
"Memory was written to after being freed. MallocBlock: %p, user "
"ptr: %p, size: %zd. If you can't find the source of the error, "
"try using ASan (http://code.google.com/p/address-sanitizer/), "
"Valgrind, or Purify, or study the "
"output of the deleter's stack printed above.",
b, b->data_addr(), size);
}
static MallocBlock* FromRawPointer(void* p) {
const size_t data_offset = MallocBlock::data_offset();
MallocBlock *mb = reinterpret_cast<MallocBlock *>(
reinterpret_cast<char *>(p) - data_offset);
if (mb->alloc_type_ == kMagicDeletedSizeT) {
RAW_LOG(FATAL, "memory allocation bug: object at %p has been already"
" deallocated; or else a word before the object has been"
" corrupted (memory stomping bug)", p);
}
return reinterpret_cast<MallocBlock *>(
reinterpret_cast<char *>(mb) - mb->offset_);
}
static const MallocBlock* FromRawPointer(const void* p) {
return FromRawPointer(const_cast<void*>(p));
}
void Check(int type) const {
alloc_map_lock_.Lock();
CheckLocked(type);
alloc_map_lock_.Unlock();
}
static bool CheckEverything() {
alloc_map_lock_.Lock();
if (alloc_map_ != NULL) alloc_map_->Iterate(CheckCallback, 0);
alloc_map_lock_.Unlock();
return true;
}
static bool MemoryStats(int* blocks, size_t* total,
int histogram[kMallocHistogramSize]) {
memset(histogram, 0, kMallocHistogramSize * sizeof(int));
alloc_map_lock_.Lock();
stats_blocks_ = 0;
stats_total_ = 0;
stats_histogram_ = histogram;
if (alloc_map_ != NULL) alloc_map_->Iterate(StatsCallback, 0);
*blocks = stats_blocks_;
*total = stats_total_;
alloc_map_lock_.Unlock();
return true;
}
private:
static void CheckCallback(const void* ptr, int* type, int dummy) {
if ((*type & kDeallocatedTypeBit) == 0) {
FromRawPointer(ptr)->CheckLocked(*type);
}
}
static int stats_blocks_;
static size_t stats_total_;
static int* stats_histogram_;
static void StatsCallback(const void* ptr, int* type, int dummy) {
if ((*type & kDeallocatedTypeBit) == 0) {
const MallocBlock* b = FromRawPointer(ptr);
b->CheckLocked(*type);
++stats_blocks_;
size_t mysize = b->size1_;
int entry = 0;
stats_total_ += mysize;
while (mysize) {
++entry;
mysize >>= 1;
}
RAW_CHECK(entry < kMallocHistogramSize,
"kMallocHistogramSize should be at least as large as log2 "
"of the maximum process memory size");
stats_histogram_[entry] += 1;
}
}
};
void DanglingWriteChecker() {
MallocBlock::ProcessFreeQueue(NULL, 0, 0);
}
const int MallocBlock::kMagicMalloc;
const int MallocBlock::kMagicMMap;
MallocBlock::AllocMap* MallocBlock::alloc_map_ = NULL;
SpinLock MallocBlock::alloc_map_lock_(SpinLock::LINKER_INITIALIZED);
FreeQueue<MallocBlockQueueEntry>* MallocBlock::free_queue_ = NULL;
size_t MallocBlock::free_queue_size_ = 0;
SpinLock MallocBlock::free_queue_lock_(SpinLock::LINKER_INITIALIZED);
unsigned char MallocBlock::kMagicDeletedBuffer[1024];
pthread_once_t MallocBlock::deleted_buffer_initialized_ = PTHREAD_ONCE_INIT;
bool MallocBlock::deleted_buffer_initialized_no_pthreads_ = false;
const char* const MallocBlock::kAllocName[] = {
"malloc",
"new",
"new []",
NULL,
};
const char* const MallocBlock::kDeallocName[] = {
"free",
"delete",
"delete []",
NULL,
};
int MallocBlock::stats_blocks_;
size_t MallocBlock::stats_total_;
int* MallocBlock::stats_histogram_;
static void TracePrintf(int fd, const char *fmt, ...) {
char buf[64];
int i = 0;
va_list ap;
va_start(ap, fmt);
const char *p = fmt;
char numbuf[25];
numbuf[sizeof(numbuf)-1] = 0;
while (*p != '\0') {
char *s = &numbuf[sizeof(numbuf)-1];
if (p[0] == '%' && p[1] != 0) {
int64 l = 0;
unsigned long base = 0;
if (*++p == 's') {
s = va_arg(ap, char *);
} else if (*p == 'l' && p[1] == 'd') {
l = va_arg(ap, long);
base = 10;
p++;
} else if (*p == 'l' && p[1] == 'u') {
l = va_arg(ap, unsigned long);
base = 10;
p++;
} else if (*p == 'z' && p[1] == 'u') {
l = va_arg(ap, size_t);
base = 10;
p++;
} else if (*p == 'u') {
l = va_arg(ap, unsigned int);
base = 10;
} else if (*p == 'd') {
l = va_arg(ap, int);
base = 10;
} else if (*p == 'p') {
l = va_arg(ap, intptr_t);
base = 16;
} else {
write(STDERR_FILENO, "Unimplemented TracePrintf format\n", 33);
write(STDERR_FILENO, p, 2);
write(STDERR_FILENO, "\n", 1);
abort();
}
p++;
if (base != 0) {
bool minus = (l < 0 && base == 10);
uint64 ul = minus? -l : l;
do {
*--s = "0123456789abcdef"[ul % base];
ul /= base;
} while (ul != 0);
if (base == 16) {
*--s = 'x';
*--s = '0';
} else if (minus) {
*--s = '-';
}
}
} else {
*--s = *p++;
}
while (*s != 0) {
if (i == sizeof(buf)) {
write(fd, buf, i);
i = 0;
}
buf[i++] = *s++;
}
}
if (i != 0) {
write(fd, buf, i);
}
va_end(ap);
}
static int TraceFd() {
static int trace_fd = -1;
if (trace_fd == -1) {
trace_fd = open("/tmp/google.alloc", O_CREAT|O_TRUNC|O_WRONLY, 0666);
if (trace_fd == -1) {
trace_fd = 2;
TracePrintf(trace_fd,
"Can't open /tmp/google.alloc. Logging to stderr.\n");
}
TracePrintf(trace_fd, "Trace started: %lu\n",
static_cast<unsigned long>(time(NULL)));
TracePrintf(trace_fd,
"func\tsize\tptr\tthread_id\tstack pcs for tools/symbolize\n");
}
return trace_fd;
}
static void TraceStack(void) {
void *pcs[16];
int n = GetStackTrace(pcs, sizeof(pcs)/sizeof(pcs[0]), 0);
for (int i = 0; i != n; i++) {
TracePrintf(TraceFd(), "\t%p", pcs[i]);
}
}
static SpinLock malloc_trace_lock(SpinLock::LINKER_INITIALIZED);
#define MALLOC_TRACE(name, size, addr) \
do { \
if (FLAGS_malloctrace) { \
SpinLockHolder l(&malloc_trace_lock); \
TracePrintf(TraceFd(), "%s\t%"PRIuS"\t%p\t%"GPRIuPTHREAD, \
name, size, addr, PRINTABLE_PTHREAD(pthread_self())); \
TraceStack(); \
TracePrintf(TraceFd(), "\n"); \
} \
} while (0)
void __malloctrace_write(const char *buf, size_t size) {
if (FLAGS_malloctrace) {
write(TraceFd(), buf, size);
}
}
static inline void* DebugAllocate(size_t size, int type) {
MallocBlock* ptr = MallocBlock::Allocate(size, type);
if (ptr == NULL) return NULL;
MALLOC_TRACE("malloc", size, ptr->data_addr());
return ptr->data_addr();
}
static inline void DebugDeallocate(void* ptr, int type) {
MALLOC_TRACE("free",
(ptr != 0 ? MallocBlock::FromRawPointer(ptr)->data_size() : 0),
ptr);
if (ptr) MallocBlock::FromRawPointer(ptr)->Deallocate(type);
}
class DebugMallocImplementation : public TCMallocImplementation {
public:
virtual bool GetNumericProperty(const char* name, size_t* value) {
bool result = TCMallocImplementation::GetNumericProperty(name, value);
if (result && (strcmp(name, "generic.current_allocated_bytes") == 0)) {
size_t qsize = MallocBlock::FreeQueueSize();
if (*value >= qsize) {
*value -= qsize;
}
}
return result;
}
virtual bool VerifyNewMemory(const void* p) {
if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kNewType);
return true;
}
virtual bool VerifyArrayNewMemory(const void* p) {
if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kArrayNewType);
return true;
}
virtual bool VerifyMallocMemory(const void* p) {
if (p) MallocBlock::FromRawPointer(p)->Check(MallocBlock::kMallocType);
return true;
}
virtual bool VerifyAllMemory() {
return MallocBlock::CheckEverything();
}
virtual bool MallocMemoryStats(int* blocks, size_t* total,
int histogram[kMallocHistogramSize]) {
return MallocBlock::MemoryStats(blocks, total, histogram);
}
virtual size_t GetEstimatedAllocatedSize(size_t size) {
return size;
}
virtual size_t GetAllocatedSize(const void* p) {
if (p) {
RAW_CHECK(GetOwnership(p) != MallocExtension::kNotOwned,
"ptr not allocated by tcmalloc");
return MallocBlock::FromRawPointer(p)->data_size();
}
return 0;
}
virtual MallocExtension::Ownership GetOwnership(const void* p) {
if (p) {
const MallocBlock* mb = MallocBlock::FromRawPointer(p);
return TCMallocImplementation::GetOwnership(mb);
}
return MallocExtension::kNotOwned;
}
virtual void GetFreeListSizes(vector<MallocExtension::FreeListInfo>* v) {
static const char* kDebugFreeQueue = "debug.free_queue";
TCMallocImplementation::GetFreeListSizes(v);
MallocExtension::FreeListInfo i;
i.type = kDebugFreeQueue;
i.min_object_size = 0;
i.max_object_size = numeric_limits<size_t>::max();
i.total_bytes_free = MallocBlock::FreeQueueSize();
v->push_back(i);
}
};
static DebugMallocImplementation debug_malloc_implementation;
REGISTER_MODULE_INITIALIZER(debugallocation, {
if (!RunningOnValgrind()) {
MallocExtension::Register(&debug_malloc_implementation);
}
});
REGISTER_MODULE_DESTRUCTOR(debugallocation, {
if (!RunningOnValgrind()) {
DanglingWriteChecker();
}
});
inline void* debug_cpp_alloc(size_t size, int new_type, bool nothrow) {
for (;;) {
void* p = DebugAllocate(size, new_type);
#ifdef PREANSINEW
return p;
#else
if (p == NULL) {
std::new_handler nh;
{
SpinLockHolder h(&set_new_handler_lock);
nh = std::set_new_handler(0);
(void) std::set_new_handler(nh);
}
#if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
if (nh) {
(*nh)();
continue;
}
return 0;
#else
if (!nh) {
if (nothrow) return 0;
throw std::bad_alloc();
}
try {
(*nh)();
} catch (const std::bad_alloc&) {
if (!nothrow) throw;
return p;
}
#endif
} else {
return p;
}
#endif
}
}
inline void* do_debug_malloc_or_debug_cpp_alloc(size_t size) {
return tc_new_mode ? debug_cpp_alloc(size, MallocBlock::kMallocType, true)
: DebugAllocate(size, MallocBlock::kMallocType);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) __THROW {
void* ptr = do_debug_malloc_or_debug_cpp_alloc(size);
MallocHook::InvokeNewHook(ptr, size);
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void tc_free(void* ptr) __THROW {
MallocHook::InvokeDeleteHook(ptr);
DebugDeallocate(ptr, MallocBlock::kMallocType);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_calloc(size_t count, size_t size) __THROW {
const size_t total_size = count * size;
if (size != 0 && total_size / size != count) return NULL;
void* block = do_debug_malloc_or_debug_cpp_alloc(total_size);
MallocHook::InvokeNewHook(block, total_size);
if (block) memset(block, 0, total_size);
return block;
}
extern "C" PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) __THROW {
MallocHook::InvokeDeleteHook(ptr);
DebugDeallocate(ptr, MallocBlock::kMallocType);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) __THROW {
if (ptr == NULL) {
ptr = do_debug_malloc_or_debug_cpp_alloc(size);
MallocHook::InvokeNewHook(ptr, size);
return ptr;
}
if (size == 0) {
MallocHook::InvokeDeleteHook(ptr);
DebugDeallocate(ptr, MallocBlock::kMallocType);
return NULL;
}
MallocBlock* old = MallocBlock::FromRawPointer(ptr);
old->Check(MallocBlock::kMallocType);
MallocBlock* p = MallocBlock::Allocate(size, MallocBlock::kMallocType);
if (p == NULL) return NULL;
memcpy(p->data_addr(), old->data_addr(),
(old->data_size() < size) ? old->data_size() : size);
MallocHook::InvokeDeleteHook(ptr);
MallocHook::InvokeNewHook(p->data_addr(), size);
DebugDeallocate(ptr, MallocBlock::kMallocType);
MALLOC_TRACE("realloc", p->data_size(), p->data_addr());
return p->data_addr();
}
extern "C" PERFTOOLS_DLL_DECL void* tc_new(size_t size) {
void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, false);
MallocHook::InvokeNewHook(ptr, size);
if (ptr == NULL) {
RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new failed.", size);
}
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size, const std::nothrow_t&) __THROW {
void* ptr = debug_cpp_alloc(size, MallocBlock::kNewType, true);
MallocHook::InvokeNewHook(ptr, size);
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete(void* p) __THROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kNewType);
}
extern "C" PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p, const std::nothrow_t&) __THROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kNewType);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray(size_t size) {
void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, false);
MallocHook::InvokeNewHook(ptr, size);
if (ptr == NULL) {
RAW_LOG(FATAL, "Unable to allocate %"PRIuS" bytes: new[] failed.", size);
}
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size, const std::nothrow_t&)
__THROW {
void* ptr = debug_cpp_alloc(size, MallocBlock::kArrayNewType, true);
MallocHook::InvokeNewHook(ptr, size);
return ptr;
}
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray(void* p) __THROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kArrayNewType);
}
extern "C" PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p, const std::nothrow_t&) __THROW {
MallocHook::InvokeDeleteHook(p);
DebugDeallocate(p, MallocBlock::kArrayNewType);
}
static intptr_t RoundUp(intptr_t value, intptr_t alignment) {
return (value + alignment - 1) & ~(alignment - 1);
}
static void *do_debug_memalign(size_t alignment, size_t size) {
void *p = 0;
RAW_CHECK((alignment & (alignment-1)) == 0, "must be power of two");
const size_t data_offset = MallocBlock::data_offset();
size_t extra_bytes = data_offset + alignment - 1;
if (size + extra_bytes < size) return NULL;
p = DebugAllocate(size + extra_bytes, MallocBlock::kMallocType);
if (p != 0) {
intptr_t orig_p = reinterpret_cast<intptr_t>(p);
p = reinterpret_cast<void *>(RoundUp(orig_p + data_offset, alignment));
MallocBlock *fake_hdr = reinterpret_cast<MallocBlock *>(
reinterpret_cast<char *>(p) - data_offset);
fake_hdr->set_offset(reinterpret_cast<intptr_t>(p) - orig_p);
}
return p;
}
static void* debug_cpp_memalign(size_t align, size_t size) {
for (;;) {
void* p = do_debug_memalign(align, size);
#ifdef PREANSINEW
return p;
#else
if (p == NULL) {
std::new_handler nh;
{
SpinLockHolder h(&set_new_handler_lock);
nh = std::set_new_handler(0);
(void) std::set_new_handler(nh);
}
#if (defined(__GNUC__) && !defined(__EXCEPTIONS)) || (defined(_HAS_EXCEPTIONS) && !_HAS_EXCEPTIONS)
if (nh) {
(*nh)();
continue;
}
return 0;
#else
if (!nh)
return 0;
try {
(*nh)();
} catch (const std::bad_alloc&) {
return p;
}
#endif
} else {
return p;
}
#endif
}
}
inline void* do_debug_memalign_or_debug_cpp_memalign(size_t align,
size_t size) {
return tc_new_mode ? debug_cpp_memalign(align, size)
: do_debug_memalign(align, size);
}
extern "C" PERFTOOLS_DLL_DECL void* tc_memalign(size_t align, size_t size) __THROW {
void *p = do_debug_memalign_or_debug_cpp_memalign(align, size);
MallocHook::InvokeNewHook(p, size);
return p;
}
extern "C" PERFTOOLS_DLL_DECL int tc_posix_memalign(void** result_ptr, size_t align, size_t size)
__THROW {
if (((align % sizeof(void*)) != 0) ||
((align & (align - 1)) != 0) ||
(align == 0)) {
return EINVAL;
}
void* result = do_debug_memalign_or_debug_cpp_memalign(align, size);
MallocHook::InvokeNewHook(result, size);
if (result == NULL) {
return ENOMEM;
} else {
*result_ptr = result;
return 0;
}
}
extern "C" PERFTOOLS_DLL_DECL void* tc_valloc(size_t size) __THROW {
void *p = do_debug_memalign_or_debug_cpp_memalign(getpagesize(), size);
MallocHook::InvokeNewHook(p, size);
return p;
}
extern "C" PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t size) __THROW {
int pagesize = getpagesize();
size = RoundUp(size, pagesize);
if (size == 0) {
size = pagesize;
}
void *p = do_debug_memalign_or_debug_cpp_memalign(pagesize, size);
MallocHook::InvokeNewHook(p, size);
return p;
}
extern "C" PERFTOOLS_DLL_DECL void tc_malloc_stats(void) __THROW {
BASE_MALLOC_STATS();
}
extern "C" PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) __THROW {
return BASE_MALLOPT(cmd, value);
}
#ifdef HAVE_STRUCT_MALLINFO
extern "C" PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) __THROW {
return BASE_MALLINFO();
}
#endif
extern "C" PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) __THROW {
return MallocExtension::instance()->GetAllocatedSize(ptr);
}