#ifndef Heap_h
#define Heap_h
#include "platform/PlatformExport.h"
#include "platform/heap/AddressSanitizer.h"
#include "platform/heap/ThreadState.h"
#include "platform/heap/Visitor.h"
#include "wtf/Assertions.h"
#include "wtf/OwnPtr.h"
#include "wtf/PassRefPtr.h"
#include <stdint.h>
namespace WebCore {
const size_t blinkPageSizeLog2 = 17;
const size_t blinkPageSize = 1 << blinkPageSizeLog2;
const size_t blinkPageOffsetMask = blinkPageSize - 1;
const size_t blinkPageBaseMask = ~blinkPageOffsetMask;
const size_t allocationGranularity = 8;
const size_t allocationMask = allocationGranularity - 1;
const size_t objectStartBitMapSize = (blinkPageSize + ((8 * allocationGranularity) - 1)) / (8 * allocationGranularity);
const size_t reservedForObjectBitMap = ((objectStartBitMapSize + allocationMask) & ~allocationMask);
const size_t maxHeapObjectSize = 1 << 27;
const size_t markBitMask = 1;
const size_t freeListMask = 2;
const size_t debugBitMask = 4;
const size_t sizeMask = ~7;
const uint8_t freelistZapValue = 42;
const uint8_t finalizedZapValue = 24;
class HeapStats;
class PageMemory;
template<ThreadAffinity affinity> class ThreadLocalPersistents;
template<typename T, typename RootsAccessor = ThreadLocalPersistents<ThreadingTrait<T>::Affinity > > class Persistent;
PLATFORM_EXPORT size_t osPageSize();
inline size_t blinkPagePayloadSize()
{
return blinkPageSize - 2 * osPageSize();
}
inline Address roundToBlinkPageStart(Address address)
{
return reinterpret_cast<Address>(reinterpret_cast<uintptr_t>(address) & blinkPageBaseMask);
}
template<typename Header>
inline size_t headerPadding()
{
return (allocationGranularity - (sizeof(Header) % allocationGranularity)) % allocationGranularity;
}
inline Address blinkPageAddress(Address address)
{
return reinterpret_cast<Address>(reinterpret_cast<uintptr_t>(address) & blinkPageBaseMask);
}
#ifndef NDEBUG
inline bool isPageHeaderAddress(Address address)
{
return !((reinterpret_cast<uintptr_t>(address) & blinkPageOffsetMask) - osPageSize());
}
#endif
PLATFORM_EXPORT inline Address pageHeaderAddress(Address address)
{
return blinkPageAddress(address) + osPageSize();
}
class BaseHeapPage {
public:
BaseHeapPage(PageMemory* storage, const GCInfo* gcInfo, ThreadState* state)
: m_storage(storage)
, m_gcInfo(gcInfo)
, m_threadState(state)
, m_padding(0)
{
ASSERT(isPageHeaderAddress(reinterpret_cast<Address>(this)));
}
virtual bool checkAndMarkPointer(Visitor*, Address) = 0;
Address address() { return reinterpret_cast<Address>(this); }
PageMemory* storage() const { return m_storage; }
ThreadState* threadState() const { return m_threadState; }
const GCInfo* gcInfo() { return m_gcInfo; }
private:
void* padding() const { return m_padding; }
PageMemory* m_storage;
const GCInfo* m_gcInfo;
ThreadState* m_threadState;
void* m_padding;
};
template<typename Header>
class LargeHeapObject : public BaseHeapPage {
public:
LargeHeapObject(PageMemory* storage, const GCInfo* gcInfo, ThreadState* state) : BaseHeapPage(storage, gcInfo, state)
{
COMPILE_ASSERT(!(sizeof(LargeHeapObject<Header>) & allocationMask), large_heap_object_header_misaligned);
}
virtual bool checkAndMarkPointer(Visitor*, Address);
void link(LargeHeapObject<Header>** previousNext)
{
m_next = *previousNext;
*previousNext = this;
}
void unlink(LargeHeapObject<Header>** previousNext)
{
*previousNext = m_next;
}
bool contains(Address object)
{
return (address() <= object) && (object <= (address() + size()));
}
LargeHeapObject<Header>* next()
{
return m_next;
}
size_t size()
{
return heapObjectHeader()->size() + sizeof(LargeHeapObject<Header>) + headerPadding<Header>();
}
Address payload() { return heapObjectHeader()->payload(); }
size_t payloadSize() { return heapObjectHeader()->payloadSize(); }
Header* heapObjectHeader()
{
Address headerAddress = address() + sizeof(LargeHeapObject<Header>) + headerPadding<Header>();
return reinterpret_cast<Header*>(headerAddress);
}
bool isMarked();
void unmark();
void getStats(HeapStats&);
void mark(Visitor*);
void finalize();
private:
friend class Heap;
friend class ThreadHeap<Header>;
LargeHeapObject<Header>* m_next;
};
class PLATFORM_EXPORT BasicObjectHeader {
public:
NO_SANITIZE_ADDRESS
explicit BasicObjectHeader(size_t encodedSize)
: m_size(encodedSize) { }
static size_t freeListEncodedSize(size_t size) { return size | freeListMask; }
NO_SANITIZE_ADDRESS
bool isFree() { return m_size & freeListMask; }
NO_SANITIZE_ADDRESS
size_t size() const { return m_size & sizeMask; }
protected:
size_t m_size;
};
class PLATFORM_EXPORT HeapObjectHeader : public BasicObjectHeader {
public:
NO_SANITIZE_ADDRESS
explicit HeapObjectHeader(size_t encodedSize)
: BasicObjectHeader(encodedSize)
#ifndef NDEBUG
, m_magic(magic)
#endif
{ }
NO_SANITIZE_ADDRESS
HeapObjectHeader(size_t encodedSize, const GCInfo*)
: BasicObjectHeader(encodedSize)
#ifndef NDEBUG
, m_magic(magic)
#endif
{ }
inline void checkHeader() const;
inline bool isMarked() const;
inline void mark();
inline void unmark();
inline Address payload();
inline size_t payloadSize();
inline Address payloadEnd();
inline void setDebugMark();
inline void clearDebugMark();
inline bool hasDebugMark() const;
void zapMagic();
static void finalize(const GCInfo*, Address, size_t);
static HeapObjectHeader* fromPayload(const void*);
static const intptr_t magic = 0xc0de247;
static const intptr_t zappedMagic = 0xC0DEdead;
static const intptr_t zappedVTable = 0xd0d;
private:
#ifndef NDEBUG
intptr_t m_magic;
#endif
};
const size_t objectHeaderSize = sizeof(HeapObjectHeader);
class PLATFORM_EXPORT FinalizedHeapObjectHeader : public HeapObjectHeader {
public:
NO_SANITIZE_ADDRESS
FinalizedHeapObjectHeader(size_t encodedSize, const GCInfo* gcInfo)
: HeapObjectHeader(encodedSize)
, m_gcInfo(gcInfo)
{
}
inline Address payload();
inline size_t payloadSize();
NO_SANITIZE_ADDRESS
const GCInfo* gcInfo() { return m_gcInfo; }
NO_SANITIZE_ADDRESS
TraceCallback traceCallback() { return m_gcInfo->m_trace; }
void finalize();
NO_SANITIZE_ADDRESS
inline bool hasFinalizer() { return m_gcInfo->hasFinalizer(); }
static FinalizedHeapObjectHeader* fromPayload(const void*);
private:
const GCInfo* m_gcInfo;
};
const size_t finalizedHeaderSize = sizeof(FinalizedHeapObjectHeader);
class FreeListEntry : public HeapObjectHeader {
public:
NO_SANITIZE_ADDRESS
explicit FreeListEntry(size_t size)
: HeapObjectHeader(freeListEncodedSize(size))
, m_next(0)
{
#if !defined(NDEBUG) && !ASAN
for (size_t i = sizeof(*this); i < size; i++)
reinterpret_cast<Address>(this)[i] = freelistZapValue;
ASSERT(size >= objectHeaderSize);
zapMagic();
#endif
}
Address address() { return reinterpret_cast<Address>(this); }
NO_SANITIZE_ADDRESS
void unlink(FreeListEntry** prevNext)
{
*prevNext = m_next;
m_next = 0;
}
NO_SANITIZE_ADDRESS
void link(FreeListEntry** prevNext)
{
m_next = *prevNext;
*prevNext = this;
}
#if defined(ADDRESS_SANITIZER)
NO_SANITIZE_ADDRESS
bool shouldAddToFreeList()
{
if ((m_asanMagic & ~asanDeferMemoryReuseMask) != asanMagic) {
m_asanMagic = asanMagic | asanDeferMemoryReuseCount;
return false;
}
if (m_asanMagic & asanDeferMemoryReuseMask)
m_asanMagic--;
return !(m_asanMagic & asanDeferMemoryReuseMask);
}
#endif
private:
FreeListEntry* m_next;
#if defined(ADDRESS_SANITIZER)
unsigned m_asanMagic;
#endif
};
template<typename Header>
class HeapPage : public BaseHeapPage {
public:
HeapPage(PageMemory*, ThreadHeap<Header>*, const GCInfo*);
void link(HeapPage**);
static void unlink(HeapPage*, HeapPage**);
bool isEmpty();
bool contains(Address addr)
{
Address blinkPageStart = roundToBlinkPageStart(address());
return blinkPageStart <= addr && (blinkPageStart + blinkPageSize) > addr;
}
HeapPage* next() { return m_next; }
Address payload()
{
return address() + sizeof(*this) + headerPadding<Header>();
}
static size_t payloadSize()
{
return (blinkPagePayloadSize() - sizeof(HeapPage) - headerPadding<Header>()) & ~allocationMask;
}
Address end() { return payload() + payloadSize(); }
void getStats(HeapStats&);
void clearMarks();
void sweep();
void clearObjectStartBitMap();
void finalize(Header*);
virtual bool checkAndMarkPointer(Visitor*, Address);
ThreadHeap<Header>* heap() { return m_heap; }
#if defined(ADDRESS_SANITIZER)
void poisonUnmarkedObjects();
#endif
protected:
void populateObjectStartBitMap();
bool isObjectStartBitMapComputed() { return m_objectStartBitMapComputed; }
TraceCallback traceCallback(Header*);
HeapPage<Header>* m_next;
ThreadHeap<Header>* m_heap;
bool m_objectStartBitMapComputed;
uint8_t m_objectStartBitMap[reservedForObjectBitMap];
friend class ThreadHeap<Header>;
};
class HeapContainsCache {
public:
HeapContainsCache();
void flush();
bool contains(Address);
bool lookup(Address, BaseHeapPage**);
void addEntry(Address, BaseHeapPage*);
private:
class Entry {
public:
Entry()
: m_address(0)
, m_containingPage(0)
{
}
Entry(Address address, BaseHeapPage* containingPage)
: m_address(address)
, m_containingPage(containingPage)
{
}
BaseHeapPage* containingPage() { return m_containingPage; }
Address address() { return m_address; }
private:
Address m_address;
BaseHeapPage* m_containingPage;
};
static const int numberOfEntriesLog2 = 12;
static const int numberOfEntries = 1 << numberOfEntriesLog2;
static size_t hash(Address);
WTF::OwnPtr<HeapContainsCache::Entry[]> m_entries;
friend class ThreadState;
};
class CallbackStack {
public:
CallbackStack(CallbackStack** first)
: m_limit(&(m_buffer[bufferSize]))
, m_current(&(m_buffer[0]))
, m_next(*first)
{
#ifndef NDEBUG
clearUnused();
#endif
*first = this;
}
~CallbackStack();
void clearUnused();
void assertIsEmpty();
class Item {
public:
Item() { }
Item(void* object, VisitorCallback callback)
: m_object(object)
, m_callback(callback)
{
}
void* object() { return m_object; }
VisitorCallback callback() { return m_callback; }
private:
void* m_object;
VisitorCallback m_callback;
};
static void init(CallbackStack** first);
static void shutdown(CallbackStack** first);
bool popAndInvokeCallback(CallbackStack** first, Visitor*);
Item* allocateEntry(CallbackStack** first)
{
if (m_current < m_limit)
return m_current++;
return (new CallbackStack(first))->allocateEntry(first);
}
private:
static const size_t bufferSize = 8000;
Item m_buffer[bufferSize];
Item* m_limit;
Item* m_current;
CallbackStack* m_next;
};
class BaseHeap {
public:
virtual ~BaseHeap() { }
virtual BaseHeapPage* heapPageFromAddress(Address) = 0;
virtual BaseHeapPage* largeHeapObjectFromAddress(Address) = 0;
virtual bool checkAndMarkLargeHeapObject(Visitor*, Address) = 0;
virtual void sweep() = 0;
virtual void assertEmpty() = 0;
virtual void clearFreeLists() = 0;
virtual void clearMarks() = 0;
#ifndef NDEBUG
virtual void getScannedStats(HeapStats&) = 0;
#endif
virtual void makeConsistentForGC() = 0;
virtual bool isConsistentForGC() = 0;
static int bucketIndexForSize(size_t);
};
template<typename Header>
class ThreadHeap : public BaseHeap {
public:
ThreadHeap(ThreadState*);
virtual ~ThreadHeap();
virtual BaseHeapPage* heapPageFromAddress(Address);
virtual BaseHeapPage* largeHeapObjectFromAddress(Address);
virtual bool checkAndMarkLargeHeapObject(Visitor*, Address);
virtual void sweep();
virtual void assertEmpty();
virtual void clearFreeLists();
virtual void clearMarks();
#ifndef NDEBUG
virtual void getScannedStats(HeapStats&);
#endif
virtual void makeConsistentForGC();
virtual bool isConsistentForGC();
ThreadState* threadState() { return m_threadState; }
HeapStats& stats() { return m_threadState->stats(); }
HeapContainsCache* heapContainsCache() { return m_threadState->heapContainsCache(); }
inline Address allocate(size_t, const GCInfo*);
void addToFreeList(Address, size_t);
void addPageToPool(HeapPage<Header>*);
inline static size_t roundedAllocationSize(size_t size)
{
return allocationSizeFromSize(size) - sizeof(Header);
}
private:
class PagePoolEntry {
public:
PagePoolEntry(PageMemory* storage, PagePoolEntry* next)
: m_storage(storage)
, m_next(next)
{ }
PageMemory* storage() { return m_storage; }
PagePoolEntry* next() { return m_next; }
private:
PageMemory* m_storage;
PagePoolEntry* m_next;
};
PLATFORM_EXPORT Address outOfLineAllocate(size_t, const GCInfo*);
static size_t allocationSizeFromSize(size_t);
void addPageToHeap(const GCInfo*);
PLATFORM_EXPORT Address allocateLargeObject(size_t, const GCInfo*);
Address currentAllocationPoint() const { return m_currentAllocationPoint; }
size_t remainingAllocationSize() const { return m_remainingAllocationSize; }
bool ownsNonEmptyAllocationArea() const { return currentAllocationPoint() && remainingAllocationSize(); }
void setAllocationPoint(Address point, size_t size)
{
ASSERT(!point || heapPageFromAddress(point));
ASSERT(size <= HeapPage<Header>::payloadSize());
m_currentAllocationPoint = point;
m_remainingAllocationSize = size;
}
void ensureCurrentAllocation(size_t, const GCInfo*);
bool allocateFromFreeList(size_t);
void freeLargeObject(LargeHeapObject<Header>*, LargeHeapObject<Header>**);
void allocatePage(const GCInfo*);
PageMemory* takePageFromPool();
void clearPagePool();
void deletePages();
Address m_currentAllocationPoint;
size_t m_remainingAllocationSize;
HeapPage<Header>* m_firstPage;
LargeHeapObject<Header>* m_firstLargeHeapObject;
int m_biggestFreeListIndex;
ThreadState* m_threadState;
FreeListEntry* m_freeLists[blinkPageSizeLog2];
PagePoolEntry* m_pagePool;
};
class PLATFORM_EXPORT Heap {
public:
enum GCType {
Normal,
ForcedForTesting
};
static void init();
static void shutdown();
static BaseHeapPage* contains(Address);
static BaseHeapPage* contains(void* pointer) { return contains(reinterpret_cast<Address>(pointer)); }
static BaseHeapPage* contains(const void* pointer) { return contains(const_cast<void*>(pointer)); }
static void pushTraceCallback(void* containerObject, TraceCallback);
static void pushWeakObjectPointerCallback(void* closure, void* containerObject, WeakPointerCallback);
static void pushWeakCellPointerCallback(void** cell, WeakPointerCallback);
static bool popAndInvokeTraceCallback(Visitor*);
static bool popAndInvokeWeakPointerCallback(Visitor*);
template<typename T> static Address allocate(size_t);
template<typename T> static Address reallocate(void* previous, size_t);
static void collectGarbage(ThreadState::StackState, GCType = Normal);
static void collectAllGarbage(ThreadState::StackState, GCType = Normal);
static void prepareForGC();
static Address checkAndMarkPointer(Visitor*, Address);
static void getStats(HeapStats*);
static bool isConsistentForGC();
static void makeConsistentForGC();
static Visitor* s_markingVisitor;
static CallbackStack* s_markingStack;
static CallbackStack* s_weakCallbackStack;
};
template<ThreadAffinity Affinity>
class NoAllocationScope {
public:
NoAllocationScope() : m_active(true) { enter(); }
explicit NoAllocationScope(bool active) : m_active(active) { enter(); }
NoAllocationScope(const NoAllocationScope& other) : m_active(other.m_active) { enter(); }
NoAllocationScope& operator=(const NoAllocationScope& other)
{
release();
m_active = other.m_active;
enter();
return *this;
}
~NoAllocationScope() { release(); }
void release()
{
if (m_active) {
ThreadStateFor<Affinity>::state()->leaveNoAllocationScope();
m_active = false;
}
}
private:
void enter() const
{
if (m_active)
ThreadStateFor<Affinity>::state()->enterNoAllocationScope();
}
bool m_active;
};
template<typename T>
class GarbageCollected {
WTF_MAKE_NONCOPYABLE(GarbageCollected);
void* operator new[](size_t size);
void operator delete[](void* p);
public:
typedef T GarbageCollectedBase;
void* operator new(size_t size)
{
return Heap::allocate<T>(size);
}
void operator delete(void* p)
{
ASSERT_NOT_REACHED();
}
protected:
GarbageCollected()
{
}
};
template<typename T>
class GarbageCollectedFinalized : public GarbageCollected<T> {
WTF_MAKE_NONCOPYABLE(GarbageCollectedFinalized);
protected:
void finalizeGarbageCollectedObject()
{
static_cast<T*>(this)->~T();
}
GarbageCollectedFinalized() { }
~GarbageCollectedFinalized() { }
template<typename U> friend struct HasFinalizer;
template<typename U, bool> friend struct FinalizerTraitImpl;
};
template<typename T>
class RefCountedGarbageCollected : public GarbageCollectedFinalized<T> {
WTF_MAKE_NONCOPYABLE(RefCountedGarbageCollected);
public:
RefCountedGarbageCollected()
: m_refCount(1)
{
m_keepAlive = new Persistent<T>(static_cast<T*>(this));
}
void ref()
{
if (UNLIKELY(!m_refCount)) {
ASSERT(!m_keepAlive);
ASSERT(ThreadStateFor<ThreadingTrait<T>::Affinity>::state()->contains(reinterpret_cast<Address>(this)));
m_keepAlive = new Persistent<T>(static_cast<T*>(this));
}
++m_refCount;
}
void deref()
{
ASSERT(m_refCount > 0);
if (!--m_refCount) {
delete m_keepAlive;
m_keepAlive = 0;
}
}
bool hasOneRef()
{
return m_refCount == 1;
}
protected:
~RefCountedGarbageCollected() { }
private:
int m_refCount;
Persistent<T>* m_keepAlive;
};
template<typename T>
T* adoptRefCountedGarbageCollected(T* ptr)
{
ASSERT(ptr->hasOneRef());
ptr->deref();
WTF::adopted(ptr);
return ptr;
}
#if COMPILER_SUPPORTS(CXX_DELETED_FUNCTIONS)
#define DISALLOW_ALLOCATION() \
private: \
void* operator new(size_t) = delete; \
void* operator new(size_t, NotNullTag, void*) = delete; \
void* operator new(size_t, void*) = delete;
#define ALLOW_ONLY_INLINE_ALLOCATION() \
public: \
void* operator new(size_t, NotNullTag, void* location) { return location; } \
void* operator new(size_t, void* location) { return location; } \
private: \
void* operator new(size_t) = delete;
#define STATIC_ONLY(Type) \
private: \
Type() = delete;
#else
#define DISALLOW_ALLOCATION() \
private: \
void* operator new(size_t); \
void* operator new(size_t, NotNullTag, void*); \
void* operator new(size_t, void*)
#define ALLOW_ONLY_INLINE_ALLOCATION() \
public: \
void* operator new(size_t, NotNullTag, void* location) { return location; } \
void* operator new(size_t, void* location) { return location; } \
private: \
void* operator new(size_t);
#define STATIC_ONLY(Type) \
private: \
Type();
#endif
#if COMPILER(CLANG) && !defined(ADDRESS_SANITIZER)
#define STACK_ALLOCATED() \
private: \
__attribute__((annotate("blink_stack_allocated"))) \
void* operator new(size_t) = delete; \
void* operator new(size_t, NotNullTag, void*) = delete; \
void* operator new(size_t, void*) = delete;
#define GC_PLUGIN_IGNORE(bug) \
__attribute__((annotate("blink_gc_plugin_ignore")))
#else
#define STACK_ALLOCATED() DISALLOW_ALLOCATION()
#define GC_PLUGIN_IGNORE(bug)
#endif
NO_SANITIZE_ADDRESS
void HeapObjectHeader::checkHeader() const
{
ASSERT(m_magic == magic);
}
Address HeapObjectHeader::payload()
{
return reinterpret_cast<Address>(this) + objectHeaderSize;
}
size_t HeapObjectHeader::payloadSize()
{
return size() - objectHeaderSize;
}
Address HeapObjectHeader::payloadEnd()
{
return reinterpret_cast<Address>(this) + size();
}
NO_SANITIZE_ADDRESS
void HeapObjectHeader::mark()
{
checkHeader();
m_size |= markBitMask;
}
Address FinalizedHeapObjectHeader::payload()
{
return reinterpret_cast<Address>(this) + finalizedHeaderSize;
}
size_t FinalizedHeapObjectHeader::payloadSize()
{
return size() - finalizedHeaderSize;
}
template<typename Header>
size_t ThreadHeap<Header>::allocationSizeFromSize(size_t size)
{
RELEASE_ASSERT(size < maxHeapObjectSize);
size_t allocationSize = size + sizeof(Header);
allocationSize = (allocationSize + allocationMask) & ~allocationMask;
return allocationSize;
}
template<typename Header>
Address ThreadHeap<Header>::allocate(size_t size, const GCInfo* gcInfo)
{
size_t allocationSize = allocationSizeFromSize(size);
bool isLargeObject = allocationSize > blinkPageSize / 2;
if (isLargeObject)
return allocateLargeObject(allocationSize, gcInfo);
if (m_remainingAllocationSize < allocationSize)
return outOfLineAllocate(size, gcInfo);
Address headerAddress = m_currentAllocationPoint;
m_currentAllocationPoint += allocationSize;
m_remainingAllocationSize -= allocationSize;
Header* header = new (NotNull, headerAddress) Header(allocationSize, gcInfo);
size_t payloadSize = allocationSize - sizeof(Header);
stats().increaseObjectSpace(payloadSize);
Address result = headerAddress + sizeof(*header);
ASSERT(!(reinterpret_cast<uintptr_t>(result) & allocationMask));
ASAN_UNPOISON_MEMORY_REGION(result, payloadSize);
memset(result, 0, payloadSize);
ASSERT(heapPageFromAddress(headerAddress + allocationSize - 1));
return result;
}
template<typename T>
Address Heap::allocate(size_t size)
{
ThreadState* state = ThreadStateFor<ThreadingTrait<T>::Affinity>::state();
ASSERT(state->isAllocationAllowed());
BaseHeap* heap = state->heap(HeapTrait<T>::index);
Address addr =
static_cast<typename HeapTrait<T>::HeapType*>(heap)->allocate(size, GCInfoTrait<T>::get());
return addr;
}
template<typename T>
Address Heap::reallocate(void* previous, size_t size)
{
if (!size) {
return 0;
}
ThreadState* state = ThreadStateFor<ThreadingTrait<T>::Affinity>::state();
ASSERT(state->isAllocationAllowed());
ASSERT(HeapTrait<T>::index == GeneralHeap);
BaseHeap* heap = state->heap(HeapTrait<T>::index);
Address address = static_cast<typename HeapTrait<T>::HeapType*>(heap)->allocate(size, GCInfoTrait<T>::get());
if (!previous) {
return address;
}
FinalizedHeapObjectHeader* previousHeader = FinalizedHeapObjectHeader::fromPayload(previous);
ASSERT(!previousHeader->hasFinalizer());
ASSERT(previousHeader->gcInfo() == GCInfoTrait<T>::get());
size_t copySize = previousHeader->payloadSize();
if (copySize > size)
copySize = size;
memcpy(address, previous, copySize);
return address;
}
class HeapAllocatorQuantizer {
public:
template<typename T>
static size_t quantizedSize(size_t count)
{
RELEASE_ASSERT(count <= kMaxUnquantizedAllocation / sizeof(T));
return HeapTrait<T>::HeapType::roundedAllocationSize(count * sizeof(T));
}
static const size_t kMaxUnquantizedAllocation = maxHeapObjectSize;
};
class HeapAllocator {
public:
typedef HeapAllocatorQuantizer Quantizer;
typedef WebCore::Visitor Visitor;
static const bool isGarbageCollected = true;
template <typename Return, typename Metadata>
static Return backingMalloc(size_t size)
{
return malloc<Return, Metadata>(size);
}
template <typename Return, typename Metadata>
static Return zeroedBackingMalloc(size_t size)
{
return malloc<Return, Metadata>(size);
}
template <typename Return, typename Metadata>
static Return malloc(size_t size)
{
return reinterpret_cast<Return>(Heap::allocate<Metadata>(size));
}
static void backingFree(void* address) { }
static void free(void* address) { }
template<typename T>
static void* newArray(size_t bytes)
{
ASSERT_NOT_REACHED();
return 0;
}
static void deleteArray(void* ptr)
{
ASSERT_NOT_REACHED();
}
static void markUsingGCInfo(Visitor* visitor, const void* buffer)
{
visitor->mark(buffer, FinalizedHeapObjectHeader::fromPayload(buffer)->traceCallback());
}
static void markNoTracing(Visitor* visitor, const void* t)
{
visitor->mark(t, reinterpret_cast<TraceCallback>(0));
}
template<typename T, typename Traits>
static void trace(Visitor* visitor, T& t)
{
CollectionBackingTraceTrait<WTF::ShouldBeTraced<Traits>::value, Traits::isWeak, false, T, Traits>::mark(visitor, t);
}
template<typename T>
static bool hasDeadMember(Visitor*, const T&)
{
return false;
}
template<typename T>
static bool hasDeadMember(Visitor* visitor, const Member<T>& t)
{
ASSERT(visitor->isAlive(t));
return false;
}
template<typename T>
static bool hasDeadMember(Visitor* visitor, const WeakMember<T>& t)
{
return !visitor->isAlive(t);
}
template<typename T, typename U>
static bool hasDeadMember(Visitor* visitor, const WTF::KeyValuePair<T, U>& t)
{
return hasDeadMember(visitor, t.key) || hasDeadMember(visitor, t.value);
}
static void registerWeakMembers(Visitor* visitor, const void* closure, const void* object, WeakPointerCallback callback)
{
visitor->registerWeakMembers(closure, object, callback);
}
template<typename T>
struct ResultType {
typedef T* Type;
};
template<typename T, typename Traits>
struct VectorBackingHelper {
typedef HeapVectorBacking<T, Traits> Type;
};
template<typename Table>
struct HashTableBackingHelper {
typedef HeapHashTableBacking<Table> Type;
};
template<typename T>
struct OtherType {
typedef T* Type;
};
template<typename T>
static T& getOther(T* other)
{
return *other;
}
private:
template<typename T, size_t u, typename V> friend class WTF::Vector;
template<typename T, typename U, typename V, typename W> friend class WTF::HashSet;
template<typename T, typename U, typename V, typename W, typename X, typename Y> friend class WTF::HashMap;
};
template<
typename KeyArg,
typename MappedArg,
typename HashArg = typename DefaultHash<KeyArg>::Hash,
typename KeyTraitsArg = HashTraits<KeyArg>,
typename MappedTraitsArg = HashTraits<MappedArg> >
class HeapHashMap : public HashMap<KeyArg, MappedArg, HashArg, KeyTraitsArg, MappedTraitsArg, HeapAllocator> { };
template<
typename ValueArg,
typename HashArg = typename DefaultHash<ValueArg>::Hash,
typename TraitsArg = HashTraits<ValueArg> >
class HeapHashSet : public HashSet<ValueArg, HashArg, TraitsArg, HeapAllocator> { };
template<typename T, size_t inlineCapacity = 0>
class HeapVector : public Vector<T, inlineCapacity, HeapAllocator> {
public:
HeapVector() { }
explicit HeapVector(size_t size) : Vector<T, inlineCapacity, HeapAllocator>(size)
{
}
HeapVector(size_t size, const T& val) : Vector<T, inlineCapacity, HeapAllocator>(size, val)
{
}
template<size_t otherCapacity>
HeapVector(const HeapVector<T, otherCapacity>& other)
: Vector<T, inlineCapacity, HeapAllocator>(other)
{
}
template<typename U>
void append(const U& other)
{
Vector<T, inlineCapacity, HeapAllocator>::append(other);
}
template<typename U, size_t otherCapacity>
void appendVector(const HeapVector<U, otherCapacity>& other)
{
const Vector<U, otherCapacity, HeapAllocator>& otherVector = other;
Vector<T, inlineCapacity, HeapAllocator>::appendVector(otherVector);
}
};
template<typename T>
struct ThreadingTrait<Member<T> > {
static const ThreadAffinity Affinity = ThreadingTrait<T>::Affinity;
};
template<typename T>
struct ThreadingTrait<WeakMember<T> > {
static const ThreadAffinity Affinity = ThreadingTrait<T>::Affinity;
};
template<typename Key, typename Value, typename T, typename U, typename V>
struct ThreadingTrait<HashMap<Key, Value, T, U, V, HeapAllocator> > {
static const ThreadAffinity Affinity =
(ThreadingTrait<Key>::Affinity == MainThreadOnly)
&& (ThreadingTrait<Value>::Affinity == MainThreadOnly) ? MainThreadOnly : AnyThread;
};
template<typename First, typename Second>
struct ThreadingTrait<WTF::KeyValuePair<First, Second> > {
static const ThreadAffinity Affinity =
(ThreadingTrait<First>::Affinity == MainThreadOnly)
&& (ThreadingTrait<Second>::Affinity == MainThreadOnly) ? MainThreadOnly : AnyThread;
};
template<typename T, typename U, typename V>
struct ThreadingTrait<HashSet<T, U, V, HeapAllocator> > {
static const ThreadAffinity Affinity = ThreadingTrait<T>::Affinity;
};
template<typename T, size_t inlineCapacity>
struct ThreadingTrait<Vector<T, inlineCapacity, HeapAllocator> > {
static const ThreadAffinity Affinity = ThreadingTrait<T>::Affinity;
};
template<typename T, typename Traits>
struct ThreadingTrait<HeapVectorBacking<T, Traits> > {
static const ThreadAffinity Affinity = ThreadingTrait<T>::Affinity;
};
template<typename Table>
struct ThreadingTrait<HeapHashTableBacking<Table> > {
typedef typename Table::KeyType Key;
typedef typename Table::ValueType Value;
static const ThreadAffinity Affinity =
(ThreadingTrait<Key>::Affinity == MainThreadOnly)
&& (ThreadingTrait<Value>::Affinity == MainThreadOnly) ? MainThreadOnly : AnyThread;
};
template<typename T, typename U, typename V, typename W, typename X>
struct ThreadingTrait<HeapHashMap<T, U, V, W, X> > : public ThreadingTrait<HashMap<T, U, V, W, X, HeapAllocator> > { };
template<typename T, typename U, typename V>
struct ThreadingTrait<HeapHashSet<T, U, V> > : public ThreadingTrait<HashSet<T, U, V, HeapAllocator> > { };
template<typename T, size_t inlineCapacity>
struct ThreadingTrait<HeapVector<T, inlineCapacity> > : public ThreadingTrait<Vector<T, inlineCapacity, HeapAllocator> > { };
template<typename Key, typename Value, typename T, typename U, typename V>
struct GCInfoTrait<HashMap<Key, Value, T, U, V, HeapAllocator> > {
static const GCInfo* get() { return &info; }
static const GCInfo info;
};
template<typename Key, typename Value, typename T, typename U, typename V>
const GCInfo GCInfoTrait<HashMap<Key, Value, T, U, V, HeapAllocator> >::info = {
TraceTrait<HashMap<Key, Value, T, U, V, HeapAllocator> >::trace,
0,
false,
};
template<typename T, typename U, typename V>
struct GCInfoTrait<HashSet<T, U, V, HeapAllocator> > {
static const GCInfo* get() { return &info; }
static const GCInfo info;
};
template<typename T, typename U, typename V>
const GCInfo GCInfoTrait<HashSet<T, U, V, HeapAllocator> >::info = {
TraceTrait<HashSet<T, U, V, HeapAllocator> >::trace,
0,
false,
};
template<typename T>
struct GCInfoTrait<Vector<T, 0, HeapAllocator> > {
static const GCInfo* get() { return &info; }
static const GCInfo info;
};
template<typename T>
const GCInfo GCInfoTrait<Vector<T, 0, HeapAllocator> >::info = {
TraceTrait<Vector<T, 0, HeapAllocator> >::trace,
0,
false,
};
template<typename T, size_t inlineCapacity>
struct GCInfoTrait<Vector<T, inlineCapacity, HeapAllocator> > {
static const GCInfo* get() { return &info; }
static const GCInfo info;
};
template<typename T, size_t inlineCapacity>
struct FinalizerTrait<Vector<T, inlineCapacity, HeapAllocator> > : public FinalizerTraitImpl<Vector<T, inlineCapacity, HeapAllocator>, true> { };
template<typename T, size_t inlineCapacity>
const GCInfo GCInfoTrait<Vector<T, inlineCapacity, HeapAllocator> >::info = {
TraceTrait<Vector<T, inlineCapacity, HeapAllocator> >::trace,
FinalizerTrait<Vector<T, inlineCapacity, HeapAllocator> >::finalize,
inlineCapacity && VectorTraits<T>::needsDestruction,
};
template<typename T, typename Traits>
struct GCInfoTrait<HeapVectorBacking<T, Traits> > {
static const GCInfo* get() { return &info; }
static const GCInfo info;
};
template<typename T, typename Traits>
const GCInfo GCInfoTrait<HeapVectorBacking<T, Traits> >::info = {
TraceTrait<HeapVectorBacking<T, Traits> >::trace,
FinalizerTrait<HeapVectorBacking<T, Traits> >::finalize,
Traits::needsDestruction,
};
template<typename Table>
struct GCInfoTrait<HeapHashTableBacking<Table> > {
static const GCInfo* get() { return &info; }
static const GCInfo info;
};
template<typename Table>
const GCInfo GCInfoTrait<HeapHashTableBacking<Table> >::info = {
TraceTrait<HeapHashTableBacking<Table> >::trace,
HeapHashTableBacking<Table>::finalize,
Table::ValueTraits::needsDestruction,
};
template<bool markWeakMembersStrongly, typename T, typename Traits>
struct BaseVisitVectorBackingTrait {
static void mark(WebCore::Visitor* visitor, void* self)
{
COMPILE_ASSERT(!WTF::ShouldBeTraced<Traits>::value || sizeof(T) > allocationGranularity || Traits::canInitializeWithMemset, HeapOverallocationCanCauseSpuriousVisits);
T* array = reinterpret_cast<T*>(self);
WebCore::FinalizedHeapObjectHeader* header = WebCore::FinalizedHeapObjectHeader::fromPayload(self);
size_t length = header->payloadSize() / sizeof(T);
for (size_t i = 0; i < length; i++)
CollectionBackingTraceTrait<WTF::ShouldBeTraced<Traits>::value, Traits::isWeak, markWeakMembersStrongly, T, Traits>::mark(visitor, array[i]);
}
};
template<bool markWeakMembersStrongly, typename Table>
struct BaseVisitHashTableBackingTrait {
typedef typename Table::ValueType Value;
typedef typename Table::ValueTraits Traits;
static void mark(WebCore::Visitor* visitor, void* self)
{
Value* array = reinterpret_cast<Value*>(self);
WebCore::FinalizedHeapObjectHeader* header = WebCore::FinalizedHeapObjectHeader::fromPayload(self);
size_t length = header->payloadSize() / sizeof(Value);
for (size_t i = 0; i < length; i++) {
if (!WTF::HashTableHelper<Value, typename Table::ExtractorType, typename Table::KeyTraitsType>::isEmptyOrDeletedBucket(array[i]))
CollectionBackingTraceTrait<WTF::ShouldBeTraced<Traits>::value, Traits::isWeak, markWeakMembersStrongly, Value, Traits>::mark(visitor, array[i]);
}
}
};
template<bool markWeakMembersStrongly, typename Key, typename Value, typename Traits>
struct BaseVisitKeyValuePairTrait {
static void mark(WebCore::Visitor* visitor, WTF::KeyValuePair<Key, Value>& self)
{
ASSERT(WTF::ShouldBeTraced<Traits>::value || (Traits::isWeak && markWeakMembersStrongly));
CollectionBackingTraceTrait<WTF::ShouldBeTraced<typename Traits::KeyTraits>::value, Traits::KeyTraits::isWeak, markWeakMembersStrongly, Key, typename Traits::KeyTraits>::mark(visitor, self.key);
CollectionBackingTraceTrait<WTF::ShouldBeTraced<typename Traits::ValueTraits>::value, Traits::ValueTraits::isWeak, markWeakMembersStrongly, Value, typename Traits::ValueTraits>::mark(visitor, self.value);
}
};
template<bool markWeakMembersStrongly, typename T, typename U>
struct CollectionBackingTraceTrait<false, false, markWeakMembersStrongly, T, U> {
static void mark(Visitor*, const T&) { }
static void mark(Visitor*, const void*) { }
};
template<typename T, typename U>
struct CollectionBackingTraceTrait<false, true, false, T, U> {
static void mark(Visitor*, const T&) { }
static void mark(Visitor*, const void*) { }
};
template<typename T, typename Traits>
struct CollectionBackingTraceTrait<false, true, true, HeapVectorBacking<T, Traits>, void> : public BaseVisitVectorBackingTrait<true, T, Traits> {
};
template<bool isWeak, bool markWeakMembersStrongly, typename T, typename Traits>
struct CollectionBackingTraceTrait<true, isWeak, markWeakMembersStrongly, HeapVectorBacking<T, Traits>, void> : public BaseVisitVectorBackingTrait<markWeakMembersStrongly, T, Traits> {
};
template<typename Table>
struct CollectionBackingTraceTrait<false, true, true, HeapHashTableBacking<Table>, void> : public BaseVisitHashTableBackingTrait<true, Table> {
};
template<bool isWeak, bool markWeakMembersStrongly, typename Table>
struct CollectionBackingTraceTrait<true, isWeak, markWeakMembersStrongly, HeapHashTableBacking<Table>, void> : public BaseVisitHashTableBackingTrait<markWeakMembersStrongly, Table> {
};
template<typename Key, typename Value, typename Traits>
struct CollectionBackingTraceTrait<false, true, true, WTF::KeyValuePair<Key, Value>, Traits> : public BaseVisitKeyValuePairTrait<true, Key, Value, Traits> {
};
template<bool isWeak, bool markWeakMembersStrongly, typename Key, typename Value, typename Traits>
struct CollectionBackingTraceTrait<true, isWeak, markWeakMembersStrongly, WTF::KeyValuePair<Key, Value>, Traits> : public BaseVisitKeyValuePairTrait<markWeakMembersStrongly, Key, Value, Traits> {
};
template<bool markWeakMembersStrongly, typename T, typename Traits>
struct CollectionBackingTraceTrait<true, false, markWeakMembersStrongly, Member<T>, Traits> {
static void mark(WebCore::Visitor* visitor, Member<T> self)
{
visitor->mark(self.get());
}
};
template<typename T, typename Traits>
struct CollectionBackingTraceTrait<false, true, true, WeakMember<T>, Traits> {
static void mark(WebCore::Visitor* visitor, WeakMember<T> self)
{
visitor->mark(self.get());
}
};
template<bool isWeak, bool markWeakMembersStrongly, typename T, typename Traits>
struct CollectionBackingTraceTrait<true, isWeak, markWeakMembersStrongly, T, Traits> {
static void mark(WebCore::Visitor* visitor, T& t)
{
TraceTrait<T>::trace(visitor, &t);
}
};
template<typename T, typename Traits>
struct TraceTrait<HeapVectorBacking<T, Traits> > {
typedef HeapVectorBacking<T, Traits> Backing;
static void trace(WebCore::Visitor* visitor, void* self)
{
COMPILE_ASSERT(!Traits::isWeak, WeDontSupportWeaknessInHeapVectors);
if (WTF::ShouldBeTraced<Traits>::value)
CollectionBackingTraceTrait<WTF::ShouldBeTraced<Traits>::value, false, false, HeapVectorBacking<T, Traits>, void>::mark(visitor, self);
}
static void mark(Visitor* visitor, const Backing* backing)
{
visitor->mark(backing, &trace);
}
static void checkGCInfo(Visitor* visitor, const Backing* backing)
{
#ifndef NDEBUG
visitor->checkGCInfo(const_cast<Backing*>(backing), GCInfoTrait<Backing>::get());
#endif
}
};
template<typename Table>
struct TraceTrait<HeapHashTableBacking<Table> > {
typedef HeapHashTableBacking<Table> Backing;
typedef typename Table::ValueTraits Traits;
static void trace(WebCore::Visitor* visitor, void* self)
{
if (WTF::ShouldBeTraced<Traits>::value || Traits::isWeak)
CollectionBackingTraceTrait<WTF::ShouldBeTraced<Traits>::value, Traits::isWeak, true, Backing, void>::mark(visitor, self);
}
static void mark(Visitor* visitor, const Backing* backing)
{
if (WTF::ShouldBeTraced<Traits>::value || Traits::isWeak)
visitor->mark(backing, &trace);
else
visitor->mark(backing, 0);
}
static void checkGCInfo(Visitor* visitor, const Backing* backing)
{
#ifndef NDEBUG
visitor->checkGCInfo(const_cast<Backing*>(backing), GCInfoTrait<Backing>::get());
#endif
}
};
template<typename Table>
void HeapHashTableBacking<Table>::finalize(void* pointer)
{
typedef typename Table::ValueType Value;
ASSERT(Table::ValueTraits::needsDestruction);
FinalizedHeapObjectHeader* header = FinalizedHeapObjectHeader::fromPayload(pointer);
size_t length = header->payloadSize() / sizeof(Value);
Value* table = reinterpret_cast<Value*>(pointer);
for (unsigned i = 0; i < length; i++) {
if (!Table::isEmptyOrDeletedBucket(table[i]))
table[i].~Value();
}
}
template<typename T, typename U, typename V, typename W, typename X>
struct GCInfoTrait<HeapHashMap<T, U, V, W, X> > : public GCInfoTrait<HashMap<T, U, V, W, X, HeapAllocator> > { };
template<typename T, typename U, typename V>
struct GCInfoTrait<HeapHashSet<T, U, V> > : public GCInfoTrait<HashSet<T, U, V, HeapAllocator> > { };
template<typename T, size_t inlineCapacity>
struct GCInfoTrait<HeapVector<T, inlineCapacity> > : public GCInfoTrait<Vector<T, inlineCapacity, HeapAllocator> > { };
template<typename T>
struct IfWeakMember;
template<typename T>
struct IfWeakMember {
template<typename U>
static bool isDead(Visitor*, const U&) { return false; }
};
template<typename T>
struct IfWeakMember<WeakMember<T> > {
static bool isDead(Visitor* visitor, const WeakMember<T>& t) { return !visitor->isAlive(t.get()); }
};
#if COMPILER(CLANG)
template<> void ThreadHeap<FinalizedHeapObjectHeader>::addPageToHeap(const GCInfo*);
template<> void ThreadHeap<HeapObjectHeader>::addPageToHeap(const GCInfo*);
extern template class PLATFORM_EXPORT ThreadHeap<FinalizedHeapObjectHeader>;
extern template class PLATFORM_EXPORT ThreadHeap<HeapObjectHeader>;
#endif
}
#endif