diff options
Diffstat (limited to 'Source/JavaScriptCore/heap/Heap.cpp')
| -rw-r--r-- | Source/JavaScriptCore/heap/Heap.cpp | 1533 |
1 files changed, 495 insertions, 1038 deletions
diff --git a/Source/JavaScriptCore/heap/Heap.cpp b/Source/JavaScriptCore/heap/Heap.cpp index 10eaa0205..26ec23980 100644 --- a/Source/JavaScriptCore/heap/Heap.cpp +++ b/Source/JavaScriptCore/heap/Heap.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2003-2009, 2011, 2013-2015 Apple Inc. All rights reserved. + * Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2011, 2013 Apple Inc. All rights reserved. * Copyright (C) 2007 Eric Seidel <eric@webkit.org> * * This library is free software; you can redistribute it and/or @@ -27,38 +27,33 @@ #include "CopiedSpaceInlines.h" #include "CopyVisitorInlines.h" #include "DFGWorklist.h" -#include "EdenGCActivityCallback.h" -#include "FullGCActivityCallback.h" +#include "DelayedReleaseScope.h" #include "GCActivityCallback.h" #include "GCIncomingRefCountedSetInlines.h" -#include "HeapHelperPool.h" #include "HeapIterationScope.h" #include "HeapRootVisitor.h" #include "HeapStatistics.h" -#include "HeapVerifier.h" #include "IncrementalSweeper.h" #include "Interpreter.h" -#include "JSCInlines.h" #include "JSGlobalObject.h" #include "JSLock.h" -#include "JSVirtualMachineInternal.h" -#include "SamplingProfiler.h" +#include "JSONObject.h" +#include "Operations.h" +#include "RecursiveAllocationScope.h" #include "Tracing.h" -#include "TypeProfilerLog.h" #include "UnlinkedCodeBlock.h" #include "VM.h" #include "WeakSetInlines.h" #include <algorithm> -#include <wtf/CurrentTime.h> -#include <wtf/ParallelVectorIterator.h> -#include <wtf/ProcessID.h> #include <wtf/RAMSize.h> +#include <wtf/CurrentTime.h> using namespace std; +using namespace JSC; namespace JSC { -namespace { +namespace { static const size_t largeHeapSize = 32 * MB; // About 1.5X the average webpage. static const size_t smallHeapSize = 1 * MB; // Matches the FastMalloc per-thread cache. @@ -80,129 +75,82 @@ static type name arguments; struct GCTimer { GCTimer(const char* name) - : name(name) + : m_time(0) + , m_min(100000000) + , m_max(0) + , m_count(0) + , m_name(name) { } ~GCTimer() { - logData(allCollectionData, "(All)"); - logData(edenCollectionData, "(Eden)"); - logData(fullCollectionData, "(Full)"); - } - - struct TimeRecord { - TimeRecord() - : time(0) - , min(std::numeric_limits<double>::infinity()) - , max(0) - , count(0) - { - } - - double time; - double min; - double max; - size_t count; - }; - - void logData(const TimeRecord& data, const char* extra) - { - dataLogF("[%d] %s (Parent: %s) %s: %.2lfms (avg. %.2lf, min. %.2lf, max. %.2lf, count %lu)\n", - getCurrentProcessID(), - name, - parent ? parent->name : "nullptr", - extra, - data.time * 1000, - data.time * 1000 / data.count, - data.min * 1000, - data.max * 1000, - data.count); - } - - void updateData(TimeRecord& data, double duration) - { - if (duration < data.min) - data.min = duration; - if (duration > data.max) - data.max = duration; - data.count++; - data.time += duration; - } - - void didFinishPhase(HeapOperation collectionType, double duration) - { - TimeRecord& data = collectionType == EdenCollection ? edenCollectionData : fullCollectionData; - updateData(data, duration); - updateData(allCollectionData, duration); + dataLogF("%s: %.2lfms (avg. %.2lf, min. %.2lf, max. %.2lf)\n", m_name, m_time * 1000, m_time * 1000 / m_count, m_min*1000, m_max*1000); } - - static GCTimer* s_currentGlobalTimer; - - TimeRecord allCollectionData; - TimeRecord fullCollectionData; - TimeRecord edenCollectionData; - const char* name; - GCTimer* parent { nullptr }; + double m_time; + double m_min; + double m_max; + size_t m_count; + const char* m_name; }; -GCTimer* GCTimer::s_currentGlobalTimer = nullptr; - struct GCTimerScope { - GCTimerScope(GCTimer& timer, HeapOperation collectionType) - : timer(timer) - , start(WTF::monotonicallyIncreasingTime()) - , collectionType(collectionType) + GCTimerScope(GCTimer* timer) + : m_timer(timer) + , m_start(WTF::monotonicallyIncreasingTime()) { - timer.parent = GCTimer::s_currentGlobalTimer; - GCTimer::s_currentGlobalTimer = &timer; } ~GCTimerScope() { - double delta = WTF::monotonicallyIncreasingTime() - start; - timer.didFinishPhase(collectionType, delta); - GCTimer::s_currentGlobalTimer = timer.parent; + double delta = WTF::monotonicallyIncreasingTime() - m_start; + if (delta < m_timer->m_min) + m_timer->m_min = delta; + if (delta > m_timer->m_max) + m_timer->m_max = delta; + m_timer->m_count++; + m_timer->m_time += delta; } - GCTimer& timer; - double start; - HeapOperation collectionType; + GCTimer* m_timer; + double m_start; }; struct GCCounter { GCCounter(const char* name) - : name(name) - , count(0) - , total(0) - , min(10000000) - , max(0) + : m_name(name) + , m_count(0) + , m_total(0) + , m_min(10000000) + , m_max(0) { } - void add(size_t amount) + void count(size_t amount) { - count++; - total += amount; - if (amount < min) - min = amount; - if (amount > max) - max = amount; + m_count++; + m_total += amount; + if (amount < m_min) + m_min = amount; + if (amount > m_max) + m_max = amount; } ~GCCounter() { - dataLogF("[%d] %s: %zu values (avg. %zu, min. %zu, max. %zu)\n", getCurrentProcessID(), name, total, total / count, min, max); + dataLogF("%s: %zu values (avg. %zu, min. %zu, max. %zu)\n", m_name, m_total, m_total / m_count, m_min, m_max); } - const char* name; - size_t count; - size_t total; - size_t min; - size_t max; + const char* m_name; + size_t m_count; + size_t m_total; + size_t m_min; + size_t m_max; }; -#define GCPHASE(name) DEFINE_GC_LOGGING_GLOBAL(GCTimer, name##Timer, (#name)); GCTimerScope name##TimerScope(name##Timer, m_operationInProgress) -#define GCCOUNTER(name, value) do { DEFINE_GC_LOGGING_GLOBAL(GCCounter, name##Counter, (#name)); name##Counter.add(value); } while (false) +#define GCPHASE(name) DEFINE_GC_LOGGING_GLOBAL(GCTimer, name##Timer, (#name)); GCTimerScope name##TimerScope(&name##Timer) +#define COND_GCPHASE(cond, name1, name2) DEFINE_GC_LOGGING_GLOBAL(GCTimer, name1##Timer, (#name1)); DEFINE_GC_LOGGING_GLOBAL(GCTimer, name2##Timer, (#name2)); GCTimerScope name1##CondTimerScope(cond ? &name1##Timer : &name2##Timer) +#define GCCOUNTER(name, value) do { DEFINE_GC_LOGGING_GLOBAL(GCCounter, name##Counter, (#name)); name##Counter.count(value); } while (false) #else #define GCPHASE(name) do { } while (false) +#define COND_GCPHASE(cond, name1, name2) do { } while (false) #define GCCOUNTER(name, value) do { } while (false) #endif @@ -230,7 +178,7 @@ static inline bool isValidSharedInstanceThreadState(VM* vm) static inline bool isValidThreadState(VM* vm) { - if (vm->atomicStringTable() != wtfThreadData().atomicStringTable()) + if (vm->identifierTable != wtfThreadData().currentIdentifierTable()) return false; if (vm->isSharedInstance() && !isValidSharedInstanceThreadState(vm)) @@ -240,17 +188,12 @@ static inline bool isValidThreadState(VM* vm) } struct MarkObject : public MarkedBlock::VoidFunctor { - inline void visit(JSCell* cell) + void operator()(JSCell* cell) { if (cell->isZapped()) return; Heap::heap(cell)->setMarked(cell); } - IterationStatus operator()(JSCell* cell) - { - visit(cell); - return IterationStatus::Continue; - } }; struct Count : public MarkedBlock::CountFunctor { @@ -258,36 +201,30 @@ struct Count : public MarkedBlock::CountFunctor { }; struct CountIfGlobalObject : MarkedBlock::CountFunctor { - inline void visit(JSCell* cell) - { + void operator()(JSCell* cell) { if (!cell->isObject()) return; if (!asObject(cell)->isGlobalObject()) return; count(1); } - IterationStatus operator()(JSCell* cell) - { - visit(cell); - return IterationStatus::Continue; - } }; class RecordType { public: - typedef std::unique_ptr<TypeCountSet> ReturnType; + typedef PassOwnPtr<TypeCountSet> ReturnType; RecordType(); - IterationStatus operator()(JSCell*); + void operator()(JSCell*); ReturnType returnValue(); private: const char* typeName(JSCell*); - std::unique_ptr<TypeCountSet> m_typeCountSet; + OwnPtr<TypeCountSet> m_typeCountSet; }; inline RecordType::RecordType() - : m_typeCountSet(std::make_unique<TypeCountSet>()) + : m_typeCountSet(adoptPtr(new TypeCountSet)) { } @@ -299,72 +236,54 @@ inline const char* RecordType::typeName(JSCell* cell) return info->className; } -inline IterationStatus RecordType::operator()(JSCell* cell) +inline void RecordType::operator()(JSCell* cell) { m_typeCountSet->add(typeName(cell)); - return IterationStatus::Continue; } -inline std::unique_ptr<TypeCountSet> RecordType::returnValue() +inline PassOwnPtr<TypeCountSet> RecordType::returnValue() { - return WTFMove(m_typeCountSet); + return m_typeCountSet.release(); } } // anonymous namespace Heap::Heap(VM* vm, HeapType heapType) : m_heapType(heapType) - , m_ramSize(Options::forceRAMSize() ? Options::forceRAMSize() : ramSize()) + , m_ramSize(ramSize()) , m_minBytesPerCycle(minHeapSize(m_heapType, m_ramSize)) , m_sizeAfterLastCollect(0) - , m_sizeAfterLastFullCollect(0) - , m_sizeBeforeLastFullCollect(0) - , m_sizeAfterLastEdenCollect(0) - , m_sizeBeforeLastEdenCollect(0) , m_bytesAllocatedThisCycle(0) - , m_bytesAbandonedSinceLastFullCollect(0) + , m_bytesAbandonedThisCycle(0) , m_maxEdenSize(m_minBytesPerCycle) , m_maxHeapSize(m_minBytesPerCycle) , m_shouldDoFullCollection(false) , m_totalBytesVisited(0) , m_totalBytesCopied(0) , m_operationInProgress(NoOperation) + , m_blockAllocator() , m_objectSpace(this) , m_storageSpace(this) - , m_extraMemorySize(0) - , m_deprecatedExtraMemorySize(0) + , m_extraMemoryUsage(0) , m_machineThreads(this) - , m_slotVisitor(*this) + , m_sharedData(vm) + , m_slotVisitor(m_sharedData) + , m_copyVisitor(m_sharedData) , m_handleSet(vm) , m_isSafeToCollect(false) , m_writeBarrierBuffer(256) , m_vm(vm) - // We seed with 10ms so that GCActivityCallback::didAllocate doesn't continuously - // schedule the timer if we've never done a collection. - , m_lastFullGCLength(0.01) - , m_lastEdenGCLength(0.01) - , m_fullActivityCallback(GCActivityCallback::createFullTimer(this)) - , m_edenActivityCallback(GCActivityCallback::createEdenTimer(this)) -#if USE(CF) - , m_sweeper(std::make_unique<IncrementalSweeper>(this, CFRunLoopGetCurrent())) -#else - , m_sweeper(std::make_unique<IncrementalSweeper>(this)) -#endif + , m_lastGCLength(0) + , m_lastCodeDiscardTime(WTF::monotonicallyIncreasingTime()) + , m_activityCallback(DefaultGCActivityCallback::create(this)) + , m_sweeper(IncrementalSweeper::create(this)) , m_deferralDepth(0) -#if USE(CF) - , m_delayedReleaseRecursionCount(0) -#endif - , m_helperClient(&heapHelperPool()) { m_storageSpace.init(); - if (Options::verifyHeap()) - m_verifier = std::make_unique<HeapVerifier>(this, Options::numberOfGCCyclesToRecordForVerification()); } Heap::~Heap() { - for (WeakBlock* block : m_logicallyEmptyWeakBlocks) - WeakBlock::destroy(*this, block); } bool Heap::isPagedOut(double deadline) @@ -379,58 +298,31 @@ void Heap::lastChanceToFinalize() RELEASE_ASSERT(!m_vm->entryScope); RELEASE_ASSERT(m_operationInProgress == NoOperation); - m_codeBlocks.lastChanceToFinalize(); m_objectSpace.lastChanceToFinalize(); - releaseDelayedReleasedObjects(); - - sweepAllLogicallyEmptyWeakBlocks(); } -void Heap::releaseDelayedReleasedObjects() +void Heap::reportExtraMemoryCostSlowCase(size_t cost) { -#if USE(CF) - // We need to guard against the case that releasing an object can create more objects due to the - // release calling into JS. When those JS call(s) exit and all locks are being dropped we end up - // back here and could try to recursively release objects. We guard that with a recursive entry - // count. Only the initial call will release objects, recursive calls simple return and let the - // the initial call to the function take care of any objects created during release time. - // This also means that we need to loop until there are no objects in m_delayedReleaseObjects - // and use a temp Vector for the actual releasing. - if (!m_delayedReleaseRecursionCount++) { - while (!m_delayedReleaseObjects.isEmpty()) { - ASSERT(m_vm->currentThreadIsHoldingAPILock()); - - Vector<RetainPtr<CFTypeRef>> objectsToRelease = WTFMove(m_delayedReleaseObjects); - - { - // We need to drop locks before calling out to arbitrary code. - JSLock::DropAllLocks dropAllLocks(m_vm); + // Our frequency of garbage collection tries to balance memory use against speed + // by collecting based on the number of newly created values. However, for values + // that hold on to a great deal of memory that's not in the form of other JS values, + // that is not good enough - in some cases a lot of those objects can pile up and + // use crazy amounts of memory without a GC happening. So we track these extra + // memory costs. Only unusually large objects are noted, and we only keep track + // of this extra cost until the next GC. In garbage collected languages, most values + // are either very short lived temporaries, or have extremely long lifetimes. So + // if a large value survives one garbage collection, there is not much point to + // collecting more frequently as long as it stays alive. - objectsToRelease.clear(); - } - } - } - m_delayedReleaseRecursionCount--; -#endif -} - -void Heap::reportExtraMemoryAllocatedSlowCase(size_t size) -{ - didAllocate(size); + didAllocate(cost); collectIfNecessaryOrDefer(); } -void Heap::deprecatedReportExtraMemorySlowCase(size_t size) -{ - m_deprecatedExtraMemorySize += size; - reportExtraMemoryAllocatedSlowCase(size); -} - void Heap::reportAbandonedObjectGraph() { // Our clients don't know exactly how much memory they // are abandoning so we just guess for them. - double abandonedBytes = 0.1 * m_sizeAfterLastCollect; + double abandonedBytes = 0.10 * m_sizeAfterLastCollect; // We want to accelerate the next collection. Because memory has just // been abandoned, the next collection has the potential to @@ -441,11 +333,9 @@ void Heap::reportAbandonedObjectGraph() void Heap::didAbandon(size_t bytes) { - if (m_fullActivityCallback) { - m_fullActivityCallback->didAllocate( - m_sizeAfterLastCollect - m_sizeAfterLastFullCollect + m_bytesAllocatedThisCycle + m_bytesAbandonedSinceLastFullCollect); - } - m_bytesAbandonedSinceLastFullCollect += bytes; + if (m_activityCallback) + m_activityCallback->didAllocate(m_bytesAllocatedThisCycle + m_bytesAbandonedThisCycle); + m_bytesAbandonedThisCycle += bytes; } void Heap::protect(JSValue k) @@ -478,6 +368,40 @@ void Heap::addReference(JSCell* cell, ArrayBuffer* buffer) } } +void Heap::markProtectedObjects(HeapRootVisitor& heapRootVisitor) +{ + ProtectCountSet::iterator end = m_protectedValues.end(); + for (ProtectCountSet::iterator it = m_protectedValues.begin(); it != end; ++it) + heapRootVisitor.visit(&it->key); +} + +void Heap::pushTempSortVector(Vector<ValueStringPair, 0, UnsafeVectorOverflow>* tempVector) +{ + m_tempSortingVectors.append(tempVector); +} + +void Heap::popTempSortVector(Vector<ValueStringPair, 0, UnsafeVectorOverflow>* tempVector) +{ + ASSERT_UNUSED(tempVector, tempVector == m_tempSortingVectors.last()); + m_tempSortingVectors.removeLast(); +} + +void Heap::markTempSortVectors(HeapRootVisitor& heapRootVisitor) +{ + typedef Vector<Vector<ValueStringPair, 0, UnsafeVectorOverflow>* > VectorOfValueStringVectors; + + VectorOfValueStringVectors::iterator end = m_tempSortingVectors.end(); + for (VectorOfValueStringVectors::iterator it = m_tempSortingVectors.begin(); it != end; ++it) { + Vector<ValueStringPair, 0, UnsafeVectorOverflow>* tempSortingVector = *it; + + Vector<ValueStringPair>::iterator vectorEnd = tempSortingVector->end(); + for (Vector<ValueStringPair>::iterator vectorIt = tempSortingVector->begin(); vectorIt != vectorEnd; ++vectorIt) { + if (vectorIt->first) + heapRootVisitor.visit(&vectorIt->first); + } + } +} + void Heap::harvestWeakReferences() { m_slotVisitor.harvestWeakReferences(); @@ -485,7 +409,6 @@ void Heap::harvestWeakReferences() void Heap::finalizeUnconditionalFinalizers() { - GCPHASE(FinalizeUnconditionalFinalizers); m_slotVisitor.finalizeUnconditionalFinalizers(); } @@ -504,413 +427,231 @@ void Heap::didFinishIterating() m_objectSpace.didFinishIterating(); } -void Heap::completeAllDFGPlans() +void Heap::getConservativeRegisterRoots(HashSet<JSCell*>& roots) { -#if ENABLE(DFG_JIT) - DFG::completeAllPlansForVM(*m_vm); -#endif + ASSERT(isValidThreadState(m_vm)); + ConservativeRoots stackRoots(&m_objectSpace.blocks(), &m_storageSpace); + stack().gatherConservativeRoots(stackRoots); + size_t stackRootCount = stackRoots.size(); + JSCell** registerRoots = stackRoots.roots(); + for (size_t i = 0; i < stackRootCount; i++) { + setMarked(registerRoots[i]); + roots.add(registerRoots[i]); + } } -void Heap::markRoots(double gcStartTime, void* stackOrigin, void* stackTop, MachineThreads::RegisterState& calleeSavedRegisters) +void Heap::markRoots() { - SamplingRegion samplingRegion("Garbage Collection: Marking"); + SamplingRegion samplingRegion("Garbage Collection: Tracing"); GCPHASE(MarkRoots); ASSERT(isValidThreadState(m_vm)); +#if ENABLE(OBJECT_MARK_LOGGING) + double gcStartTime = WTF::monotonicallyIncreasingTime(); +#endif + + void* dummy; + // We gather conservative roots before clearing mark bits because conservative // gathering uses the mark bits to determine whether a reference is valid. - ConservativeRoots conservativeRoots(&m_objectSpace.blocks(), &m_storageSpace); - gatherStackRoots(conservativeRoots, stackOrigin, stackTop, calleeSavedRegisters); - gatherJSStackRoots(conservativeRoots); - gatherScratchBufferRoots(conservativeRoots); + ConservativeRoots machineThreadRoots(&m_objectSpace.blocks(), &m_storageSpace); + m_jitStubRoutines.clearMarks(); + { + GCPHASE(GatherConservativeRoots); + m_machineThreads.gatherConservativeRoots(machineThreadRoots, &dummy); + } -#if ENABLE(DFG_JIT) - DFG::rememberCodeBlocks(*m_vm); -#endif + ConservativeRoots stackRoots(&m_objectSpace.blocks(), &m_storageSpace); + m_codeBlocks.clearMarks(); + { + GCPHASE(GatherStackRoots); + stack().gatherConservativeRoots(stackRoots, m_jitStubRoutines, m_codeBlocks); + } -#if ENABLE(SAMPLING_PROFILER) - if (SamplingProfiler* samplingProfiler = m_vm->samplingProfiler()) { - // Note that we need to own the lock from now until we're done - // marking the SamplingProfiler's data because once we verify the - // SamplingProfiler's stack traces, we don't want it to accumulate - // more stack traces before we get the chance to mark it. - // This lock is released inside visitSamplingProfiler(). - samplingProfiler->getLock().lock(); - samplingProfiler->processUnverifiedStackTraces(); +#if ENABLE(DFG_JIT) + ConservativeRoots scratchBufferRoots(&m_objectSpace.blocks(), &m_storageSpace); + { + GCPHASE(GatherScratchBufferRoots); + m_vm->gatherConservativeRoots(scratchBufferRoots); } -#endif // ENABLE(SAMPLING_PROFILER) +#endif - if (m_operationInProgress == FullCollection) { - m_opaqueRoots.clear(); - m_slotVisitor.clearMarkStack(); + { + GCPHASE(ClearLivenessData); + m_objectSpace.clearNewlyAllocated(); + m_objectSpace.clearMarks(); } - clearLivenessData(); + m_sharedData.didStartMarking(); + SlotVisitor& visitor = m_slotVisitor; + visitor.setup(); + HeapRootVisitor heapRootVisitor(visitor); - m_parallelMarkersShouldExit = false; +#if ENABLE(GGC) + Vector<const JSCell*> rememberedSet(m_slotVisitor.markStack().size()); + m_slotVisitor.markStack().fillVector(rememberedSet); +#endif - m_helperClient.setFunction( - [this] () { - SlotVisitor* slotVisitor; - { - LockHolder locker(m_parallelSlotVisitorLock); - if (m_availableParallelSlotVisitors.isEmpty()) { - std::unique_ptr<SlotVisitor> newVisitor = - std::make_unique<SlotVisitor>(*this); - slotVisitor = newVisitor.get(); - m_parallelSlotVisitors.append(WTFMove(newVisitor)); - } else - slotVisitor = m_availableParallelSlotVisitors.takeLast(); - } + { + ParallelModeEnabler enabler(visitor); - WTF::registerGCThread(); + m_vm->smallStrings.visitStrongReferences(visitor); - { - ParallelModeEnabler parallelModeEnabler(*slotVisitor); - slotVisitor->didStartMarking(); - slotVisitor->drainFromShared(SlotVisitor::SlaveDrain); - } + { + GCPHASE(VisitMachineRoots); + MARK_LOG_ROOT(visitor, "C++ Stack"); + visitor.append(machineThreadRoots); + visitor.donateAndDrain(); + } + { + GCPHASE(VisitStackRoots); + MARK_LOG_ROOT(visitor, "Stack"); + visitor.append(stackRoots); + visitor.donateAndDrain(); + } +#if ENABLE(DFG_JIT) + { + GCPHASE(VisitScratchBufferRoots); + MARK_LOG_ROOT(visitor, "Scratch Buffers"); + visitor.append(scratchBufferRoots); + visitor.donateAndDrain(); + } +#endif + { + GCPHASE(VisitProtectedObjects); + MARK_LOG_ROOT(visitor, "Protected Objects"); + markProtectedObjects(heapRootVisitor); + visitor.donateAndDrain(); + } + { + GCPHASE(VisitTempSortVectors); + MARK_LOG_ROOT(visitor, "Temp Sort Vectors"); + markTempSortVectors(heapRootVisitor); + visitor.donateAndDrain(); + } - { - LockHolder locker(m_parallelSlotVisitorLock); - m_availableParallelSlotVisitors.append(slotVisitor); + { + GCPHASE(MarkingArgumentBuffers); + if (m_markListSet && m_markListSet->size()) { + MARK_LOG_ROOT(visitor, "Argument Buffers"); + MarkedArgumentBuffer::markLists(heapRootVisitor, *m_markListSet); + visitor.donateAndDrain(); } - }); - - m_slotVisitor.didStartMarking(); + } + if (m_vm->exception()) { + GCPHASE(MarkingException); + MARK_LOG_ROOT(visitor, "Exceptions"); + heapRootVisitor.visit(m_vm->addressOfException()); + visitor.donateAndDrain(); + } - HeapRootVisitor heapRootVisitor(m_slotVisitor); - - { - ParallelModeEnabler enabler(m_slotVisitor); - - m_slotVisitor.donateAndDrain(); - visitExternalRememberedSet(); - visitSmallStrings(); - visitConservativeRoots(conservativeRoots); - visitProtectedObjects(heapRootVisitor); - visitArgumentBuffers(heapRootVisitor); - visitException(heapRootVisitor); - visitStrongHandles(heapRootVisitor); - visitHandleStack(heapRootVisitor); - visitSamplingProfiler(); - traceCodeBlocksAndJITStubRoutines(); - converge(); + { + GCPHASE(VisitStrongHandles); + MARK_LOG_ROOT(visitor, "Strong Handles"); + m_handleSet.visitStrongHandles(heapRootVisitor); + visitor.donateAndDrain(); + } + + { + GCPHASE(HandleStack); + MARK_LOG_ROOT(visitor, "Handle Stack"); + m_handleStack.visit(heapRootVisitor); + visitor.donateAndDrain(); + } + + { + GCPHASE(TraceCodeBlocksAndJITStubRoutines); + MARK_LOG_ROOT(visitor, "Trace Code Blocks and JIT Stub Routines"); + m_codeBlocks.traceMarked(visitor); + m_jitStubRoutines.traceMarkedStubRoutines(visitor); + visitor.donateAndDrain(); + } + +#if ENABLE(PARALLEL_GC) + { + GCPHASE(Convergence); + visitor.drainFromShared(SlotVisitor::MasterDrain); + } +#endif } // Weak references must be marked last because their liveness depends on // the liveness of the rest of the object graph. - visitWeakHandles(heapRootVisitor); - { - std::lock_guard<Lock> lock(m_markingMutex); - m_parallelMarkersShouldExit = true; - m_markingConditionVariable.notifyAll(); - } - m_helperClient.finish(); - updateObjectCounts(gcStartTime); - resetVisitors(); -} - -void Heap::copyBackingStores() -{ - GCPHASE(CopyBackingStores); - if (m_operationInProgress == EdenCollection) - m_storageSpace.startedCopying<EdenCollection>(); - else { - ASSERT(m_operationInProgress == FullCollection); - m_storageSpace.startedCopying<FullCollection>(); + GCPHASE(VisitingLiveWeakHandles); + MARK_LOG_ROOT(visitor, "Live Weak Handles"); + while (true) { + m_objectSpace.visitWeakSets(heapRootVisitor); + harvestWeakReferences(); + if (visitor.isEmpty()) + break; + { + ParallelModeEnabler enabler(visitor); + visitor.donateAndDrain(); +#if ENABLE(PARALLEL_GC) + visitor.drainFromShared(SlotVisitor::MasterDrain); +#endif + } + } } - if (m_storageSpace.shouldDoCopyPhase()) { - if (m_operationInProgress == EdenCollection) { - // Reset the vector to be empty, but don't throw away the backing store. - m_blocksToCopy.shrink(0); - for (CopiedBlock* block = m_storageSpace.m_newGen.fromSpace->head(); block; block = block->next()) - m_blocksToCopy.append(block); - } else { - ASSERT(m_operationInProgress == FullCollection); - WTF::copyToVector(m_storageSpace.m_blockSet, m_blocksToCopy); +#if ENABLE(GGC) + { + GCPHASE(ClearRememberedSet); + for (unsigned i = 0; i < rememberedSet.size(); ++i) { + const JSCell* cell = rememberedSet[i]; + MarkedBlock::blockFor(cell)->clearRemembered(cell); } - - ParallelVectorIterator<Vector<CopiedBlock*>> iterator( - m_blocksToCopy, s_blockFragmentLength); - - // Note that it's safe to use the [&] capture list here, even though we're creating a task - // that other threads run. That's because after runFunctionInParallel() returns, the task - // we have created is not going to be running anymore. Hence, everything on the stack here - // outlives the task. - m_helperClient.runFunctionInParallel( - [&] () { - CopyVisitor copyVisitor(*this); - - iterator.iterate( - [&] (CopiedBlock* block) { - if (!block->hasWorkList()) - return; - - CopyWorkList& workList = block->workList(); - for (CopyWorklistItem item : workList) { - if (item.token() == ButterflyCopyToken) { - JSObject::copyBackingStore( - item.cell(), copyVisitor, ButterflyCopyToken); - continue; - } - - item.cell()->methodTable()->copyBackingStore( - item.cell(), copyVisitor, item.token()); - } - - ASSERT(!block->liveBytes()); - m_storageSpace.recycleEvacuatedBlock(block, m_operationInProgress); - }); - }); } - - m_storageSpace.doneCopying(); -} - -void Heap::gatherStackRoots(ConservativeRoots& roots, void* stackOrigin, void* stackTop, MachineThreads::RegisterState& calleeSavedRegisters) -{ - GCPHASE(GatherStackRoots); - m_jitStubRoutines.clearMarks(); - m_machineThreads.gatherConservativeRoots(roots, m_jitStubRoutines, m_codeBlocks, stackOrigin, stackTop, calleeSavedRegisters); -} - -void Heap::gatherJSStackRoots(ConservativeRoots& roots) -{ -#if !ENABLE(JIT) - GCPHASE(GatherJSStackRoots); - stack().gatherConservativeRoots(roots, m_jitStubRoutines, m_codeBlocks); -#else - UNUSED_PARAM(roots); #endif -} -void Heap::gatherScratchBufferRoots(ConservativeRoots& roots) -{ -#if ENABLE(DFG_JIT) - GCPHASE(GatherScratchBufferRoots); - m_vm->gatherConservativeRoots(roots); -#else - UNUSED_PARAM(roots); -#endif -} + GCCOUNTER(VisitedValueCount, visitor.visitCount()); -void Heap::clearLivenessData() -{ - GCPHASE(ClearLivenessData); - if (m_operationInProgress == FullCollection) - m_codeBlocks.clearMarksForFullCollection(); - - m_objectSpace.clearNewlyAllocated(); - m_objectSpace.clearMarks(); -} - -void Heap::visitExternalRememberedSet() -{ -#if JSC_OBJC_API_ENABLED - scanExternalRememberedSet(*m_vm, m_slotVisitor); -#endif -} - -void Heap::visitSmallStrings() -{ - GCPHASE(VisitSmallStrings); - if (!m_vm->smallStrings.needsToBeVisited(m_operationInProgress)) - return; - - m_vm->smallStrings.visitStrongReferences(m_slotVisitor); - if (Options::logGC() == GCLogging::Verbose) - dataLog("Small strings:\n", m_slotVisitor); - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitConservativeRoots(ConservativeRoots& roots) -{ - GCPHASE(VisitConservativeRoots); - m_slotVisitor.append(roots); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Conservative Roots:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitCompilerWorklistWeakReferences() -{ -#if ENABLE(DFG_JIT) - for (auto worklist : m_suspendedCompilerWorklists) - worklist->visitWeakReferences(m_slotVisitor); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("DFG Worklists:\n", m_slotVisitor); + m_sharedData.didFinishMarking(); +#if ENABLE(OBJECT_MARK_LOGGING) + size_t visitCount = visitor.visitCount(); +#if ENABLE(PARALLEL_GC) + visitCount += m_sharedData.childVisitCount(); #endif -} - -void Heap::removeDeadCompilerWorklistEntries() -{ -#if ENABLE(DFG_JIT) - GCPHASE(FinalizeDFGWorklists); - for (auto worklist : m_suspendedCompilerWorklists) - worklist->removeDeadPlans(*m_vm); + MARK_LOG_MESSAGE2("\nNumber of live Objects after full GC %lu, took %.6f secs\n", visitCount, WTF::monotonicallyIncreasingTime() - gcStartTime); #endif -} - -void Heap::visitProtectedObjects(HeapRootVisitor& heapRootVisitor) -{ - GCPHASE(VisitProtectedObjects); - - for (auto& pair : m_protectedValues) - heapRootVisitor.visit(&pair.key); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Protected Objects:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitArgumentBuffers(HeapRootVisitor& visitor) -{ - GCPHASE(MarkingArgumentBuffers); - if (!m_markListSet || !m_markListSet->size()) - return; - - MarkedArgumentBuffer::markLists(visitor, *m_markListSet); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Argument Buffers:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitException(HeapRootVisitor& visitor) -{ - GCPHASE(MarkingException); - if (!m_vm->exception() && !m_vm->lastException()) - return; - - visitor.visit(m_vm->addressOfException()); - visitor.visit(m_vm->addressOfLastException()); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Exceptions:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitStrongHandles(HeapRootVisitor& visitor) -{ - GCPHASE(VisitStrongHandles); - m_handleSet.visitStrongHandles(visitor); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Strong Handles:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::visitHandleStack(HeapRootVisitor& visitor) -{ - GCPHASE(VisitHandleStack); - m_handleStack.visit(visitor); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Handle Stack:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} -void Heap::visitSamplingProfiler() -{ -#if ENABLE(SAMPLING_PROFILER) - if (SamplingProfiler* samplingProfiler = m_vm->samplingProfiler()) { - ASSERT(samplingProfiler->getLock().isLocked()); - GCPHASE(VisitSamplingProfiler); - samplingProfiler->visit(m_slotVisitor); - if (Options::logGC() == GCLogging::Verbose) - dataLog("Sampling Profiler data:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); - samplingProfiler->getLock().unlock(); + if (m_operationInProgress == EdenCollection) { + m_totalBytesVisited += visitor.bytesVisited(); + m_totalBytesCopied += visitor.bytesCopied(); + } else { + ASSERT(m_operationInProgress == FullCollection); + m_totalBytesVisited = visitor.bytesVisited(); + m_totalBytesCopied = visitor.bytesCopied(); } -#endif // ENABLE(SAMPLING_PROFILER) -} - -void Heap::traceCodeBlocksAndJITStubRoutines() -{ - GCPHASE(TraceCodeBlocksAndJITStubRoutines); - m_jitStubRoutines.traceMarkedStubRoutines(m_slotVisitor); - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Code Blocks and JIT Stub Routines:\n", m_slotVisitor); - - m_slotVisitor.donateAndDrain(); -} - -void Heap::converge() -{ - GCPHASE(Convergence); - m_slotVisitor.drainFromShared(SlotVisitor::MasterDrain); -} - -void Heap::visitWeakHandles(HeapRootVisitor& visitor) -{ - GCPHASE(VisitingLiveWeakHandles); - while (true) { - m_objectSpace.visitWeakSets(visitor); - harvestWeakReferences(); - visitCompilerWorklistWeakReferences(); - if (m_slotVisitor.isEmpty()) - break; - - if (Options::logGC() == GCLogging::Verbose) - dataLog("Live Weak Handles:\n", m_slotVisitor); +#if ENABLE(PARALLEL_GC) + m_totalBytesVisited += m_sharedData.childBytesVisited(); + m_totalBytesCopied += m_sharedData.childBytesCopied(); +#endif - { - ParallelModeEnabler enabler(m_slotVisitor); - m_slotVisitor.donateAndDrain(); - m_slotVisitor.drainFromShared(SlotVisitor::MasterDrain); - } - } + visitor.reset(); +#if ENABLE(PARALLEL_GC) + m_sharedData.resetChildren(); +#endif + m_sharedData.reset(); } -void Heap::updateObjectCounts(double gcStartTime) +template <HeapOperation collectionType> +void Heap::copyBackingStores() { - GCCOUNTER(VisitedValueCount, m_slotVisitor.visitCount() + threadVisitCount()); - - if (Options::logGC() == GCLogging::Verbose) { - size_t visitCount = m_slotVisitor.visitCount(); - visitCount += threadVisitCount(); - dataLogF("\nNumber of live Objects after GC %lu, took %.6f secs\n", static_cast<unsigned long>(visitCount), WTF::monotonicallyIncreasingTime() - gcStartTime); - } - - size_t bytesRemovedFromOldSpaceDueToReallocation = - m_storageSpace.takeBytesRemovedFromOldSpaceDueToReallocation(); - - if (m_operationInProgress == FullCollection) { - m_totalBytesVisited = 0; - m_totalBytesCopied = 0; + m_storageSpace.startedCopying<collectionType>(); + if (m_storageSpace.shouldDoCopyPhase()) { + m_sharedData.didStartCopying(); + m_copyVisitor.startCopying(); + m_copyVisitor.copyFromShared(); + m_copyVisitor.doneCopying(); + // We need to wait for everybody to finish and return their CopiedBlocks + // before signaling that the phase is complete. + m_storageSpace.doneCopying(); + m_sharedData.didFinishCopying(); } else - m_totalBytesCopied -= bytesRemovedFromOldSpaceDueToReallocation; - - m_totalBytesVisitedThisCycle = m_slotVisitor.bytesVisited() + threadBytesVisited(); - m_totalBytesCopiedThisCycle = m_slotVisitor.bytesCopied() + threadBytesCopied(); - - m_totalBytesVisited += m_totalBytesVisitedThisCycle; - m_totalBytesCopied += m_totalBytesCopiedThisCycle; -} - -void Heap::resetVisitors() -{ - m_slotVisitor.reset(); - - for (auto& parallelVisitor : m_parallelSlotVisitors) - parallelVisitor->reset(); - - ASSERT(m_sharedMarkStack.isEmpty()); - m_weakReferenceHarvesters.removeAll(); + m_storageSpace.doneCopying(); } size_t Heap::objectCount() @@ -918,19 +659,29 @@ size_t Heap::objectCount() return m_objectSpace.objectCount(); } -size_t Heap::extraMemorySize() +size_t Heap::extraSize() { - return m_extraMemorySize + m_deprecatedExtraMemorySize + m_arrayBuffers.size(); + return m_extraMemoryUsage + m_arrayBuffers.size(); } size_t Heap::size() { - return m_objectSpace.size() + m_storageSpace.size() + extraMemorySize(); + return m_objectSpace.size() + m_storageSpace.size() + extraSize(); } size_t Heap::capacity() { - return m_objectSpace.capacity() + m_storageSpace.capacity() + extraMemorySize(); + return m_objectSpace.capacity() + m_storageSpace.capacity() + extraSize(); +} + +size_t Heap::sizeAfterCollect() +{ + // The result here may not agree with the normal Heap::size(). + // This is due to the fact that we only count live copied bytes + // rather than all used (including dead) copied bytes, thus it's + // always the case that m_totalBytesCopied <= m_storageSpace.size(). + ASSERT(m_totalBytesCopied <= m_storageSpace.size()); + return m_totalBytesVisited + m_totalBytesCopied + extraSize(); } size_t Heap::protectedGlobalObjectCount() @@ -949,116 +700,79 @@ size_t Heap::protectedObjectCount() return forEachProtectedCell<Count>(); } -std::unique_ptr<TypeCountSet> Heap::protectedObjectTypeCounts() +PassOwnPtr<TypeCountSet> Heap::protectedObjectTypeCounts() { return forEachProtectedCell<RecordType>(); } -std::unique_ptr<TypeCountSet> Heap::objectTypeCounts() +PassOwnPtr<TypeCountSet> Heap::objectTypeCounts() { HeapIterationScope iterationScope(*this); return m_objectSpace.forEachLiveCell<RecordType>(iterationScope); } -void Heap::deleteAllCodeBlocks() +void Heap::deleteAllCompiledCode() { - // If JavaScript is running, it's not safe to delete all JavaScript code, since - // we'll end up returning to deleted code. - RELEASE_ASSERT(!m_vm->entryScope); - ASSERT(m_operationInProgress == NoOperation); - - completeAllDFGPlans(); - - for (ExecutableBase* executable : m_executables) - executable->clearCode(); -} + // If JavaScript is running, it's not safe to delete code, since we'll end + // up deleting code that is live on the stack. + if (m_vm->entryScope) + return; -void Heap::deleteAllUnlinkedCodeBlocks() -{ - for (ExecutableBase* current : m_executables) { + for (ExecutableBase* current = m_compiledCode.head(); current; current = current->next()) { if (!current->isFunctionExecutable()) continue; - static_cast<FunctionExecutable*>(current)->unlinkedExecutable()->clearCode(); + static_cast<FunctionExecutable*>(current)->clearCodeIfNotCompiling(); } + + m_codeBlocks.clearMarks(); + m_codeBlocks.deleteUnmarkedAndUnreferenced(); } -void Heap::clearUnmarkedExecutables() +void Heap::deleteUnmarkedCompiledCode() { - GCPHASE(ClearUnmarkedExecutables); - for (unsigned i = m_executables.size(); i--;) { - ExecutableBase* current = m_executables[i]; + ExecutableBase* next; + for (ExecutableBase* current = m_compiledCode.head(); current; current = next) { + next = current->next(); if (isMarked(current)) continue; - // Eagerly dereference the Executable's JITCode in order to run watchpoint - // destructors. Otherwise, watchpoints might fire for deleted CodeBlocks. - current->clearCode(); - std::swap(m_executables[i], m_executables.last()); - m_executables.removeLast(); + // We do this because executable memory is limited on some platforms and because + // CodeBlock requires eager finalization. + ExecutableBase::clearCodeVirtual(current); + m_compiledCode.remove(current); } - m_executables.shrinkToFit(); -} - -void Heap::deleteUnmarkedCompiledCode() -{ - GCPHASE(DeleteCodeBlocks); - clearUnmarkedExecutables(); - m_codeBlocks.deleteUnmarkedAndUnreferenced(m_operationInProgress); + m_codeBlocks.deleteUnmarkedAndUnreferenced(); m_jitStubRoutines.deleteUnmarkedJettisonedStubRoutines(); } void Heap::addToRememberedSet(const JSCell* cell) { ASSERT(cell); - ASSERT(!Options::useConcurrentJIT() || !isCompilationThread()); - ASSERT(cell->cellState() == CellState::OldBlack); - // Indicate that this object is grey and that it's one of the following: - // - A re-greyed object during a concurrent collection. - // - An old remembered object. - // "OldGrey" doesn't tell us which of these things is true, but we usually treat the two cases the - // same. - cell->setCellState(CellState::OldGrey); - m_slotVisitor.appendToMarkStack(const_cast<JSCell*>(cell)); -} - -void* Heap::copyBarrier(const JSCell*, void*& pointer) -{ - // Do nothing for now, except making sure that the low bits are masked off. This helps to - // simulate enough of this barrier that at least we can test the low bits assumptions. - pointer = bitwise_cast<void*>( - bitwise_cast<uintptr_t>(pointer) & ~static_cast<uintptr_t>(CopyBarrierBase::spaceBits)); - - return pointer; + ASSERT(!Options::enableConcurrentJIT() || !isCompilationThread()); + if (isInRememberedSet(cell)) + return; + MarkedBlock::blockFor(cell)->setRemembered(cell); + m_slotVisitor.unconditionallyAppend(const_cast<JSCell*>(cell)); } -void Heap::collectAndSweep(HeapOperation collectionType) +void Heap::collectAllGarbage() { if (!m_isSafeToCollect) return; - collect(collectionType); + m_shouldDoFullCollection = true; + collect(); SamplingRegion samplingRegion("Garbage Collection: Sweeping"); - - DeferGCForAWhile deferGC(*this); + DelayedReleaseScope delayedReleaseScope(m_objectSpace); m_objectSpace.sweep(); m_objectSpace.shrink(); - - sweepAllLogicallyEmptyWeakBlocks(); } -NEVER_INLINE void Heap::collect(HeapOperation collectionType) -{ - void* stackTop; - ALLOCATE_AND_GET_REGISTER_STATE(registers); - - collectImpl(collectionType, wtfThreadData().stack().origin(), &stackTop, registers); +static double minute = 60.0; - sanitizeStackForVM(m_vm); -} - -NEVER_INLINE void Heap::collectImpl(HeapOperation collectionType, void* stackOrigin, void* stackTop, MachineThreads::RegisterState& calleeSavedRegisters) +void Heap::collect() { #if ENABLE(ALLOCATION_LOGGING) dataLogF("JSC GC starting collection.\n"); @@ -1066,276 +780,127 @@ NEVER_INLINE void Heap::collectImpl(HeapOperation collectionType, void* stackOri double before = 0; if (Options::logGC()) { - dataLog("[GC: ", capacity() / 1024, " kb "); + dataLog("[GC: "); before = currentTimeMS(); } SamplingRegion samplingRegion("Garbage Collection"); - if (vm()->typeProfiler()) { - DeferGCForAWhile awhile(*this); - vm()->typeProfilerLog()->processLogEntries(ASCIILiteral("GC")); - } - RELEASE_ASSERT(!m_deferralDepth); + GCPHASE(Collect); ASSERT(vm()->currentThreadIsHoldingAPILock()); - RELEASE_ASSERT(vm()->atomicStringTable() == wtfThreadData().atomicStringTable()); + RELEASE_ASSERT(vm()->identifierTable == wtfThreadData().currentIdentifierTable()); ASSERT(m_isSafeToCollect); JAVASCRIPTCORE_GC_BEGIN(); RELEASE_ASSERT(m_operationInProgress == NoOperation); - - suspendCompilerThreads(); - willStartCollection(collectionType); - GCPHASE(Collect); - - double gcStartTime = WTF::monotonicallyIncreasingTime(); - if (m_verifier) { - // Verify that live objects from the last GC cycle haven't been corrupted by - // mutators before we begin this new GC cycle. - m_verifier->verify(HeapVerifier::Phase::BeforeGC); - - m_verifier->initializeGCCycle(); - m_verifier->gatherLiveObjects(HeapVerifier::Phase::BeforeMarking); - } - - flushOldStructureIDTables(); - stopAllocation(); - flushWriteBarrierBuffer(); - - markRoots(gcStartTime, stackOrigin, stackTop, calleeSavedRegisters); - - if (m_verifier) { - m_verifier->gatherLiveObjects(HeapVerifier::Phase::AfterMarking); - m_verifier->verify(HeapVerifier::Phase::AfterMarking); - } - JAVASCRIPTCORE_GC_MARKED(); - - if (vm()->typeProfiler()) - vm()->typeProfiler()->invalidateTypeSetCache(); - - reapWeakHandles(); - pruneStaleEntriesFromWeakGCMaps(); - sweepArrayBuffers(); - snapshotMarkedSpace(); - - copyBackingStores(); - - finalizeUnconditionalFinalizers(); - removeDeadCompilerWorklistEntries(); - deleteUnmarkedCompiledCode(); - deleteSourceProviderCaches(); - notifyIncrementalSweeper(); - writeBarrierCurrentlyExecutingCodeBlocks(); - - resetAllocators(); - updateAllocationLimits(); - didFinishCollection(gcStartTime); - resumeCompilerThreads(); - - if (m_verifier) { - m_verifier->trimDeadObjects(); - m_verifier->verify(HeapVerifier::Phase::AfterGC); - } - - if (Options::logGC()) { - double after = currentTimeMS(); - dataLog(after - before, " ms]\n"); - } -} - -void Heap::suspendCompilerThreads() -{ -#if ENABLE(DFG_JIT) - GCPHASE(SuspendCompilerThreads); - ASSERT(m_suspendedCompilerWorklists.isEmpty()); - for (unsigned i = DFG::numberOfWorklists(); i--;) { - if (DFG::Worklist* worklist = DFG::worklistForIndexOrNull(i)) { - m_suspendedCompilerWorklists.append(worklist); - worklist->suspendAllThreads(); - } + + { + RecursiveAllocationScope scope(*this); + m_vm->prepareToDiscardCode(); } -#endif -} -void Heap::willStartCollection(HeapOperation collectionType) -{ - GCPHASE(StartingCollection); - - if (Options::logGC()) - dataLog("=> "); - - if (shouldDoFullCollection(collectionType)) { + bool isFullCollection = m_shouldDoFullCollection; + if (isFullCollection) { m_operationInProgress = FullCollection; + m_slotVisitor.clearMarkStack(); m_shouldDoFullCollection = false; if (Options::logGC()) dataLog("FullCollection, "); } else { +#if ENABLE(GGC) m_operationInProgress = EdenCollection; if (Options::logGC()) dataLog("EdenCollection, "); +#else + m_operationInProgress = FullCollection; + m_slotVisitor.clearMarkStack(); + if (Options::logGC()) + dataLog("FullCollection, "); +#endif } - if (m_operationInProgress == FullCollection) { - m_sizeBeforeLastFullCollect = m_sizeAfterLastCollect + m_bytesAllocatedThisCycle; - m_extraMemorySize = 0; - m_deprecatedExtraMemorySize = 0; - - if (m_fullActivityCallback) - m_fullActivityCallback->willCollect(); - } else { - ASSERT(m_operationInProgress == EdenCollection); - m_sizeBeforeLastEdenCollect = m_sizeAfterLastCollect + m_bytesAllocatedThisCycle; - } - - if (m_edenActivityCallback) - m_edenActivityCallback->willCollect(); - - for (auto* observer : m_observers) - observer->willGarbageCollect(); -} - -void Heap::flushOldStructureIDTables() -{ - GCPHASE(FlushOldStructureIDTables); - m_structureIDTable.flushOldTables(); -} - -void Heap::flushWriteBarrierBuffer() -{ - GCPHASE(FlushWriteBarrierBuffer); - if (m_operationInProgress == EdenCollection) { - m_writeBarrierBuffer.flush(*this); - return; - } - m_writeBarrierBuffer.reset(); -} - -void Heap::stopAllocation() -{ - GCPHASE(StopAllocation); - m_objectSpace.stopAllocating(); if (m_operationInProgress == FullCollection) - m_storageSpace.didStartFullCollection(); -} - -void Heap::reapWeakHandles() -{ - GCPHASE(ReapingWeakHandles); - m_objectSpace.reapWeakSets(); -} + m_extraMemoryUsage = 0; -void Heap::pruneStaleEntriesFromWeakGCMaps() -{ - GCPHASE(PruningStaleEntriesFromWeakGCMaps); - if (m_operationInProgress != FullCollection) - return; - for (auto& pruneCallback : m_weakGCMaps.values()) - pruneCallback(); -} + if (m_activityCallback) + m_activityCallback->willCollect(); -void Heap::sweepArrayBuffers() -{ - GCPHASE(SweepingArrayBuffers); - m_arrayBuffers.sweep(); -} + double lastGCStartTime = WTF::monotonicallyIncreasingTime(); + if (lastGCStartTime - m_lastCodeDiscardTime > minute) { + deleteAllCompiledCode(); + m_lastCodeDiscardTime = WTF::monotonicallyIncreasingTime(); + } -struct MarkedBlockSnapshotFunctor : public MarkedBlock::VoidFunctor { - MarkedBlockSnapshotFunctor(Vector<MarkedBlock*>& blocks) - : m_index(0) - , m_blocks(blocks) { + GCPHASE(StopAllocation); + m_objectSpace.stopAllocating(); + if (m_operationInProgress == FullCollection) + m_storageSpace.didStartFullCollection(); } - void operator()(MarkedBlock* block) { m_blocks[m_index++] = block; } + { + GCPHASE(FlushWriteBarrierBuffer); + if (m_operationInProgress == EdenCollection) + m_writeBarrierBuffer.flush(*this); + else + m_writeBarrierBuffer.reset(); + } - size_t m_index; - Vector<MarkedBlock*>& m_blocks; -}; + markRoots(); + + { + GCPHASE(ReapingWeakHandles); + m_objectSpace.reapWeakSets(); + } -void Heap::snapshotMarkedSpace() -{ - GCPHASE(SnapshotMarkedSpace); + JAVASCRIPTCORE_GC_MARKED(); + + { + GCPHASE(SweepingArrayBuffers); + m_arrayBuffers.sweep(); + } - if (m_operationInProgress == EdenCollection) { - m_blockSnapshot.appendVector(m_objectSpace.blocksWithNewObjects()); - // Sort and deduplicate the block snapshot since we might be appending to an unfinished work list. - std::sort(m_blockSnapshot.begin(), m_blockSnapshot.end()); - m_blockSnapshot.shrink(std::unique(m_blockSnapshot.begin(), m_blockSnapshot.end()) - m_blockSnapshot.begin()); - } else { - m_blockSnapshot.resizeToFit(m_objectSpace.blocks().set().size()); + if (m_operationInProgress == FullCollection) { + m_blockSnapshot.resize(m_objectSpace.blocks().set().size()); MarkedBlockSnapshotFunctor functor(m_blockSnapshot); m_objectSpace.forEachBlock(functor); } -} -void Heap::deleteSourceProviderCaches() -{ - GCPHASE(DeleteSourceProviderCaches); - m_vm->clearSourceProviderCaches(); -} + if (m_operationInProgress == FullCollection) + copyBackingStores<FullCollection>(); + else + copyBackingStores<EdenCollection>(); -void Heap::notifyIncrementalSweeper() -{ - GCPHASE(NotifyIncrementalSweeper); + { + GCPHASE(FinalizeUnconditionalFinalizers); + finalizeUnconditionalFinalizers(); + } - if (m_operationInProgress == FullCollection) { - if (!m_logicallyEmptyWeakBlocks.isEmpty()) - m_indexOfNextLogicallyEmptyWeakBlockToSweep = 0; + { + GCPHASE(DeleteCodeBlocks); + deleteUnmarkedCompiledCode(); } - m_sweeper->startSweeping(); -} + { + GCPHASE(DeleteSourceProviderCaches); + m_vm->clearSourceProviderCaches(); + } -void Heap::writeBarrierCurrentlyExecutingCodeBlocks() -{ - GCPHASE(WriteBarrierCurrentlyExecutingCodeBlocks); - m_codeBlocks.writeBarrierCurrentlyExecutingCodeBlocks(this); -} + if (m_operationInProgress == FullCollection) + m_sweeper->startSweeping(m_blockSnapshot); -void Heap::resetAllocators() -{ - GCPHASE(ResetAllocators); - m_objectSpace.resetAllocators(); -} + { + GCPHASE(AddCurrentlyExecutingCodeBlocksToRememberedSet); + m_codeBlocks.rememberCurrentlyExecutingCodeBlocks(this); + } -void Heap::updateAllocationLimits() -{ - GCPHASE(UpdateAllocationLimits); - - // Calculate our current heap size threshold for the purpose of figuring out when we should - // run another collection. This isn't the same as either size() or capacity(), though it should - // be somewhere between the two. The key is to match the size calculations involved calls to - // didAllocate(), while never dangerously underestimating capacity(). In extreme cases of - // fragmentation, we may have size() much smaller than capacity(). Our collector sometimes - // temporarily allows very high fragmentation because it doesn't defragment old blocks in copied - // space. - size_t currentHeapSize = 0; - - // For marked space, we use the total number of bytes visited. This matches the logic for - // MarkedAllocator's calls to didAllocate(), which effectively accounts for the total size of - // objects allocated rather than blocks used. This will underestimate capacity(), and in case - // of fragmentation, this may be substantial. Fortunately, marked space rarely fragments because - // cells usually have a narrow range of sizes. So, the underestimation is probably OK. - currentHeapSize += m_totalBytesVisited; - - // For copied space, we use the capacity of storage space. This is because copied space may get - // badly fragmented between full collections. This arises when each eden collection evacuates - // much less than one CopiedBlock's worth of stuff. It can also happen when CopiedBlocks get - // pinned due to very short-lived objects. In such a case, we want to get to a full collection - // sooner rather than later. If we used m_totalBytesCopied, then for for each CopiedBlock that an - // eden allocation promoted, we would only deduct the one object's size from eden size. This - // would mean that we could "leak" many CopiedBlocks before we did a full collection and - // defragmented all of them. It would be great to use m_totalBytesCopied, but we'd need to - // augment it with something that accounts for those fragmented blocks. - // FIXME: Make it possible to compute heap size using m_totalBytesCopied rather than - // m_storageSpace.capacity() - // https://bugs.webkit.org/show_bug.cgi?id=150268 - ASSERT(m_totalBytesCopied <= m_storageSpace.size()); - currentHeapSize += m_storageSpace.capacity(); + m_bytesAbandonedThisCycle = 0; - // It's up to the user to ensure that extraMemorySize() ends up corresponding to allocation-time - // extra memory reporting. - currentHeapSize += extraMemorySize(); + { + GCPHASE(ResetAllocators); + m_objectSpace.resetAllocators(); + } + size_t currentHeapSize = sizeAfterCollect(); if (Options::gcMaxHeapSize() && currentHeapSize > Options::gcMaxHeapSize()) HeapStatistics::exitWithFailure(); @@ -1345,82 +910,55 @@ void Heap::updateAllocationLimits() // fixed minimum. m_maxHeapSize = max(minHeapSize(m_heapType, m_ramSize), proportionalHeapSize(currentHeapSize, m_ramSize)); m_maxEdenSize = m_maxHeapSize - currentHeapSize; - m_sizeAfterLastFullCollect = currentHeapSize; - m_bytesAbandonedSinceLastFullCollect = 0; } else { - static const bool verbose = false; - ASSERT(currentHeapSize >= m_sizeAfterLastCollect); m_maxEdenSize = m_maxHeapSize - currentHeapSize; - m_sizeAfterLastEdenCollect = currentHeapSize; - if (verbose) { - dataLog("Max heap size: ", m_maxHeapSize, "\n"); - dataLog("Current heap size: ", currentHeapSize, "\n"); - dataLog("Size after last eden collection: ", m_sizeAfterLastEdenCollect, "\n"); - } double edenToOldGenerationRatio = (double)m_maxEdenSize / (double)m_maxHeapSize; - if (verbose) - dataLog("Eden to old generation ratio: ", edenToOldGenerationRatio, "\n"); double minEdenToOldGenerationRatio = 1.0 / 3.0; if (edenToOldGenerationRatio < minEdenToOldGenerationRatio) m_shouldDoFullCollection = true; - // This seems suspect at first, but what it does is ensure that the nursery size is fixed. m_maxHeapSize += currentHeapSize - m_sizeAfterLastCollect; m_maxEdenSize = m_maxHeapSize - currentHeapSize; - if (m_fullActivityCallback) { - ASSERT(currentHeapSize >= m_sizeAfterLastFullCollect); - m_fullActivityCallback->didAllocate(currentHeapSize - m_sizeAfterLastFullCollect); - } } m_sizeAfterLastCollect = currentHeapSize; - m_bytesAllocatedThisCycle = 0; - - if (Options::logGC()) - dataLog(currentHeapSize / 1024, " kb, "); -} -void Heap::didFinishCollection(double gcStartTime) -{ - GCPHASE(FinishingCollection); - double gcEndTime = WTF::monotonicallyIncreasingTime(); - HeapOperation operation = m_operationInProgress; - if (m_operationInProgress == FullCollection) - m_lastFullGCLength = gcEndTime - gcStartTime; - else - m_lastEdenGCLength = gcEndTime - gcStartTime; + m_bytesAllocatedThisCycle = 0; + double lastGCEndTime = WTF::monotonicallyIncreasingTime(); + m_lastGCLength = lastGCEndTime - lastGCStartTime; if (Options::recordGCPauseTimes()) - HeapStatistics::recordGCPauseTime(gcStartTime, gcEndTime); + HeapStatistics::recordGCPauseTime(lastGCStartTime, lastGCEndTime); + RELEASE_ASSERT(m_operationInProgress == EdenCollection || m_operationInProgress == FullCollection); + + m_operationInProgress = NoOperation; + JAVASCRIPTCORE_GC_END(); if (Options::useZombieMode()) zombifyDeadObjects(); - if (Options::useImmortalObjects()) + if (Options::objectsAreImmortal()) markDeadObjects(); - if (Options::dumpObjectStatistics()) - HeapStatistics::dumpObjectStatistics(this); - - if (Options::logGC() == GCLogging::Verbose) - GCLogging::dumpObjectGraph(this); - - RELEASE_ASSERT(m_operationInProgress == EdenCollection || m_operationInProgress == FullCollection); - m_operationInProgress = NoOperation; - JAVASCRIPTCORE_GC_END(); - - for (auto* observer : m_observers) - observer->didGarbageCollect(operation); + if (Options::showObjectStatistics()) + HeapStatistics::showObjectStatistics(this); + + if (Options::logGC()) { + double after = currentTimeMS(); + dataLog(after - before, " ms, ", currentHeapSize / 1024, " kb]\n"); + } } -void Heap::resumeCompilerThreads() +bool Heap::collectIfNecessaryOrDefer() { -#if ENABLE(DFG_JIT) - GCPHASE(ResumeCompilerThreads); - for (auto worklist : m_suspendedCompilerWorklists) - worklist->resumeAllThreads(); - m_suspendedCompilerWorklists.clear(); -#endif + if (m_deferralDepth) + return false; + + if (!shouldCollect()) + return false; + + collect(); + return true; } void Heap::markDeadObjects() @@ -1429,29 +967,19 @@ void Heap::markDeadObjects() m_objectSpace.forEachDeadCell<MarkObject>(iterationScope); } -void Heap::setFullActivityCallback(PassRefPtr<FullGCActivityCallback> activityCallback) +void Heap::setActivityCallback(PassOwnPtr<GCActivityCallback> activityCallback) { - m_fullActivityCallback = activityCallback; + m_activityCallback = activityCallback; } -void Heap::setEdenActivityCallback(PassRefPtr<EdenGCActivityCallback> activityCallback) +GCActivityCallback* Heap::activityCallback() { - m_edenActivityCallback = activityCallback; + return m_activityCallback.get(); } -GCActivityCallback* Heap::fullActivityCallback() +void Heap::setIncrementalSweeper(PassOwnPtr<IncrementalSweeper> sweeper) { - return m_fullActivityCallback.get(); -} - -GCActivityCallback* Heap::edenActivityCallback() -{ - return m_edenActivityCallback.get(); -} - -void Heap::setIncrementalSweeper(std::unique_ptr<IncrementalSweeper> sweeper) -{ - m_sweeper = WTFMove(sweeper); + m_sweeper = sweeper; } IncrementalSweeper* Heap::sweeper() @@ -1461,16 +989,14 @@ IncrementalSweeper* Heap::sweeper() void Heap::setGarbageCollectionTimerEnabled(bool enable) { - if (m_fullActivityCallback) - m_fullActivityCallback->setEnabled(enable); - if (m_edenActivityCallback) - m_edenActivityCallback->setEnabled(enable); + if (m_activityCallback) + m_activityCallback->setEnabled(enable); } void Heap::didAllocate(size_t bytes) { - if (m_edenActivityCallback) - m_edenActivityCallback->didAllocate(m_bytesAllocatedThisCycle + m_bytesAbandonedSinceLastFullCollect); + if (m_activityCallback) + m_activityCallback->didAllocate(m_bytesAllocatedThisCycle + m_bytesAbandonedThisCycle); m_bytesAllocatedThisCycle += bytes; } @@ -1498,31 +1024,14 @@ void Heap::FinalizerOwner::finalize(Handle<Unknown> handle, void* context) WeakSet::deallocate(WeakImpl::asWeakImpl(slot)); } -void Heap::addExecutable(ExecutableBase* executable) +void Heap::addCompiledCode(ExecutableBase* executable) { - m_executables.append(executable); -} - -void Heap::collectAllGarbageIfNotDoneRecently() -{ - if (!m_fullActivityCallback) { - collectAllGarbage(); - return; - } - - if (m_fullActivityCallback->didSyncGCRecently()) { - // A synchronous GC was already requested recently so we merely accelerate next collection. - reportAbandonedObjectGraph(); - return; - } - - m_fullActivityCallback->setDidSyncGCRecently(); - collectAllGarbage(); + m_compiledCode.append(executable); } class Zombify : public MarkedBlock::VoidFunctor { public: - inline void visit(JSCell* cell) + void operator()(JSCell* cell) { void** current = reinterpret_cast<void**>(cell); @@ -1533,111 +1042,59 @@ public: void* limit = static_cast<void*>(reinterpret_cast<char*>(cell) + MarkedBlock::blockFor(cell)->cellSize()); for (; current < limit; current++) - *current = zombifiedBits; - } - IterationStatus operator()(JSCell* cell) - { - visit(cell); - return IterationStatus::Continue; + *current = reinterpret_cast<void*>(0xbbadbeef); } }; void Heap::zombifyDeadObjects() { // Sweep now because destructors will crash once we're zombified. - { - SamplingRegion samplingRegion("Garbage Collection: Sweeping"); - m_objectSpace.zombifySweep(); - } + m_objectSpace.sweep(); HeapIterationScope iterationScope(*this); m_objectSpace.forEachDeadCell<Zombify>(iterationScope); } -void Heap::flushWriteBarrierBuffer(JSCell* cell) +void Heap::incrementDeferralDepth() { - m_writeBarrierBuffer.flush(*this); - m_writeBarrierBuffer.add(cell); + RELEASE_ASSERT(m_deferralDepth < 100); // Sanity check to make sure this doesn't get ridiculous. + + m_deferralDepth++; } -bool Heap::shouldDoFullCollection(HeapOperation requestedCollectionType) const +void Heap::decrementDeferralDepth() { - if (!Options::useGenerationalGC()) - return true; - - switch (requestedCollectionType) { - case EdenCollection: - return false; - case FullCollection: - return true; - case AnyCollection: - return m_shouldDoFullCollection; - default: - RELEASE_ASSERT_NOT_REACHED(); - return false; - } - RELEASE_ASSERT_NOT_REACHED(); - return false; + RELEASE_ASSERT(m_deferralDepth >= 1); + + m_deferralDepth--; } -void Heap::addLogicallyEmptyWeakBlock(WeakBlock* block) +void Heap::decrementDeferralDepthAndGCIfNeeded() { - m_logicallyEmptyWeakBlocks.append(block); + decrementDeferralDepth(); + collectIfNecessaryOrDefer(); } -void Heap::sweepAllLogicallyEmptyWeakBlocks() +void Heap::writeBarrier(const JSCell* from) { - if (m_logicallyEmptyWeakBlocks.isEmpty()) +#if ENABLE(GGC) + ASSERT_GC_OBJECT_LOOKS_VALID(const_cast<JSCell*>(from)); + if (!from || !isMarked(from)) return; - - m_indexOfNextLogicallyEmptyWeakBlockToSweep = 0; - while (sweepNextLogicallyEmptyWeakBlock()) { } + Heap* heap = Heap::heap(from); + heap->addToRememberedSet(from); +#else + UNUSED_PARAM(from); +#endif } -bool Heap::sweepNextLogicallyEmptyWeakBlock() +void Heap::flushWriteBarrierBuffer(JSCell* cell) { - if (m_indexOfNextLogicallyEmptyWeakBlockToSweep == WTF::notFound) - return false; - - WeakBlock* block = m_logicallyEmptyWeakBlocks[m_indexOfNextLogicallyEmptyWeakBlockToSweep]; - - block->sweep(); - if (block->isEmpty()) { - std::swap(m_logicallyEmptyWeakBlocks[m_indexOfNextLogicallyEmptyWeakBlockToSweep], m_logicallyEmptyWeakBlocks.last()); - m_logicallyEmptyWeakBlocks.removeLast(); - WeakBlock::destroy(*this, block); - } else - m_indexOfNextLogicallyEmptyWeakBlockToSweep++; - - if (m_indexOfNextLogicallyEmptyWeakBlockToSweep >= m_logicallyEmptyWeakBlocks.size()) { - m_indexOfNextLogicallyEmptyWeakBlockToSweep = WTF::notFound; - return false; - } - - return true; -} - -size_t Heap::threadVisitCount() -{ - unsigned long result = 0; - for (auto& parallelVisitor : m_parallelSlotVisitors) - result += parallelVisitor->visitCount(); - return result; -} - -size_t Heap::threadBytesVisited() -{ - size_t result = 0; - for (auto& parallelVisitor : m_parallelSlotVisitors) - result += parallelVisitor->bytesVisited(); - return result; -} - -size_t Heap::threadBytesCopied() -{ - size_t result = 0; - for (auto& parallelVisitor : m_parallelSlotVisitors) - result += parallelVisitor->bytesCopied(); - return result; +#if ENABLE(GGC) + m_writeBarrierBuffer.flush(*this); + m_writeBarrierBuffer.add(cell); +#else + UNUSED_PARAM(cell); +#endif } } // namespace JSC |