// Copyright 2010 the V8 project authors. All rights reserved. // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are // met: // // * Redistributions of source code must retain the above copyright // notice, this list of conditions and the following disclaimer. // * Redistributions in binary form must reproduce the above // copyright notice, this list of conditions and the following // disclaimer in the documentation and/or other materials provided // with the distribution. // * Neither the name of Google Inc. nor the names of its // contributors may be used to endorse or promote products derived // from this software without specific prior written permission. // // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. #include "v8.h" #include "accessors.h" #include "api.h" #include "bootstrapper.h" #include "codegen-inl.h" #include "compilation-cache.h" #include "debug.h" #include "heap-profiler.h" #include "global-handles.h" #include "liveobjectlist-inl.h" #include "mark-compact.h" #include "natives.h" #include "objects-visiting.h" #include "runtime-profiler.h" #include "scanner-base.h" #include "scopeinfo.h" #include "snapshot.h" #include "v8threads.h" #include "vm-state-inl.h" #if V8_TARGET_ARCH_ARM && !V8_INTERPRETED_REGEXP #include "regexp-macro-assembler.h" #include "arm/regexp-macro-assembler-arm.h" #endif namespace v8 { namespace internal { String* Heap::hidden_symbol_; Object* Heap::roots_[Heap::kRootListLength]; Object* Heap::global_contexts_list_; NewSpace Heap::new_space_; OldSpace* Heap::old_pointer_space_ = NULL; OldSpace* Heap::old_data_space_ = NULL; OldSpace* Heap::code_space_ = NULL; MapSpace* Heap::map_space_ = NULL; CellSpace* Heap::cell_space_ = NULL; LargeObjectSpace* Heap::lo_space_ = NULL; static const intptr_t kMinimumPromotionLimit = 2 * MB; static const intptr_t kMinimumAllocationLimit = 8 * MB; intptr_t Heap::old_gen_promotion_limit_ = kMinimumPromotionLimit; intptr_t Heap::old_gen_allocation_limit_ = kMinimumAllocationLimit; int Heap::old_gen_exhausted_ = false; int Heap::amount_of_external_allocated_memory_ = 0; int Heap::amount_of_external_allocated_memory_at_last_global_gc_ = 0; // semispace_size_ should be a power of 2 and old_generation_size_ should be // a multiple of Page::kPageSize. #if defined(ANDROID) static const int default_max_semispace_size_ = 2*MB; intptr_t Heap::max_old_generation_size_ = 192*MB; int Heap::initial_semispace_size_ = 128*KB; intptr_t Heap::code_range_size_ = 0; intptr_t Heap::max_executable_size_ = max_old_generation_size_; #elif defined(V8_TARGET_ARCH_X64) static const int default_max_semispace_size_ = 16*MB; intptr_t Heap::max_old_generation_size_ = 1*GB; int Heap::initial_semispace_size_ = 1*MB; intptr_t Heap::code_range_size_ = 512*MB; intptr_t Heap::max_executable_size_ = 256*MB; #else static const int default_max_semispace_size_ = 8*MB; intptr_t Heap::max_old_generation_size_ = 512*MB; int Heap::initial_semispace_size_ = 512*KB; intptr_t Heap::code_range_size_ = 0; intptr_t Heap::max_executable_size_ = 128*MB; #endif // Allow build-time customization of the max semispace size. Building // V8 with snapshots and a non-default max semispace size is much // easier if you can define it as part of the build environment. #if defined(V8_MAX_SEMISPACE_SIZE) int Heap::max_semispace_size_ = V8_MAX_SEMISPACE_SIZE; #else int Heap::max_semispace_size_ = default_max_semispace_size_; #endif // The snapshot semispace size will be the default semispace size if // snapshotting is used and will be the requested semispace size as // set up by ConfigureHeap otherwise. int Heap::reserved_semispace_size_ = Heap::max_semispace_size_; List Heap::gc_prologue_callbacks_; List Heap::gc_epilogue_callbacks_; GCCallback Heap::global_gc_prologue_callback_ = NULL; GCCallback Heap::global_gc_epilogue_callback_ = NULL; HeapObjectCallback Heap::gc_safe_size_of_old_object_ = NULL; // Variables set based on semispace_size_ and old_generation_size_ in // ConfigureHeap. // Will be 4 * reserved_semispace_size_ to ensure that young // generation can be aligned to its size. int Heap::survived_since_last_expansion_ = 0; intptr_t Heap::external_allocation_limit_ = 0; Heap::HeapState Heap::gc_state_ = NOT_IN_GC; int Heap::mc_count_ = 0; int Heap::ms_count_ = 0; int Heap::gc_count_ = 0; GCTracer* Heap::tracer_ = NULL; int Heap::unflattened_strings_length_ = 0; int Heap::always_allocate_scope_depth_ = 0; int Heap::linear_allocation_scope_depth_ = 0; int Heap::contexts_disposed_ = 0; int Heap::young_survivors_after_last_gc_ = 0; int Heap::high_survival_rate_period_length_ = 0; double Heap::survival_rate_ = 0; Heap::SurvivalRateTrend Heap::previous_survival_rate_trend_ = Heap::STABLE; Heap::SurvivalRateTrend Heap::survival_rate_trend_ = Heap::STABLE; #ifdef DEBUG bool Heap::allocation_allowed_ = true; int Heap::allocation_timeout_ = 0; bool Heap::disallow_allocation_failure_ = false; #endif // DEBUG intptr_t GCTracer::alive_after_last_gc_ = 0; double GCTracer::last_gc_end_timestamp_ = 0.0; int GCTracer::max_gc_pause_ = 0; intptr_t GCTracer::max_alive_after_gc_ = 0; int GCTracer::min_in_mutator_ = kMaxInt; intptr_t Heap::Capacity() { if (!HasBeenSetup()) return 0; return new_space_.Capacity() + old_pointer_space_->Capacity() + old_data_space_->Capacity() + code_space_->Capacity() + map_space_->Capacity() + cell_space_->Capacity(); } intptr_t Heap::CommittedMemory() { if (!HasBeenSetup()) return 0; return new_space_.CommittedMemory() + old_pointer_space_->CommittedMemory() + old_data_space_->CommittedMemory() + code_space_->CommittedMemory() + map_space_->CommittedMemory() + cell_space_->CommittedMemory() + lo_space_->Size(); } intptr_t Heap::CommittedMemoryExecutable() { if (!HasBeenSetup()) return 0; return MemoryAllocator::SizeExecutable(); } intptr_t Heap::Available() { if (!HasBeenSetup()) return 0; return new_space_.Available() + old_pointer_space_->Available() + old_data_space_->Available() + code_space_->Available() + map_space_->Available() + cell_space_->Available(); } bool Heap::HasBeenSetup() { return old_pointer_space_ != NULL && old_data_space_ != NULL && code_space_ != NULL && map_space_ != NULL && cell_space_ != NULL && lo_space_ != NULL; } int Heap::GcSafeSizeOfOldObject(HeapObject* object) { ASSERT(!Heap::InNewSpace(object)); // Code only works for old objects. ASSERT(!MarkCompactCollector::are_map_pointers_encoded()); MapWord map_word = object->map_word(); map_word.ClearMark(); map_word.ClearOverflow(); return object->SizeFromMap(map_word.ToMap()); } int Heap::GcSafeSizeOfOldObjectWithEncodedMap(HeapObject* object) { ASSERT(!Heap::InNewSpace(object)); // Code only works for old objects. ASSERT(MarkCompactCollector::are_map_pointers_encoded()); uint32_t marker = Memory::uint32_at(object->address()); if (marker == MarkCompactCollector::kSingleFreeEncoding) { return kIntSize; } else if (marker == MarkCompactCollector::kMultiFreeEncoding) { return Memory::int_at(object->address() + kIntSize); } else { MapWord map_word = object->map_word(); Address map_address = map_word.DecodeMapAddress(Heap::map_space()); Map* map = reinterpret_cast(HeapObject::FromAddress(map_address)); return object->SizeFromMap(map); } } GarbageCollector Heap::SelectGarbageCollector(AllocationSpace space) { // Is global GC requested? if (space != NEW_SPACE || FLAG_gc_global) { Counters::gc_compactor_caused_by_request.Increment(); return MARK_COMPACTOR; } // Is enough data promoted to justify a global GC? if (OldGenerationPromotionLimitReached()) { Counters::gc_compactor_caused_by_promoted_data.Increment(); return MARK_COMPACTOR; } // Have allocation in OLD and LO failed? if (old_gen_exhausted_) { Counters::gc_compactor_caused_by_oldspace_exhaustion.Increment(); return MARK_COMPACTOR; } // Is there enough space left in OLD to guarantee that a scavenge can // succeed? // // Note that MemoryAllocator->MaxAvailable() undercounts the memory available // for object promotion. It counts only the bytes that the memory // allocator has not yet allocated from the OS and assigned to any space, // and does not count available bytes already in the old space or code // space. Undercounting is safe---we may get an unrequested full GC when // a scavenge would have succeeded. if (MemoryAllocator::MaxAvailable() <= new_space_.Size()) { Counters::gc_compactor_caused_by_oldspace_exhaustion.Increment(); return MARK_COMPACTOR; } // Default return SCAVENGER; } // TODO(1238405): Combine the infrastructure for --heap-stats and // --log-gc to avoid the complicated preprocessor and flag testing. #if defined(DEBUG) || defined(ENABLE_LOGGING_AND_PROFILING) void Heap::ReportStatisticsBeforeGC() { // Heap::ReportHeapStatistics will also log NewSpace statistics when // compiled with ENABLE_LOGGING_AND_PROFILING and --log-gc is set. The // following logic is used to avoid double logging. #if defined(DEBUG) && defined(ENABLE_LOGGING_AND_PROFILING) if (FLAG_heap_stats || FLAG_log_gc) new_space_.CollectStatistics(); if (FLAG_heap_stats) { ReportHeapStatistics("Before GC"); } else if (FLAG_log_gc) { new_space_.ReportStatistics(); } if (FLAG_heap_stats || FLAG_log_gc) new_space_.ClearHistograms(); #elif defined(DEBUG) if (FLAG_heap_stats) { new_space_.CollectStatistics(); ReportHeapStatistics("Before GC"); new_space_.ClearHistograms(); } #elif defined(ENABLE_LOGGING_AND_PROFILING) if (FLAG_log_gc) { new_space_.CollectStatistics(); new_space_.ReportStatistics(); new_space_.ClearHistograms(); } #endif } #if defined(ENABLE_LOGGING_AND_PROFILING) void Heap::PrintShortHeapStatistics() { if (!FLAG_trace_gc_verbose) return; PrintF("Memory allocator, used: %8" V8_PTR_PREFIX "d" ", available: %8" V8_PTR_PREFIX "d\n", MemoryAllocator::Size(), MemoryAllocator::Available()); PrintF("New space, used: %8" V8_PTR_PREFIX "d" ", available: %8" V8_PTR_PREFIX "d\n", Heap::new_space_.Size(), new_space_.Available()); PrintF("Old pointers, used: %8" V8_PTR_PREFIX "d" ", available: %8" V8_PTR_PREFIX "d" ", waste: %8" V8_PTR_PREFIX "d\n", old_pointer_space_->Size(), old_pointer_space_->Available(), old_pointer_space_->Waste()); PrintF("Old data space, used: %8" V8_PTR_PREFIX "d" ", available: %8" V8_PTR_PREFIX "d" ", waste: %8" V8_PTR_PREFIX "d\n", old_data_space_->Size(), old_data_space_->Available(), old_data_space_->Waste()); PrintF("Code space, used: %8" V8_PTR_PREFIX "d" ", available: %8" V8_PTR_PREFIX "d" ", waste: %8" V8_PTR_PREFIX "d\n", code_space_->Size(), code_space_->Available(), code_space_->Waste()); PrintF("Map space, used: %8" V8_PTR_PREFIX "d" ", available: %8" V8_PTR_PREFIX "d" ", waste: %8" V8_PTR_PREFIX "d\n", map_space_->Size(), map_space_->Available(), map_space_->Waste()); PrintF("Cell space, used: %8" V8_PTR_PREFIX "d" ", available: %8" V8_PTR_PREFIX "d" ", waste: %8" V8_PTR_PREFIX "d\n", cell_space_->Size(), cell_space_->Available(), cell_space_->Waste()); PrintF("Large object space, used: %8" V8_PTR_PREFIX "d" ", available: %8" V8_PTR_PREFIX "d\n", lo_space_->Size(), lo_space_->Available()); } #endif // TODO(1238405): Combine the infrastructure for --heap-stats and // --log-gc to avoid the complicated preprocessor and flag testing. void Heap::ReportStatisticsAfterGC() { // Similar to the before GC, we use some complicated logic to ensure that // NewSpace statistics are logged exactly once when --log-gc is turned on. #if defined(DEBUG) && defined(ENABLE_LOGGING_AND_PROFILING) if (FLAG_heap_stats) { new_space_.CollectStatistics(); ReportHeapStatistics("After GC"); } else if (FLAG_log_gc) { new_space_.ReportStatistics(); } #elif defined(DEBUG) if (FLAG_heap_stats) ReportHeapStatistics("After GC"); #elif defined(ENABLE_LOGGING_AND_PROFILING) if (FLAG_log_gc) new_space_.ReportStatistics(); #endif } #endif // defined(DEBUG) || defined(ENABLE_LOGGING_AND_PROFILING) void Heap::GarbageCollectionPrologue() { TranscendentalCache::Clear(); ClearJSFunctionResultCaches(); gc_count_++; unflattened_strings_length_ = 0; #ifdef DEBUG ASSERT(allocation_allowed_ && gc_state_ == NOT_IN_GC); allow_allocation(false); if (FLAG_verify_heap) { Verify(); } if (FLAG_gc_verbose) Print(); #endif #if defined(DEBUG) || defined(ENABLE_LOGGING_AND_PROFILING) ReportStatisticsBeforeGC(); #endif LiveObjectList::GCPrologue(); } intptr_t Heap::SizeOfObjects() { intptr_t total = 0; AllSpaces spaces; for (Space* space = spaces.next(); space != NULL; space = spaces.next()) { total += space->SizeOfObjects(); } return total; } void Heap::GarbageCollectionEpilogue() { LiveObjectList::GCEpilogue(); #ifdef DEBUG allow_allocation(true); ZapFromSpace(); if (FLAG_verify_heap) { Verify(); } if (FLAG_print_global_handles) GlobalHandles::Print(); if (FLAG_print_handles) PrintHandles(); if (FLAG_gc_verbose) Print(); if (FLAG_code_stats) ReportCodeStatistics("After GC"); #endif Counters::alive_after_last_gc.Set(static_cast(SizeOfObjects())); Counters::symbol_table_capacity.Set(symbol_table()->Capacity()); Counters::number_of_symbols.Set(symbol_table()->NumberOfElements()); #if defined(DEBUG) || defined(ENABLE_LOGGING_AND_PROFILING) ReportStatisticsAfterGC(); #endif #ifdef ENABLE_DEBUGGER_SUPPORT Debug::AfterGarbageCollection(); #endif } void Heap::CollectAllGarbage(bool force_compaction) { // Since we are ignoring the return value, the exact choice of space does // not matter, so long as we do not specify NEW_SPACE, which would not // cause a full GC. MarkCompactCollector::SetForceCompaction(force_compaction); CollectGarbage(OLD_POINTER_SPACE); MarkCompactCollector::SetForceCompaction(false); } void Heap::CollectAllAvailableGarbage() { // Since we are ignoring the return value, the exact choice of space does // not matter, so long as we do not specify NEW_SPACE, which would not // cause a full GC. MarkCompactCollector::SetForceCompaction(true); // Major GC would invoke weak handle callbacks on weakly reachable // handles, but won't collect weakly reachable objects until next // major GC. Therefore if we collect aggressively and weak handle callback // has been invoked, we rerun major GC to release objects which become // garbage. // Note: as weak callbacks can execute arbitrary code, we cannot // hope that eventually there will be no weak callbacks invocations. // Therefore stop recollecting after several attempts. const int kMaxNumberOfAttempts = 7; for (int attempt = 0; attempt < kMaxNumberOfAttempts; attempt++) { if (!CollectGarbage(OLD_POINTER_SPACE, MARK_COMPACTOR)) { break; } } MarkCompactCollector::SetForceCompaction(false); } bool Heap::CollectGarbage(AllocationSpace space, GarbageCollector collector) { // The VM is in the GC state until exiting this function. VMState state(GC); #ifdef DEBUG // Reset the allocation timeout to the GC interval, but make sure to // allow at least a few allocations after a collection. The reason // for this is that we have a lot of allocation sequences and we // assume that a garbage collection will allow the subsequent // allocation attempts to go through. allocation_timeout_ = Max(6, FLAG_gc_interval); #endif bool next_gc_likely_to_collect_more = false; { GCTracer tracer; GarbageCollectionPrologue(); // The GC count was incremented in the prologue. Tell the tracer about // it. tracer.set_gc_count(gc_count_); // Tell the tracer which collector we've selected. tracer.set_collector(collector); HistogramTimer* rate = (collector == SCAVENGER) ? &Counters::gc_scavenger : &Counters::gc_compactor; rate->Start(); next_gc_likely_to_collect_more = PerformGarbageCollection(collector, &tracer); rate->Stop(); GarbageCollectionEpilogue(); } #ifdef ENABLE_LOGGING_AND_PROFILING if (FLAG_log_gc) HeapProfiler::WriteSample(); if (CpuProfiler::is_profiling()) CpuProfiler::ProcessMovedFunctions(); #endif return next_gc_likely_to_collect_more; } void Heap::PerformScavenge() { GCTracer tracer; PerformGarbageCollection(SCAVENGER, &tracer); } #ifdef DEBUG // Helper class for verifying the symbol table. class SymbolTableVerifier : public ObjectVisitor { public: SymbolTableVerifier() { } void VisitPointers(Object** start, Object** end) { // Visit all HeapObject pointers in [start, end). for (Object** p = start; p < end; p++) { if ((*p)->IsHeapObject()) { // Check that the symbol is actually a symbol. ASSERT((*p)->IsNull() || (*p)->IsUndefined() || (*p)->IsSymbol()); } } } }; #endif // DEBUG static void VerifySymbolTable() { #ifdef DEBUG SymbolTableVerifier verifier; Heap::symbol_table()->IterateElements(&verifier); #endif // DEBUG } void Heap::ReserveSpace( int new_space_size, int pointer_space_size, int data_space_size, int code_space_size, int map_space_size, int cell_space_size, int large_object_size) { NewSpace* new_space = Heap::new_space(); PagedSpace* old_pointer_space = Heap::old_pointer_space(); PagedSpace* old_data_space = Heap::old_data_space(); PagedSpace* code_space = Heap::code_space(); PagedSpace* map_space = Heap::map_space(); PagedSpace* cell_space = Heap::cell_space(); LargeObjectSpace* lo_space = Heap::lo_space(); bool gc_performed = true; while (gc_performed) { gc_performed = false; if (!new_space->ReserveSpace(new_space_size)) { Heap::CollectGarbage(NEW_SPACE); gc_performed = true; } if (!old_pointer_space->ReserveSpace(pointer_space_size)) { Heap::CollectGarbage(OLD_POINTER_SPACE); gc_performed = true; } if (!(old_data_space->ReserveSpace(data_space_size))) { Heap::CollectGarbage(OLD_DATA_SPACE); gc_performed = true; } if (!(code_space->ReserveSpace(code_space_size))) { Heap::CollectGarbage(CODE_SPACE); gc_performed = true; } if (!(map_space->ReserveSpace(map_space_size))) { Heap::CollectGarbage(MAP_SPACE); gc_performed = true; } if (!(cell_space->ReserveSpace(cell_space_size))) { Heap::CollectGarbage(CELL_SPACE); gc_performed = true; } // We add a slack-factor of 2 in order to have space for a series of // large-object allocations that are only just larger than the page size. large_object_size *= 2; // The ReserveSpace method on the large object space checks how much // we can expand the old generation. This includes expansion caused by // allocation in the other spaces. large_object_size += cell_space_size + map_space_size + code_space_size + data_space_size + pointer_space_size; if (!(lo_space->ReserveSpace(large_object_size))) { Heap::CollectGarbage(LO_SPACE); gc_performed = true; } } } void Heap::EnsureFromSpaceIsCommitted() { if (new_space_.CommitFromSpaceIfNeeded()) return; // Committing memory to from space failed. // Try shrinking and try again. PagedSpaces spaces; for (PagedSpace* space = spaces.next(); space != NULL; space = spaces.next()) { space->RelinkPageListInChunkOrder(true); } Shrink(); if (new_space_.CommitFromSpaceIfNeeded()) return; // Committing memory to from space failed again. // Memory is exhausted and we will die. V8::FatalProcessOutOfMemory("Committing semi space failed."); } void Heap::ClearJSFunctionResultCaches() { if (Bootstrapper::IsActive()) return; Object* context = global_contexts_list_; while (!context->IsUndefined()) { // Get the caches for this context: FixedArray* caches = Context::cast(context)->jsfunction_result_caches(); // Clear the caches: int length = caches->length(); for (int i = 0; i < length; i++) { JSFunctionResultCache::cast(caches->get(i))->Clear(); } // Get the next context: context = Context::cast(context)->get(Context::NEXT_CONTEXT_LINK); } } void Heap::ClearNormalizedMapCaches() { if (Bootstrapper::IsActive()) return; Object* context = global_contexts_list_; while (!context->IsUndefined()) { Context::cast(context)->normalized_map_cache()->Clear(); context = Context::cast(context)->get(Context::NEXT_CONTEXT_LINK); } } #ifdef DEBUG enum PageWatermarkValidity { ALL_VALID, ALL_INVALID }; static void VerifyPageWatermarkValidity(PagedSpace* space, PageWatermarkValidity validity) { PageIterator it(space, PageIterator::PAGES_IN_USE); bool expected_value = (validity == ALL_VALID); while (it.has_next()) { Page* page = it.next(); ASSERT(page->IsWatermarkValid() == expected_value); } } #endif void Heap::UpdateSurvivalRateTrend(int start_new_space_size) { double survival_rate = (static_cast(young_survivors_after_last_gc_) * 100) / start_new_space_size; if (survival_rate > kYoungSurvivalRateThreshold) { high_survival_rate_period_length_++; } else { high_survival_rate_period_length_ = 0; } double survival_rate_diff = survival_rate_ - survival_rate; if (survival_rate_diff > kYoungSurvivalRateAllowedDeviation) { set_survival_rate_trend(DECREASING); } else if (survival_rate_diff < -kYoungSurvivalRateAllowedDeviation) { set_survival_rate_trend(INCREASING); } else { set_survival_rate_trend(STABLE); } survival_rate_ = survival_rate; } bool Heap::PerformGarbageCollection(GarbageCollector collector, GCTracer* tracer) { bool next_gc_likely_to_collect_more = false; if (collector != SCAVENGER) { PROFILE(CodeMovingGCEvent()); } VerifySymbolTable(); if (collector == MARK_COMPACTOR && global_gc_prologue_callback_) { ASSERT(!allocation_allowed_); GCTracer::Scope scope(tracer, GCTracer::Scope::EXTERNAL); global_gc_prologue_callback_(); } GCType gc_type = collector == MARK_COMPACTOR ? kGCTypeMarkSweepCompact : kGCTypeScavenge; for (int i = 0; i < gc_prologue_callbacks_.length(); ++i) { if (gc_type & gc_prologue_callbacks_[i].gc_type) { gc_prologue_callbacks_[i].callback(gc_type, kNoGCCallbackFlags); } } EnsureFromSpaceIsCommitted(); int start_new_space_size = Heap::new_space()->SizeAsInt(); if (collector == MARK_COMPACTOR) { // Perform mark-sweep with optional compaction. MarkCompact(tracer); bool high_survival_rate_during_scavenges = IsHighSurvivalRate() && IsStableOrIncreasingSurvivalTrend(); UpdateSurvivalRateTrend(start_new_space_size); intptr_t old_gen_size = PromotedSpaceSize(); old_gen_promotion_limit_ = old_gen_size + Max(kMinimumPromotionLimit, old_gen_size / 3); old_gen_allocation_limit_ = old_gen_size + Max(kMinimumAllocationLimit, old_gen_size / 2); if (high_survival_rate_during_scavenges && IsStableOrIncreasingSurvivalTrend()) { // Stable high survival rates of young objects both during partial and // full collection indicate that mutator is either building or modifying // a structure with a long lifetime. // In this case we aggressively raise old generation memory limits to // postpone subsequent mark-sweep collection and thus trade memory // space for the mutation speed. old_gen_promotion_limit_ *= 2; old_gen_allocation_limit_ *= 2; } old_gen_exhausted_ = false; } else { tracer_ = tracer; Scavenge(); tracer_ = NULL; UpdateSurvivalRateTrend(start_new_space_size); } Counters::objs_since_last_young.Set(0); if (collector == MARK_COMPACTOR) { DisableAssertNoAllocation allow_allocation; GCTracer::Scope scope(tracer, GCTracer::Scope::EXTERNAL); next_gc_likely_to_collect_more = GlobalHandles::PostGarbageCollectionProcessing(); } // Update relocatables. Relocatable::PostGarbageCollectionProcessing(); if (collector == MARK_COMPACTOR) { // Register the amount of external allocated memory. amount_of_external_allocated_memory_at_last_global_gc_ = amount_of_external_allocated_memory_; } GCCallbackFlags callback_flags = tracer->is_compacting() ? kGCCallbackFlagCompacted : kNoGCCallbackFlags; for (int i = 0; i < gc_epilogue_callbacks_.length(); ++i) { if (gc_type & gc_epilogue_callbacks_[i].gc_type) { gc_epilogue_callbacks_[i].callback(gc_type, callback_flags); } } if (collector == MARK_COMPACTOR && global_gc_epilogue_callback_) { ASSERT(!allocation_allowed_); GCTracer::Scope scope(tracer, GCTracer::Scope::EXTERNAL); global_gc_epilogue_callback_(); } VerifySymbolTable(); return next_gc_likely_to_collect_more; } void Heap::MarkCompact(GCTracer* tracer) { gc_state_ = MARK_COMPACT; LOG(ResourceEvent("markcompact", "begin")); MarkCompactCollector::Prepare(tracer); bool is_compacting = MarkCompactCollector::IsCompacting(); if (is_compacting) { mc_count_++; } else { ms_count_++; } tracer->set_full_gc_count(mc_count_ + ms_count_); MarkCompactPrologue(is_compacting); MarkCompactCollector::CollectGarbage(); LOG(ResourceEvent("markcompact", "end")); gc_state_ = NOT_IN_GC; Shrink(); Counters::objs_since_last_full.Set(0); contexts_disposed_ = 0; } void Heap::MarkCompactPrologue(bool is_compacting) { // At any old GC clear the keyed lookup cache to enable collection of unused // maps. KeyedLookupCache::Clear(); ContextSlotCache::Clear(); DescriptorLookupCache::Clear(); RuntimeProfiler::MarkCompactPrologue(is_compacting); CompilationCache::MarkCompactPrologue(); CompletelyClearInstanceofCache(); if (is_compacting) FlushNumberStringCache(); ClearNormalizedMapCaches(); } Object* Heap::FindCodeObject(Address a) { Object* obj = NULL; // Initialization to please compiler. { MaybeObject* maybe_obj = code_space_->FindObject(a); if (!maybe_obj->ToObject(&obj)) { obj = lo_space_->FindObject(a)->ToObjectUnchecked(); } } return obj; } // Helper class for copying HeapObjects class ScavengeVisitor: public ObjectVisitor { public: void VisitPointer(Object** p) { ScavengePointer(p); } void VisitPointers(Object** start, Object** end) { // Copy all HeapObject pointers in [start, end) for (Object** p = start; p < end; p++) ScavengePointer(p); } private: void ScavengePointer(Object** p) { Object* object = *p; if (!Heap::InNewSpace(object)) return; Heap::ScavengeObject(reinterpret_cast(p), reinterpret_cast(object)); } }; // A queue of objects promoted during scavenge. Each object is accompanied // by it's size to avoid dereferencing a map pointer for scanning. class PromotionQueue { public: void Initialize(Address start_address) { front_ = rear_ = reinterpret_cast(start_address); } bool is_empty() { return front_ <= rear_; } void insert(HeapObject* target, int size) { *(--rear_) = reinterpret_cast(target); *(--rear_) = size; // Assert no overflow into live objects. ASSERT(reinterpret_cast
(rear_) >= Heap::new_space()->top()); } void remove(HeapObject** target, int* size) { *target = reinterpret_cast(*(--front_)); *size = static_cast(*(--front_)); // Assert no underflow. ASSERT(front_ >= rear_); } private: // The front of the queue is higher in memory than the rear. intptr_t* front_; intptr_t* rear_; }; // Shared state read by the scavenge collector and set by ScavengeObject. static PromotionQueue promotion_queue; #ifdef DEBUG // Visitor class to verify pointers in code or data space do not point into // new space. class VerifyNonPointerSpacePointersVisitor: public ObjectVisitor { public: void VisitPointers(Object** start, Object**end) { for (Object** current = start; current < end; current++) { if ((*current)->IsHeapObject()) { ASSERT(!Heap::InNewSpace(HeapObject::cast(*current))); } } } }; static void VerifyNonPointerSpacePointers() { // Verify that there are no pointers to new space in spaces where we // do not expect them. VerifyNonPointerSpacePointersVisitor v; HeapObjectIterator code_it(Heap::code_space()); for (HeapObject* object = code_it.next(); object != NULL; object = code_it.next()) object->Iterate(&v); HeapObjectIterator data_it(Heap::old_data_space()); for (HeapObject* object = data_it.next(); object != NULL; object = data_it.next()) object->Iterate(&v); } #endif void Heap::CheckNewSpaceExpansionCriteria() { if (new_space_.Capacity() < new_space_.MaximumCapacity() && survived_since_last_expansion_ > new_space_.Capacity()) { // Grow the size of new space if there is room to grow and enough // data has survived scavenge since the last expansion. new_space_.Grow(); survived_since_last_expansion_ = 0; } } void Heap::Scavenge() { #ifdef DEBUG if (FLAG_enable_slow_asserts) VerifyNonPointerSpacePointers(); #endif gc_state_ = SCAVENGE; Page::FlipMeaningOfInvalidatedWatermarkFlag(); #ifdef DEBUG VerifyPageWatermarkValidity(old_pointer_space_, ALL_VALID); VerifyPageWatermarkValidity(map_space_, ALL_VALID); #endif // We do not update an allocation watermark of the top page during linear // allocation to avoid overhead. So to maintain the watermark invariant // we have to manually cache the watermark and mark the top page as having an // invalid watermark. This guarantees that dirty regions iteration will use a // correct watermark even if a linear allocation happens. old_pointer_space_->FlushTopPageWatermark(); map_space_->FlushTopPageWatermark(); // Implements Cheney's copying algorithm LOG(ResourceEvent("scavenge", "begin")); // Clear descriptor cache. DescriptorLookupCache::Clear(); // Used for updating survived_since_last_expansion_ at function end. intptr_t survived_watermark = PromotedSpaceSize(); CheckNewSpaceExpansionCriteria(); // Flip the semispaces. After flipping, to space is empty, from space has // live objects. new_space_.Flip(); new_space_.ResetAllocationInfo(); // We need to sweep newly copied objects which can be either in the // to space or promoted to the old generation. For to-space // objects, we treat the bottom of the to space as a queue. Newly // copied and unswept objects lie between a 'front' mark and the // allocation pointer. // // Promoted objects can go into various old-generation spaces, and // can be allocated internally in the spaces (from the free list). // We treat the top of the to space as a queue of addresses of // promoted objects. The addresses of newly promoted and unswept // objects lie between a 'front' mark and a 'rear' mark that is // updated as a side effect of promoting an object. // // There is guaranteed to be enough room at the top of the to space // for the addresses of promoted objects: every object promoted // frees up its size in bytes from the top of the new space, and // objects are at least one pointer in size. Address new_space_front = new_space_.ToSpaceLow(); promotion_queue.Initialize(new_space_.ToSpaceHigh()); ScavengeVisitor scavenge_visitor; // Copy roots. IterateRoots(&scavenge_visitor, VISIT_ALL_IN_SCAVENGE); // Copy objects reachable from the old generation. By definition, // there are no intergenerational pointers in code or data spaces. IterateDirtyRegions(old_pointer_space_, &IteratePointersInDirtyRegion, &ScavengePointer, WATERMARK_CAN_BE_INVALID); IterateDirtyRegions(map_space_, &IteratePointersInDirtyMapsRegion, &ScavengePointer, WATERMARK_CAN_BE_INVALID); lo_space_->IterateDirtyRegions(&ScavengePointer); // Copy objects reachable from cells by scavenging cell values directly. HeapObjectIterator cell_iterator(cell_space_); for (HeapObject* cell = cell_iterator.next(); cell != NULL; cell = cell_iterator.next()) { if (cell->IsJSGlobalPropertyCell()) { Address value_address = reinterpret_cast
(cell) + (JSGlobalPropertyCell::kValueOffset - kHeapObjectTag); scavenge_visitor.VisitPointer(reinterpret_cast(value_address)); } } // Scavenge object reachable from the global contexts list directly. scavenge_visitor.VisitPointer(BitCast(&global_contexts_list_)); // Scavenge objects reachable from the runtime-profiler sampler // window directly. Object** sampler_window_address = RuntimeProfiler::SamplerWindowAddress(); int sampler_window_size = RuntimeProfiler::SamplerWindowSize(); scavenge_visitor.VisitPointers( sampler_window_address, sampler_window_address + sampler_window_size); new_space_front = DoScavenge(&scavenge_visitor, new_space_front); UpdateNewSpaceReferencesInExternalStringTable( &UpdateNewSpaceReferenceInExternalStringTableEntry); LiveObjectList::UpdateReferencesForScavengeGC(); ASSERT(new_space_front == new_space_.top()); // Set age mark. new_space_.set_age_mark(new_space_.top()); // Update how much has survived scavenge. IncrementYoungSurvivorsCounter(static_cast( (PromotedSpaceSize() - survived_watermark) + new_space_.Size())); LOG(ResourceEvent("scavenge", "end")); gc_state_ = NOT_IN_GC; } String* Heap::UpdateNewSpaceReferenceInExternalStringTableEntry(Object** p) { MapWord first_word = HeapObject::cast(*p)->map_word(); if (!first_word.IsForwardingAddress()) { // Unreachable external string can be finalized. FinalizeExternalString(String::cast(*p)); return NULL; } // String is still reachable. return String::cast(first_word.ToForwardingAddress()); } void Heap::UpdateNewSpaceReferencesInExternalStringTable( ExternalStringTableUpdaterCallback updater_func) { ExternalStringTable::Verify(); if (ExternalStringTable::new_space_strings_.is_empty()) return; Object** start = &ExternalStringTable::new_space_strings_[0]; Object** end = start + ExternalStringTable::new_space_strings_.length(); Object** last = start; for (Object** p = start; p < end; ++p) { ASSERT(Heap::InFromSpace(*p)); String* target = updater_func(p); if (target == NULL) continue; ASSERT(target->IsExternalString()); if (Heap::InNewSpace(target)) { // String is still in new space. Update the table entry. *last = target; ++last; } else { // String got promoted. Move it to the old string list. ExternalStringTable::AddOldString(target); } } ASSERT(last <= end); ExternalStringTable::ShrinkNewStrings(static_cast(last - start)); } static Object* ProcessFunctionWeakReferences(Object* function, WeakObjectRetainer* retainer) { Object* head = Heap::undefined_value(); JSFunction* tail = NULL; Object* candidate = function; while (!candidate->IsUndefined()) { // Check whether to keep the candidate in the list. JSFunction* candidate_function = reinterpret_cast(candidate); Object* retain = retainer->RetainAs(candidate); if (retain != NULL) { if (head->IsUndefined()) { // First element in the list. head = candidate_function; } else { // Subsequent elements in the list. ASSERT(tail != NULL); tail->set_next_function_link(candidate_function); } // Retained function is new tail. tail = candidate_function; } // Move to next element in the list. candidate = candidate_function->next_function_link(); } // Terminate the list if there is one or more elements. if (tail != NULL) { tail->set_next_function_link(Heap::undefined_value()); } return head; } void Heap::ProcessWeakReferences(WeakObjectRetainer* retainer) { Object* head = undefined_value(); Context* tail = NULL; Object* candidate = global_contexts_list_; while (!candidate->IsUndefined()) { // Check whether to keep the candidate in the list. Context* candidate_context = reinterpret_cast(candidate); Object* retain = retainer->RetainAs(candidate); if (retain != NULL) { if (head->IsUndefined()) { // First element in the list. head = candidate_context; } else { // Subsequent elements in the list. ASSERT(tail != NULL); tail->set_unchecked(Context::NEXT_CONTEXT_LINK, candidate_context, UPDATE_WRITE_BARRIER); } // Retained context is new tail. tail = candidate_context; // Process the weak list of optimized functions for the context. Object* function_list_head = ProcessFunctionWeakReferences( candidate_context->get(Context::OPTIMIZED_FUNCTIONS_LIST), retainer); candidate_context->set_unchecked(Context::OPTIMIZED_FUNCTIONS_LIST, function_list_head, UPDATE_WRITE_BARRIER); } // Move to next element in the list. candidate = candidate_context->get(Context::NEXT_CONTEXT_LINK); } // Terminate the list if there is one or more elements. if (tail != NULL) { tail->set_unchecked(Context::NEXT_CONTEXT_LINK, Heap::undefined_value(), UPDATE_WRITE_BARRIER); } // Update the head of the list of contexts. Heap::global_contexts_list_ = head; } class NewSpaceScavenger : public StaticNewSpaceVisitor { public: static inline void VisitPointer(Object** p) { Object* object = *p; if (!Heap::InNewSpace(object)) return; Heap::ScavengeObject(reinterpret_cast(p), reinterpret_cast(object)); } }; Address Heap::DoScavenge(ObjectVisitor* scavenge_visitor, Address new_space_front) { do { ASSERT(new_space_front <= new_space_.top()); // The addresses new_space_front and new_space_.top() define a // queue of unprocessed copied objects. Process them until the // queue is empty. while (new_space_front < new_space_.top()) { HeapObject* object = HeapObject::FromAddress(new_space_front); new_space_front += NewSpaceScavenger::IterateBody(object->map(), object); } // Promote and process all the to-be-promoted objects. while (!promotion_queue.is_empty()) { HeapObject* target; int size; promotion_queue.remove(&target, &size); // Promoted object might be already partially visited // during dirty regions iteration. Thus we search specificly // for pointers to from semispace instead of looking for pointers // to new space. ASSERT(!target->IsMap()); IterateAndMarkPointersToFromSpace(target->address(), target->address() + size, &ScavengePointer); } // Take another spin if there are now unswept objects in new space // (there are currently no more unswept promoted objects). } while (new_space_front < new_space_.top()); return new_space_front; } class ScavengingVisitor : public StaticVisitorBase { public: static void Initialize() { table_.Register(kVisitSeqAsciiString, &EvacuateSeqAsciiString); table_.Register(kVisitSeqTwoByteString, &EvacuateSeqTwoByteString); table_.Register(kVisitShortcutCandidate, &EvacuateShortcutCandidate); table_.Register(kVisitByteArray, &EvacuateByteArray); table_.Register(kVisitFixedArray, &EvacuateFixedArray); table_.Register(kVisitGlobalContext, &ObjectEvacuationStrategy:: VisitSpecialized); typedef ObjectEvacuationStrategy PointerObject; table_.Register(kVisitConsString, &ObjectEvacuationStrategy:: VisitSpecialized); table_.Register(kVisitSharedFunctionInfo, &ObjectEvacuationStrategy:: VisitSpecialized); table_.Register(kVisitJSFunction, &ObjectEvacuationStrategy:: VisitSpecialized); table_.RegisterSpecializations, kVisitDataObject, kVisitDataObjectGeneric>(); table_.RegisterSpecializations, kVisitJSObject, kVisitJSObjectGeneric>(); table_.RegisterSpecializations, kVisitStruct, kVisitStructGeneric>(); } static inline void Scavenge(Map* map, HeapObject** slot, HeapObject* obj) { table_.GetVisitor(map)(map, slot, obj); } private: enum ObjectContents { DATA_OBJECT, POINTER_OBJECT }; enum SizeRestriction { SMALL, UNKNOWN_SIZE }; #if defined(DEBUG) || defined(ENABLE_LOGGING_AND_PROFILING) static void RecordCopiedObject(HeapObject* obj) { bool should_record = false; #ifdef DEBUG should_record = FLAG_heap_stats; #endif #ifdef ENABLE_LOGGING_AND_PROFILING should_record = should_record || FLAG_log_gc; #endif if (should_record) { if (Heap::new_space()->Contains(obj)) { Heap::new_space()->RecordAllocation(obj); } else { Heap::new_space()->RecordPromotion(obj); } } } #endif // defined(DEBUG) || defined(ENABLE_LOGGING_AND_PROFILING) // Helper function used by CopyObject to copy a source object to an // allocated target object and update the forwarding pointer in the source // object. Returns the target object. INLINE(static HeapObject* MigrateObject(HeapObject* source, HeapObject* target, int size)) { // Copy the content of source to target. Heap::CopyBlock(target->address(), source->address(), size); // Set the forwarding address. source->set_map_word(MapWord::FromForwardingAddress(target)); #if defined(DEBUG) || defined(ENABLE_LOGGING_AND_PROFILING) // Update NewSpace stats if necessary. RecordCopiedObject(target); #endif HEAP_PROFILE(ObjectMoveEvent(source->address(), target->address())); #if defined(ENABLE_LOGGING_AND_PROFILING) if (Logger::is_logging() || CpuProfiler::is_profiling()) { if (target->IsJSFunction()) { PROFILE(FunctionMoveEvent(source->address(), target->address())); PROFILE(FunctionCreateEventFromMove(JSFunction::cast(target))); } } #endif return target; } template static inline void EvacuateObject(Map* map, HeapObject** slot, HeapObject* object, int object_size) { ASSERT((size_restriction != SMALL) || (object_size <= Page::kMaxHeapObjectSize)); ASSERT(object->Size() == object_size); if (Heap::ShouldBePromoted(object->address(), object_size)) { MaybeObject* maybe_result; if ((size_restriction != SMALL) && (object_size > Page::kMaxHeapObjectSize)) { maybe_result = Heap::lo_space()->AllocateRawFixedArray(object_size); } else { if (object_contents == DATA_OBJECT) { maybe_result = Heap::old_data_space()->AllocateRaw(object_size); } else { maybe_result = Heap::old_pointer_space()->AllocateRaw(object_size); } } Object* result = NULL; // Initialization to please compiler. if (maybe_result->ToObject(&result)) { HeapObject* target = HeapObject::cast(result); *slot = MigrateObject(object, target, object_size); if (object_contents == POINTER_OBJECT) { promotion_queue.insert(target, object_size); } Heap::tracer()->increment_promoted_objects_size(object_size); return; } } Object* result = Heap::new_space()->AllocateRaw(object_size)->ToObjectUnchecked(); *slot = MigrateObject(object, HeapObject::cast(result), object_size); return; } static inline void EvacuateFixedArray(Map* map, HeapObject** slot, HeapObject* object) { int object_size = FixedArray::BodyDescriptor::SizeOf(map, object); EvacuateObject(map, slot, object, object_size); } static inline void EvacuateByteArray(Map* map, HeapObject** slot, HeapObject* object) { int object_size = reinterpret_cast(object)->ByteArraySize(); EvacuateObject(map, slot, object, object_size); } static inline void EvacuateSeqAsciiString(Map* map, HeapObject** slot, HeapObject* object) { int object_size = SeqAsciiString::cast(object)-> SeqAsciiStringSize(map->instance_type()); EvacuateObject(map, slot, object, object_size); } static inline void EvacuateSeqTwoByteString(Map* map, HeapObject** slot, HeapObject* object) { int object_size = SeqTwoByteString::cast(object)-> SeqTwoByteStringSize(map->instance_type()); EvacuateObject(map, slot, object, object_size); } static inline bool IsShortcutCandidate(int type) { return ((type & kShortcutTypeMask) == kShortcutTypeTag); } static inline void EvacuateShortcutCandidate(Map* map, HeapObject** slot, HeapObject* object) { ASSERT(IsShortcutCandidate(map->instance_type())); if (ConsString::cast(object)->unchecked_second() == Heap::empty_string()) { HeapObject* first = HeapObject::cast(ConsString::cast(object)->unchecked_first()); *slot = first; if (!Heap::InNewSpace(first)) { object->set_map_word(MapWord::FromForwardingAddress(first)); return; } MapWord first_word = first->map_word(); if (first_word.IsForwardingAddress()) { HeapObject* target = first_word.ToForwardingAddress(); *slot = target; object->set_map_word(MapWord::FromForwardingAddress(target)); return; } Scavenge(first->map(), slot, first); object->set_map_word(MapWord::FromForwardingAddress(*slot)); return; } int object_size = ConsString::kSize; EvacuateObject(map, slot, object, object_size); } template class ObjectEvacuationStrategy { public: template static inline void VisitSpecialized(Map* map, HeapObject** slot, HeapObject* object) { EvacuateObject(map, slot, object, object_size); } static inline void Visit(Map* map, HeapObject** slot, HeapObject* object) { int object_size = map->instance_size(); EvacuateObject(map, slot, object, object_size); } }; typedef void (*Callback)(Map* map, HeapObject** slot, HeapObject* object); static VisitorDispatchTable table_; }; VisitorDispatchTable ScavengingVisitor::table_; void Heap::ScavengeObjectSlow(HeapObject** p, HeapObject* object) { ASSERT(InFromSpace(object)); MapWord first_word = object->map_word(); ASSERT(!first_word.IsForwardingAddress()); Map* map = first_word.ToMap(); ScavengingVisitor::Scavenge(map, p, object); } void Heap::ScavengePointer(HeapObject** p) { ScavengeObject(p, *p); } MaybeObject* Heap::AllocatePartialMap(InstanceType instance_type, int instance_size) { Object* result; { MaybeObject* maybe_result = AllocateRawMap(); if (!maybe_result->ToObject(&result)) return maybe_result; } // Map::cast cannot be used due to uninitialized map field. reinterpret_cast(result)->set_map(raw_unchecked_meta_map()); reinterpret_cast(result)->set_instance_type(instance_type); reinterpret_cast(result)->set_instance_size(instance_size); reinterpret_cast(result)-> set_visitor_id( StaticVisitorBase::GetVisitorId(instance_type, instance_size)); reinterpret_cast(result)->set_inobject_properties(0); reinterpret_cast(result)->set_pre_allocated_property_fields(0); reinterpret_cast(result)->set_unused_property_fields(0); reinterpret_cast(result)->set_bit_field(0); reinterpret_cast(result)->set_bit_field2(0); return result; } MaybeObject* Heap::AllocateMap(InstanceType instance_type, int instance_size) { Object* result; { MaybeObject* maybe_result = AllocateRawMap(); if (!maybe_result->ToObject(&result)) return maybe_result; } Map* map = reinterpret_cast(result); map->set_map(meta_map()); map->set_instance_type(instance_type); map->set_visitor_id( StaticVisitorBase::GetVisitorId(instance_type, instance_size)); map->set_prototype(null_value()); map->set_constructor(null_value()); map->set_instance_size(instance_size); map->set_inobject_properties(0); map->set_pre_allocated_property_fields(0); map->set_instance_descriptors(empty_descriptor_array()); map->set_code_cache(empty_fixed_array()); map->set_unused_property_fields(0); map->set_bit_field(0); map->set_bit_field2((1 << Map::kIsExtensible) | (1 << Map::kHasFastElements)); // If the map object is aligned fill the padding area with Smi 0 objects. if (Map::kPadStart < Map::kSize) { memset(reinterpret_cast(map) + Map::kPadStart - kHeapObjectTag, 0, Map::kSize - Map::kPadStart); } return map; } MaybeObject* Heap::AllocateCodeCache() { Object* result; { MaybeObject* maybe_result = AllocateStruct(CODE_CACHE_TYPE); if (!maybe_result->ToObject(&result)) return maybe_result; } CodeCache* code_cache = CodeCache::cast(result); code_cache->set_default_cache(empty_fixed_array()); code_cache->set_normal_type_cache(undefined_value()); return code_cache; } const Heap::StringTypeTable Heap::string_type_table[] = { #define STRING_TYPE_ELEMENT(type, size, name, camel_name) \ {type, size, k##camel_name##MapRootIndex}, STRING_TYPE_LIST(STRING_TYPE_ELEMENT) #undef STRING_TYPE_ELEMENT }; const Heap::ConstantSymbolTable Heap::constant_symbol_table[] = { #define CONSTANT_SYMBOL_ELEMENT(name, contents) \ {contents, k##name##RootIndex}, SYMBOL_LIST(CONSTANT_SYMBOL_ELEMENT) #undef CONSTANT_SYMBOL_ELEMENT }; const Heap::StructTable Heap::struct_table[] = { #define STRUCT_TABLE_ELEMENT(NAME, Name, name) \ { NAME##_TYPE, Name::kSize, k##Name##MapRootIndex }, STRUCT_LIST(STRUCT_TABLE_ELEMENT) #undef STRUCT_TABLE_ELEMENT }; bool Heap::CreateInitialMaps() { Object* obj; { MaybeObject* maybe_obj = AllocatePartialMap(MAP_TYPE, Map::kSize); if (!maybe_obj->ToObject(&obj)) return false; } // Map::cast cannot be used due to uninitialized map field. Map* new_meta_map = reinterpret_cast(obj); set_meta_map(new_meta_map); new_meta_map->set_map(new_meta_map); { MaybeObject* maybe_obj = AllocatePartialMap(FIXED_ARRAY_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } set_fixed_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocatePartialMap(ODDBALL_TYPE, Oddball::kSize); if (!maybe_obj->ToObject(&obj)) return false; } set_oddball_map(Map::cast(obj)); // Allocate the empty array. { MaybeObject* maybe_obj = AllocateEmptyFixedArray(); if (!maybe_obj->ToObject(&obj)) return false; } set_empty_fixed_array(FixedArray::cast(obj)); { MaybeObject* maybe_obj = Allocate(oddball_map(), OLD_DATA_SPACE); if (!maybe_obj->ToObject(&obj)) return false; } set_null_value(obj); // Allocate the empty descriptor array. { MaybeObject* maybe_obj = AllocateEmptyFixedArray(); if (!maybe_obj->ToObject(&obj)) return false; } set_empty_descriptor_array(DescriptorArray::cast(obj)); // Fix the instance_descriptors for the existing maps. meta_map()->set_instance_descriptors(empty_descriptor_array()); meta_map()->set_code_cache(empty_fixed_array()); fixed_array_map()->set_instance_descriptors(empty_descriptor_array()); fixed_array_map()->set_code_cache(empty_fixed_array()); oddball_map()->set_instance_descriptors(empty_descriptor_array()); oddball_map()->set_code_cache(empty_fixed_array()); // Fix prototype object for existing maps. meta_map()->set_prototype(null_value()); meta_map()->set_constructor(null_value()); fixed_array_map()->set_prototype(null_value()); fixed_array_map()->set_constructor(null_value()); oddball_map()->set_prototype(null_value()); oddball_map()->set_constructor(null_value()); { MaybeObject* maybe_obj = AllocateMap(FIXED_ARRAY_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } set_fixed_cow_array_map(Map::cast(obj)); ASSERT(fixed_array_map() != fixed_cow_array_map()); { MaybeObject* maybe_obj = AllocateMap(HEAP_NUMBER_TYPE, HeapNumber::kSize); if (!maybe_obj->ToObject(&obj)) return false; } set_heap_number_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(PROXY_TYPE, Proxy::kSize); if (!maybe_obj->ToObject(&obj)) return false; } set_proxy_map(Map::cast(obj)); for (unsigned i = 0; i < ARRAY_SIZE(string_type_table); i++) { const StringTypeTable& entry = string_type_table[i]; { MaybeObject* maybe_obj = AllocateMap(entry.type, entry.size); if (!maybe_obj->ToObject(&obj)) return false; } roots_[entry.index] = Map::cast(obj); } { MaybeObject* maybe_obj = AllocateMap(STRING_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } set_undetectable_string_map(Map::cast(obj)); Map::cast(obj)->set_is_undetectable(); { MaybeObject* maybe_obj = AllocateMap(ASCII_STRING_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } set_undetectable_ascii_string_map(Map::cast(obj)); Map::cast(obj)->set_is_undetectable(); { MaybeObject* maybe_obj = AllocateMap(BYTE_ARRAY_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } set_byte_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateByteArray(0, TENURED); if (!maybe_obj->ToObject(&obj)) return false; } set_empty_byte_array(ByteArray::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(PIXEL_ARRAY_TYPE, PixelArray::kAlignedSize); if (!maybe_obj->ToObject(&obj)) return false; } set_pixel_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(EXTERNAL_BYTE_ARRAY_TYPE, ExternalArray::kAlignedSize); if (!maybe_obj->ToObject(&obj)) return false; } set_external_byte_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(EXTERNAL_UNSIGNED_BYTE_ARRAY_TYPE, ExternalArray::kAlignedSize); if (!maybe_obj->ToObject(&obj)) return false; } set_external_unsigned_byte_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(EXTERNAL_SHORT_ARRAY_TYPE, ExternalArray::kAlignedSize); if (!maybe_obj->ToObject(&obj)) return false; } set_external_short_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(EXTERNAL_UNSIGNED_SHORT_ARRAY_TYPE, ExternalArray::kAlignedSize); if (!maybe_obj->ToObject(&obj)) return false; } set_external_unsigned_short_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(EXTERNAL_INT_ARRAY_TYPE, ExternalArray::kAlignedSize); if (!maybe_obj->ToObject(&obj)) return false; } set_external_int_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(EXTERNAL_UNSIGNED_INT_ARRAY_TYPE, ExternalArray::kAlignedSize); if (!maybe_obj->ToObject(&obj)) return false; } set_external_unsigned_int_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(EXTERNAL_FLOAT_ARRAY_TYPE, ExternalArray::kAlignedSize); if (!maybe_obj->ToObject(&obj)) return false; } set_external_float_array_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(CODE_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } set_code_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(JS_GLOBAL_PROPERTY_CELL_TYPE, JSGlobalPropertyCell::kSize); if (!maybe_obj->ToObject(&obj)) return false; } set_global_property_cell_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(FILLER_TYPE, kPointerSize); if (!maybe_obj->ToObject(&obj)) return false; } set_one_pointer_filler_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(FILLER_TYPE, 2 * kPointerSize); if (!maybe_obj->ToObject(&obj)) return false; } set_two_pointer_filler_map(Map::cast(obj)); for (unsigned i = 0; i < ARRAY_SIZE(struct_table); i++) { const StructTable& entry = struct_table[i]; { MaybeObject* maybe_obj = AllocateMap(entry.type, entry.size); if (!maybe_obj->ToObject(&obj)) return false; } roots_[entry.index] = Map::cast(obj); } { MaybeObject* maybe_obj = AllocateMap(FIXED_ARRAY_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } set_hash_table_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(FIXED_ARRAY_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } set_context_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(FIXED_ARRAY_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } set_catch_context_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(FIXED_ARRAY_TYPE, kVariableSizeSentinel); if (!maybe_obj->ToObject(&obj)) return false; } Map* global_context_map = Map::cast(obj); global_context_map->set_visitor_id(StaticVisitorBase::kVisitGlobalContext); set_global_context_map(global_context_map); { MaybeObject* maybe_obj = AllocateMap(SHARED_FUNCTION_INFO_TYPE, SharedFunctionInfo::kAlignedSize); if (!maybe_obj->ToObject(&obj)) return false; } set_shared_function_info_map(Map::cast(obj)); { MaybeObject* maybe_obj = AllocateMap(JS_MESSAGE_OBJECT_TYPE, JSMessageObject::kSize); if (!maybe_obj->ToObject(&obj)) return false; } set_message_object_map(Map::cast(obj)); ASSERT(!Heap::InNewSpace(Heap::empty_fixed_array())); return true; } MaybeObject* Heap::AllocateHeapNumber(double value, PretenureFlag pretenure) { // Statically ensure that it is safe to allocate heap numbers in paged // spaces. STATIC_ASSERT(HeapNumber::kSize <= Page::kMaxHeapObjectSize); AllocationSpace space = (pretenure == TENURED) ? OLD_DATA_SPACE : NEW_SPACE; Object* result; { MaybeObject* maybe_result = AllocateRaw(HeapNumber::kSize, space, OLD_DATA_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } HeapObject::cast(result)->set_map(heap_number_map()); HeapNumber::cast(result)->set_value(value); return result; } MaybeObject* Heap::AllocateHeapNumber(double value) { // Use general version, if we're forced to always allocate. if (always_allocate()) return AllocateHeapNumber(value, TENURED); // This version of AllocateHeapNumber is optimized for // allocation in new space. STATIC_ASSERT(HeapNumber::kSize <= Page::kMaxHeapObjectSize); ASSERT(allocation_allowed_ && gc_state_ == NOT_IN_GC); Object* result; { MaybeObject* maybe_result = new_space_.AllocateRaw(HeapNumber::kSize); if (!maybe_result->ToObject(&result)) return maybe_result; } HeapObject::cast(result)->set_map(heap_number_map()); HeapNumber::cast(result)->set_value(value); return result; } MaybeObject* Heap::AllocateJSGlobalPropertyCell(Object* value) { Object* result; { MaybeObject* maybe_result = AllocateRawCell(); if (!maybe_result->ToObject(&result)) return maybe_result; } HeapObject::cast(result)->set_map(global_property_cell_map()); JSGlobalPropertyCell::cast(result)->set_value(value); return result; } MaybeObject* Heap::CreateOddball(const char* to_string, Object* to_number) { Object* result; { MaybeObject* maybe_result = Allocate(oddball_map(), OLD_DATA_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } return Oddball::cast(result)->Initialize(to_string, to_number); } bool Heap::CreateApiObjects() { Object* obj; { MaybeObject* maybe_obj = AllocateMap(JS_OBJECT_TYPE, JSObject::kHeaderSize); if (!maybe_obj->ToObject(&obj)) return false; } set_neander_map(Map::cast(obj)); { MaybeObject* maybe_obj = Heap::AllocateJSObjectFromMap(neander_map()); if (!maybe_obj->ToObject(&obj)) return false; } Object* elements; { MaybeObject* maybe_elements = AllocateFixedArray(2); if (!maybe_elements->ToObject(&elements)) return false; } FixedArray::cast(elements)->set(0, Smi::FromInt(0)); JSObject::cast(obj)->set_elements(FixedArray::cast(elements)); set_message_listeners(JSObject::cast(obj)); return true; } void Heap::CreateCEntryStub() { CEntryStub stub(1); set_c_entry_code(*stub.GetCode()); } #if V8_TARGET_ARCH_ARM && !V8_INTERPRETED_REGEXP void Heap::CreateRegExpCEntryStub() { RegExpCEntryStub stub; set_re_c_entry_code(*stub.GetCode()); } #endif void Heap::CreateJSEntryStub() { JSEntryStub stub; set_js_entry_code(*stub.GetCode()); } void Heap::CreateJSConstructEntryStub() { JSConstructEntryStub stub; set_js_construct_entry_code(*stub.GetCode()); } void Heap::CreateFixedStubs() { // Here we create roots for fixed stubs. They are needed at GC // for cooking and uncooking (check out frames.cc). // The eliminates the need for doing dictionary lookup in the // stub cache for these stubs. HandleScope scope; // gcc-4.4 has problem generating correct code of following snippet: // { CEntryStub stub; // c_entry_code_ = *stub.GetCode(); // } // { DebuggerStatementStub stub; // debugger_statement_code_ = *stub.GetCode(); // } // To workaround the problem, make separate functions without inlining. Heap::CreateCEntryStub(); Heap::CreateJSEntryStub(); Heap::CreateJSConstructEntryStub(); #if V8_TARGET_ARCH_ARM && !V8_INTERPRETED_REGEXP Heap::CreateRegExpCEntryStub(); #endif } bool Heap::CreateInitialObjects() { Object* obj; // The -0 value must be set before NumberFromDouble works. { MaybeObject* maybe_obj = AllocateHeapNumber(-0.0, TENURED); if (!maybe_obj->ToObject(&obj)) return false; } set_minus_zero_value(obj); ASSERT(signbit(minus_zero_value()->Number()) != 0); { MaybeObject* maybe_obj = AllocateHeapNumber(OS::nan_value(), TENURED); if (!maybe_obj->ToObject(&obj)) return false; } set_nan_value(obj); { MaybeObject* maybe_obj = Allocate(oddball_map(), OLD_DATA_SPACE); if (!maybe_obj->ToObject(&obj)) return false; } set_undefined_value(obj); ASSERT(!InNewSpace(undefined_value())); // Allocate initial symbol table. { MaybeObject* maybe_obj = SymbolTable::Allocate(kInitialSymbolTableSize); if (!maybe_obj->ToObject(&obj)) return false; } // Don't use set_symbol_table() due to asserts. roots_[kSymbolTableRootIndex] = obj; // Assign the print strings for oddballs after creating symboltable. Object* symbol; { MaybeObject* maybe_symbol = LookupAsciiSymbol("undefined"); if (!maybe_symbol->ToObject(&symbol)) return false; } Oddball::cast(undefined_value())->set_to_string(String::cast(symbol)); Oddball::cast(undefined_value())->set_to_number(nan_value()); // Allocate the null_value { MaybeObject* maybe_obj = Oddball::cast(null_value())->Initialize("null", Smi::FromInt(0)); if (!maybe_obj->ToObject(&obj)) return false; } { MaybeObject* maybe_obj = CreateOddball("true", Smi::FromInt(1)); if (!maybe_obj->ToObject(&obj)) return false; } set_true_value(obj); { MaybeObject* maybe_obj = CreateOddball("false", Smi::FromInt(0)); if (!maybe_obj->ToObject(&obj)) return false; } set_false_value(obj); { MaybeObject* maybe_obj = CreateOddball("hole", Smi::FromInt(-1)); if (!maybe_obj->ToObject(&obj)) return false; } set_the_hole_value(obj); { MaybeObject* maybe_obj = CreateOddball("arguments_marker", Smi::FromInt(-4)); if (!maybe_obj->ToObject(&obj)) return false; } set_arguments_marker(obj); { MaybeObject* maybe_obj = CreateOddball("no_interceptor_result_sentinel", Smi::FromInt(-2)); if (!maybe_obj->ToObject(&obj)) return false; } set_no_interceptor_result_sentinel(obj); { MaybeObject* maybe_obj = CreateOddball("termination_exception", Smi::FromInt(-3)); if (!maybe_obj->ToObject(&obj)) return false; } set_termination_exception(obj); // Allocate the empty string. { MaybeObject* maybe_obj = AllocateRawAsciiString(0, TENURED); if (!maybe_obj->ToObject(&obj)) return false; } set_empty_string(String::cast(obj)); for (unsigned i = 0; i < ARRAY_SIZE(constant_symbol_table); i++) { { MaybeObject* maybe_obj = LookupAsciiSymbol(constant_symbol_table[i].contents); if (!maybe_obj->ToObject(&obj)) return false; } roots_[constant_symbol_table[i].index] = String::cast(obj); } // Allocate the hidden symbol which is used to identify the hidden properties // in JSObjects. The hash code has a special value so that it will not match // the empty string when searching for the property. It cannot be part of the // loop above because it needs to be allocated manually with the special // hash code in place. The hash code for the hidden_symbol is zero to ensure // that it will always be at the first entry in property descriptors. { MaybeObject* maybe_obj = AllocateSymbol(CStrVector(""), 0, String::kZeroHash); if (!maybe_obj->ToObject(&obj)) return false; } hidden_symbol_ = String::cast(obj); // Allocate the proxy for __proto__. { MaybeObject* maybe_obj = AllocateProxy((Address) &Accessors::ObjectPrototype); if (!maybe_obj->ToObject(&obj)) return false; } set_prototype_accessors(Proxy::cast(obj)); // Allocate the code_stubs dictionary. The initial size is set to avoid // expanding the dictionary during bootstrapping. { MaybeObject* maybe_obj = NumberDictionary::Allocate(128); if (!maybe_obj->ToObject(&obj)) return false; } set_code_stubs(NumberDictionary::cast(obj)); // Allocate the non_monomorphic_cache used in stub-cache.cc. The initial size // is set to avoid expanding the dictionary during bootstrapping. { MaybeObject* maybe_obj = NumberDictionary::Allocate(64); if (!maybe_obj->ToObject(&obj)) return false; } set_non_monomorphic_cache(NumberDictionary::cast(obj)); set_instanceof_cache_function(Smi::FromInt(0)); set_instanceof_cache_map(Smi::FromInt(0)); set_instanceof_cache_answer(Smi::FromInt(0)); CreateFixedStubs(); // Allocate the dictionary of intrinsic function names. { MaybeObject* maybe_obj = StringDictionary::Allocate(Runtime::kNumFunctions); if (!maybe_obj->ToObject(&obj)) return false; } { MaybeObject* maybe_obj = Runtime::InitializeIntrinsicFunctionNames(obj); if (!maybe_obj->ToObject(&obj)) return false; } set_intrinsic_function_names(StringDictionary::cast(obj)); if (InitializeNumberStringCache()->IsFailure()) return false; // Allocate cache for single character ASCII strings. { MaybeObject* maybe_obj = AllocateFixedArray(String::kMaxAsciiCharCode + 1, TENURED); if (!maybe_obj->ToObject(&obj)) return false; } set_single_character_string_cache(FixedArray::cast(obj)); // Allocate cache for external strings pointing to native source code. { MaybeObject* maybe_obj = AllocateFixedArray(Natives::GetBuiltinsCount()); if (!maybe_obj->ToObject(&obj)) return false; } set_natives_source_cache(FixedArray::cast(obj)); // Handling of script id generation is in Factory::NewScript. set_last_script_id(undefined_value()); // Initialize keyed lookup cache. KeyedLookupCache::Clear(); // Initialize context slot cache. ContextSlotCache::Clear(); // Initialize descriptor cache. DescriptorLookupCache::Clear(); // Initialize compilation cache. CompilationCache::Clear(); return true; } MaybeObject* Heap::InitializeNumberStringCache() { // Compute the size of the number string cache based on the max heap size. // max_semispace_size_ == 512 KB => number_string_cache_size = 32. // max_semispace_size_ == 8 MB => number_string_cache_size = 16KB. int number_string_cache_size = max_semispace_size_ / 512; number_string_cache_size = Max(32, Min(16*KB, number_string_cache_size)); Object* obj; MaybeObject* maybe_obj = AllocateFixedArray(number_string_cache_size * 2, TENURED); if (maybe_obj->ToObject(&obj)) set_number_string_cache(FixedArray::cast(obj)); return maybe_obj; } void Heap::FlushNumberStringCache() { // Flush the number to string cache. int len = number_string_cache()->length(); for (int i = 0; i < len; i++) { number_string_cache()->set_undefined(i); } } static inline int double_get_hash(double d) { DoubleRepresentation rep(d); return static_cast(rep.bits) ^ static_cast(rep.bits >> 32); } static inline int smi_get_hash(Smi* smi) { return smi->value(); } Object* Heap::GetNumberStringCache(Object* number) { int hash; int mask = (number_string_cache()->length() >> 1) - 1; if (number->IsSmi()) { hash = smi_get_hash(Smi::cast(number)) & mask; } else { hash = double_get_hash(number->Number()) & mask; } Object* key = number_string_cache()->get(hash * 2); if (key == number) { return String::cast(number_string_cache()->get(hash * 2 + 1)); } else if (key->IsHeapNumber() && number->IsHeapNumber() && key->Number() == number->Number()) { return String::cast(number_string_cache()->get(hash * 2 + 1)); } return undefined_value(); } void Heap::SetNumberStringCache(Object* number, String* string) { int hash; int mask = (number_string_cache()->length() >> 1) - 1; if (number->IsSmi()) { hash = smi_get_hash(Smi::cast(number)) & mask; number_string_cache()->set(hash * 2, Smi::cast(number)); } else { hash = double_get_hash(number->Number()) & mask; number_string_cache()->set(hash * 2, number); } number_string_cache()->set(hash * 2 + 1, string); } MaybeObject* Heap::NumberToString(Object* number, bool check_number_string_cache) { Counters::number_to_string_runtime.Increment(); if (check_number_string_cache) { Object* cached = GetNumberStringCache(number); if (cached != undefined_value()) { return cached; } } char arr[100]; Vector buffer(arr, ARRAY_SIZE(arr)); const char* str; if (number->IsSmi()) { int num = Smi::cast(number)->value(); str = IntToCString(num, buffer); } else { double num = HeapNumber::cast(number)->value(); str = DoubleToCString(num, buffer); } Object* js_string; MaybeObject* maybe_js_string = AllocateStringFromAscii(CStrVector(str)); if (maybe_js_string->ToObject(&js_string)) { SetNumberStringCache(number, String::cast(js_string)); } return maybe_js_string; } Map* Heap::MapForExternalArrayType(ExternalArrayType array_type) { return Map::cast(roots_[RootIndexForExternalArrayType(array_type)]); } Heap::RootListIndex Heap::RootIndexForExternalArrayType( ExternalArrayType array_type) { switch (array_type) { case kExternalByteArray: return kExternalByteArrayMapRootIndex; case kExternalUnsignedByteArray: return kExternalUnsignedByteArrayMapRootIndex; case kExternalShortArray: return kExternalShortArrayMapRootIndex; case kExternalUnsignedShortArray: return kExternalUnsignedShortArrayMapRootIndex; case kExternalIntArray: return kExternalIntArrayMapRootIndex; case kExternalUnsignedIntArray: return kExternalUnsignedIntArrayMapRootIndex; case kExternalFloatArray: return kExternalFloatArrayMapRootIndex; default: UNREACHABLE(); return kUndefinedValueRootIndex; } } MaybeObject* Heap::NumberFromDouble(double value, PretenureFlag pretenure) { // We need to distinguish the minus zero value and this cannot be // done after conversion to int. Doing this by comparing bit // patterns is faster than using fpclassify() et al. static const DoubleRepresentation minus_zero(-0.0); DoubleRepresentation rep(value); if (rep.bits == minus_zero.bits) { return AllocateHeapNumber(-0.0, pretenure); } int int_value = FastD2I(value); if (value == int_value && Smi::IsValid(int_value)) { return Smi::FromInt(int_value); } // Materialize the value in the heap. return AllocateHeapNumber(value, pretenure); } MaybeObject* Heap::AllocateProxy(Address proxy, PretenureFlag pretenure) { // Statically ensure that it is safe to allocate proxies in paged spaces. STATIC_ASSERT(Proxy::kSize <= Page::kMaxHeapObjectSize); AllocationSpace space = (pretenure == TENURED) ? OLD_DATA_SPACE : NEW_SPACE; Object* result; { MaybeObject* maybe_result = Allocate(proxy_map(), space); if (!maybe_result->ToObject(&result)) return maybe_result; } Proxy::cast(result)->set_proxy(proxy); return result; } MaybeObject* Heap::AllocateSharedFunctionInfo(Object* name) { Object* result; { MaybeObject* maybe_result = Allocate(shared_function_info_map(), OLD_POINTER_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } SharedFunctionInfo* share = SharedFunctionInfo::cast(result); share->set_name(name); Code* illegal = Builtins::builtin(Builtins::Illegal); share->set_code(illegal); share->set_scope_info(SerializedScopeInfo::Empty()); Code* construct_stub = Builtins::builtin(Builtins::JSConstructStubGeneric); share->set_construct_stub(construct_stub); share->set_expected_nof_properties(0); share->set_length(0); share->set_formal_parameter_count(0); share->set_instance_class_name(Object_symbol()); share->set_function_data(undefined_value()); share->set_script(undefined_value()); share->set_start_position_and_type(0); share->set_debug_info(undefined_value()); share->set_inferred_name(empty_string()); share->set_compiler_hints(0); share->set_deopt_counter(Smi::FromInt(FLAG_deopt_every_n_times)); share->set_initial_map(undefined_value()); share->set_this_property_assignments_count(0); share->set_this_property_assignments(undefined_value()); share->set_opt_count(0); share->set_num_literals(0); share->set_end_position(0); share->set_function_token_position(0); return result; } MaybeObject* Heap::AllocateJSMessageObject(String* type, JSArray* arguments, int start_position, int end_position, Object* script, Object* stack_trace, Object* stack_frames) { Object* result; { MaybeObject* maybe_result = Allocate(message_object_map(), NEW_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } JSMessageObject* message = JSMessageObject::cast(result); message->set_properties(Heap::empty_fixed_array()); message->set_elements(Heap::empty_fixed_array()); message->set_type(type); message->set_arguments(arguments); message->set_start_position(start_position); message->set_end_position(end_position); message->set_script(script); message->set_stack_trace(stack_trace); message->set_stack_frames(stack_frames); return result; } // Returns true for a character in a range. Both limits are inclusive. static inline bool Between(uint32_t character, uint32_t from, uint32_t to) { // This makes uses of the the unsigned wraparound. return character - from <= to - from; } MUST_USE_RESULT static inline MaybeObject* MakeOrFindTwoCharacterString( uint32_t c1, uint32_t c2) { String* symbol; // Numeric strings have a different hash algorithm not known by // LookupTwoCharsSymbolIfExists, so we skip this step for such strings. if ((!Between(c1, '0', '9') || !Between(c2, '0', '9')) && Heap::symbol_table()->LookupTwoCharsSymbolIfExists(c1, c2, &symbol)) { return symbol; // Now we know the length is 2, we might as well make use of that fact // when building the new string. } else if ((c1 | c2) <= String::kMaxAsciiCharCodeU) { // We can do this ASSERT(IsPowerOf2(String::kMaxAsciiCharCodeU + 1)); // because of this. Object* result; { MaybeObject* maybe_result = Heap::AllocateRawAsciiString(2); if (!maybe_result->ToObject(&result)) return maybe_result; } char* dest = SeqAsciiString::cast(result)->GetChars(); dest[0] = c1; dest[1] = c2; return result; } else { Object* result; { MaybeObject* maybe_result = Heap::AllocateRawTwoByteString(2); if (!maybe_result->ToObject(&result)) return maybe_result; } uc16* dest = SeqTwoByteString::cast(result)->GetChars(); dest[0] = c1; dest[1] = c2; return result; } } MaybeObject* Heap::AllocateConsString(String* first, String* second) { int first_length = first->length(); if (first_length == 0) { return second; } int second_length = second->length(); if (second_length == 0) { return first; } int length = first_length + second_length; // Optimization for 2-byte strings often used as keys in a decompression // dictionary. Check whether we already have the string in the symbol // table to prevent creation of many unneccesary strings. if (length == 2) { unsigned c1 = first->Get(0); unsigned c2 = second->Get(0); return MakeOrFindTwoCharacterString(c1, c2); } bool first_is_ascii = first->IsAsciiRepresentation(); bool second_is_ascii = second->IsAsciiRepresentation(); bool is_ascii = first_is_ascii && second_is_ascii; // Make sure that an out of memory exception is thrown if the length // of the new cons string is too large. if (length > String::kMaxLength || length < 0) { Top::context()->mark_out_of_memory(); return Failure::OutOfMemoryException(); } bool is_ascii_data_in_two_byte_string = false; if (!is_ascii) { // At least one of the strings uses two-byte representation so we // can't use the fast case code for short ascii strings below, but // we can try to save memory if all chars actually fit in ascii. is_ascii_data_in_two_byte_string = first->HasOnlyAsciiChars() && second->HasOnlyAsciiChars(); if (is_ascii_data_in_two_byte_string) { Counters::string_add_runtime_ext_to_ascii.Increment(); } } // If the resulting string is small make a flat string. if (length < String::kMinNonFlatLength) { ASSERT(first->IsFlat()); ASSERT(second->IsFlat()); if (is_ascii) { Object* result; { MaybeObject* maybe_result = AllocateRawAsciiString(length); if (!maybe_result->ToObject(&result)) return maybe_result; } // Copy the characters into the new object. char* dest = SeqAsciiString::cast(result)->GetChars(); // Copy first part. const char* src; if (first->IsExternalString()) { src = ExternalAsciiString::cast(first)->resource()->data(); } else { src = SeqAsciiString::cast(first)->GetChars(); } for (int i = 0; i < first_length; i++) *dest++ = src[i]; // Copy second part. if (second->IsExternalString()) { src = ExternalAsciiString::cast(second)->resource()->data(); } else { src = SeqAsciiString::cast(second)->GetChars(); } for (int i = 0; i < second_length; i++) *dest++ = src[i]; return result; } else { if (is_ascii_data_in_two_byte_string) { Object* result; { MaybeObject* maybe_result = AllocateRawAsciiString(length); if (!maybe_result->ToObject(&result)) return maybe_result; } // Copy the characters into the new object. char* dest = SeqAsciiString::cast(result)->GetChars(); String::WriteToFlat(first, dest, 0, first_length); String::WriteToFlat(second, dest + first_length, 0, second_length); return result; } Object* result; { MaybeObject* maybe_result = AllocateRawTwoByteString(length); if (!maybe_result->ToObject(&result)) return maybe_result; } // Copy the characters into the new object. uc16* dest = SeqTwoByteString::cast(result)->GetChars(); String::WriteToFlat(first, dest, 0, first_length); String::WriteToFlat(second, dest + first_length, 0, second_length); return result; } } Map* map = (is_ascii || is_ascii_data_in_two_byte_string) ? cons_ascii_string_map() : cons_string_map(); Object* result; { MaybeObject* maybe_result = Allocate(map, NEW_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } AssertNoAllocation no_gc; ConsString* cons_string = ConsString::cast(result); WriteBarrierMode mode = cons_string->GetWriteBarrierMode(no_gc); cons_string->set_length(length); cons_string->set_hash_field(String::kEmptyHashField); cons_string->set_first(first, mode); cons_string->set_second(second, mode); return result; } MaybeObject* Heap::AllocateSubString(String* buffer, int start, int end, PretenureFlag pretenure) { int length = end - start; if (length == 1) { return Heap::LookupSingleCharacterStringFromCode( buffer->Get(start)); } else if (length == 2) { // Optimization for 2-byte strings often used as keys in a decompression // dictionary. Check whether we already have the string in the symbol // table to prevent creation of many unneccesary strings. unsigned c1 = buffer->Get(start); unsigned c2 = buffer->Get(start + 1); return MakeOrFindTwoCharacterString(c1, c2); } // Make an attempt to flatten the buffer to reduce access time. buffer = buffer->TryFlattenGetString(); Object* result; { MaybeObject* maybe_result = buffer->IsAsciiRepresentation() ? AllocateRawAsciiString(length, pretenure ) : AllocateRawTwoByteString(length, pretenure); if (!maybe_result->ToObject(&result)) return maybe_result; } String* string_result = String::cast(result); // Copy the characters into the new object. if (buffer->IsAsciiRepresentation()) { ASSERT(string_result->IsAsciiRepresentation()); char* dest = SeqAsciiString::cast(string_result)->GetChars(); String::WriteToFlat(buffer, dest, start, end); } else { ASSERT(string_result->IsTwoByteRepresentation()); uc16* dest = SeqTwoByteString::cast(string_result)->GetChars(); String::WriteToFlat(buffer, dest, start, end); } return result; } MaybeObject* Heap::AllocateExternalStringFromAscii( ExternalAsciiString::Resource* resource) { size_t length = resource->length(); if (length > static_cast(String::kMaxLength)) { Top::context()->mark_out_of_memory(); return Failure::OutOfMemoryException(); } Map* map = external_ascii_string_map(); Object* result; { MaybeObject* maybe_result = Allocate(map, NEW_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } ExternalAsciiString* external_string = ExternalAsciiString::cast(result); external_string->set_length(static_cast(length)); external_string->set_hash_field(String::kEmptyHashField); external_string->set_resource(resource); return result; } MaybeObject* Heap::AllocateExternalStringFromTwoByte( ExternalTwoByteString::Resource* resource) { size_t length = resource->length(); if (length > static_cast(String::kMaxLength)) { Top::context()->mark_out_of_memory(); return Failure::OutOfMemoryException(); } // For small strings we check whether the resource contains only // ASCII characters. If yes, we use a different string map. static const size_t kAsciiCheckLengthLimit = 32; bool is_ascii = length <= kAsciiCheckLengthLimit && String::IsAscii(resource->data(), static_cast(length)); Map* map = is_ascii ? Heap::external_string_with_ascii_data_map() : Heap::external_string_map(); Object* result; { MaybeObject* maybe_result = Allocate(map, NEW_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } ExternalTwoByteString* external_string = ExternalTwoByteString::cast(result); external_string->set_length(static_cast(length)); external_string->set_hash_field(String::kEmptyHashField); external_string->set_resource(resource); return result; } MaybeObject* Heap::LookupSingleCharacterStringFromCode(uint16_t code) { if (code <= String::kMaxAsciiCharCode) { Object* value = Heap::single_character_string_cache()->get(code); if (value != Heap::undefined_value()) return value; char buffer[1]; buffer[0] = static_cast(code); Object* result; MaybeObject* maybe_result = LookupSymbol(Vector(buffer, 1)); if (!maybe_result->ToObject(&result)) return maybe_result; Heap::single_character_string_cache()->set(code, result); return result; } Object* result; { MaybeObject* maybe_result = Heap::AllocateRawTwoByteString(1); if (!maybe_result->ToObject(&result)) return maybe_result; } String* answer = String::cast(result); answer->Set(0, code); return answer; } MaybeObject* Heap::AllocateByteArray(int length, PretenureFlag pretenure) { if (length < 0 || length > ByteArray::kMaxLength) { return Failure::OutOfMemoryException(); } if (pretenure == NOT_TENURED) { return AllocateByteArray(length); } int size = ByteArray::SizeFor(length); Object* result; { MaybeObject* maybe_result = (size <= MaxObjectSizeInPagedSpace()) ? old_data_space_->AllocateRaw(size) : lo_space_->AllocateRaw(size); if (!maybe_result->ToObject(&result)) return maybe_result; } reinterpret_cast(result)->set_map(byte_array_map()); reinterpret_cast(result)->set_length(length); return result; } MaybeObject* Heap::AllocateByteArray(int length) { if (length < 0 || length > ByteArray::kMaxLength) { return Failure::OutOfMemoryException(); } int size = ByteArray::SizeFor(length); AllocationSpace space = (size > MaxObjectSizeInPagedSpace()) ? LO_SPACE : NEW_SPACE; Object* result; { MaybeObject* maybe_result = AllocateRaw(size, space, OLD_DATA_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } reinterpret_cast(result)->set_map(byte_array_map()); reinterpret_cast(result)->set_length(length); return result; } void Heap::CreateFillerObjectAt(Address addr, int size) { if (size == 0) return; HeapObject* filler = HeapObject::FromAddress(addr); if (size == kPointerSize) { filler->set_map(one_pointer_filler_map()); } else if (size == 2 * kPointerSize) { filler->set_map(two_pointer_filler_map()); } else { filler->set_map(byte_array_map()); ByteArray::cast(filler)->set_length(ByteArray::LengthFor(size)); } } MaybeObject* Heap::AllocatePixelArray(int length, uint8_t* external_pointer, PretenureFlag pretenure) { AllocationSpace space = (pretenure == TENURED) ? OLD_DATA_SPACE : NEW_SPACE; Object* result; { MaybeObject* maybe_result = AllocateRaw(PixelArray::kAlignedSize, space, OLD_DATA_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } reinterpret_cast(result)->set_map(pixel_array_map()); reinterpret_cast(result)->set_length(length); reinterpret_cast(result)->set_external_pointer(external_pointer); return result; } MaybeObject* Heap::AllocateExternalArray(int length, ExternalArrayType array_type, void* external_pointer, PretenureFlag pretenure) { AllocationSpace space = (pretenure == TENURED) ? OLD_DATA_SPACE : NEW_SPACE; Object* result; { MaybeObject* maybe_result = AllocateRaw(ExternalArray::kAlignedSize, space, OLD_DATA_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } reinterpret_cast(result)->set_map( MapForExternalArrayType(array_type)); reinterpret_cast(result)->set_length(length); reinterpret_cast(result)->set_external_pointer( external_pointer); return result; } MaybeObject* Heap::CreateCode(const CodeDesc& desc, Code::Flags flags, Handle self_reference) { // Allocate ByteArray before the Code object, so that we do not risk // leaving uninitialized Code object (and breaking the heap). Object* reloc_info; { MaybeObject* maybe_reloc_info = AllocateByteArray(desc.reloc_size, TENURED); if (!maybe_reloc_info->ToObject(&reloc_info)) return maybe_reloc_info; } // Compute size int body_size = RoundUp(desc.instr_size, kObjectAlignment); int obj_size = Code::SizeFor(body_size); ASSERT(IsAligned(static_cast(obj_size), kCodeAlignment)); MaybeObject* maybe_result; if (obj_size > MaxObjectSizeInPagedSpace()) { maybe_result = lo_space_->AllocateRawCode(obj_size); } else { maybe_result = code_space_->AllocateRaw(obj_size); } Object* result; if (!maybe_result->ToObject(&result)) return maybe_result; // Initialize the object HeapObject::cast(result)->set_map(code_map()); Code* code = Code::cast(result); ASSERT(!CodeRange::exists() || CodeRange::contains(code->address())); code->set_instruction_size(desc.instr_size); code->set_relocation_info(ByteArray::cast(reloc_info)); code->set_flags(flags); if (code->is_call_stub() || code->is_keyed_call_stub()) { code->set_check_type(RECEIVER_MAP_CHECK); } code->set_deoptimization_data(empty_fixed_array()); // Allow self references to created code object by patching the handle to // point to the newly allocated Code object. if (!self_reference.is_null()) { *(self_reference.location()) = code; } // Migrate generated code. // The generated code can contain Object** values (typically from handles) // that are dereferenced during the copy to point directly to the actual heap // objects. These pointers can include references to the code object itself, // through the self_reference parameter. code->CopyFrom(desc); #ifdef DEBUG code->Verify(); #endif return code; } MaybeObject* Heap::CopyCode(Code* code) { // Allocate an object the same size as the code object. int obj_size = code->Size(); MaybeObject* maybe_result; if (obj_size > MaxObjectSizeInPagedSpace()) { maybe_result = lo_space_->AllocateRawCode(obj_size); } else { maybe_result = code_space_->AllocateRaw(obj_size); } Object* result; if (!maybe_result->ToObject(&result)) return maybe_result; // Copy code object. Address old_addr = code->address(); Address new_addr = reinterpret_cast(result)->address(); CopyBlock(new_addr, old_addr, obj_size); // Relocate the copy. Code* new_code = Code::cast(result); ASSERT(!CodeRange::exists() || CodeRange::contains(code->address())); new_code->Relocate(new_addr - old_addr); return new_code; } MaybeObject* Heap::CopyCode(Code* code, Vector reloc_info) { // Allocate ByteArray before the Code object, so that we do not risk // leaving uninitialized Code object (and breaking the heap). Object* reloc_info_array; { MaybeObject* maybe_reloc_info_array = AllocateByteArray(reloc_info.length(), TENURED); if (!maybe_reloc_info_array->ToObject(&reloc_info_array)) { return maybe_reloc_info_array; } } int new_body_size = RoundUp(code->instruction_size(), kObjectAlignment); int new_obj_size = Code::SizeFor(new_body_size); Address old_addr = code->address(); size_t relocation_offset = static_cast(code->instruction_end() - old_addr); MaybeObject* maybe_result; if (new_obj_size > MaxObjectSizeInPagedSpace()) { maybe_result = lo_space_->AllocateRawCode(new_obj_size); } else { maybe_result = code_space_->AllocateRaw(new_obj_size); } Object* result; if (!maybe_result->ToObject(&result)) return maybe_result; // Copy code object. Address new_addr = reinterpret_cast(result)->address(); // Copy header and instructions. memcpy(new_addr, old_addr, relocation_offset); Code* new_code = Code::cast(result); new_code->set_relocation_info(ByteArray::cast(reloc_info_array)); // Copy patched rinfo. memcpy(new_code->relocation_start(), reloc_info.start(), reloc_info.length()); // Relocate the copy. ASSERT(!CodeRange::exists() || CodeRange::contains(code->address())); new_code->Relocate(new_addr - old_addr); #ifdef DEBUG code->Verify(); #endif return new_code; } MaybeObject* Heap::Allocate(Map* map, AllocationSpace space) { ASSERT(gc_state_ == NOT_IN_GC); ASSERT(map->instance_type() != MAP_TYPE); // If allocation failures are disallowed, we may allocate in a different // space when new space is full and the object is not a large object. AllocationSpace retry_space = (space != NEW_SPACE) ? space : TargetSpaceId(map->instance_type()); Object* result; { MaybeObject* maybe_result = AllocateRaw(map->instance_size(), space, retry_space); if (!maybe_result->ToObject(&result)) return maybe_result; } HeapObject::cast(result)->set_map(map); #ifdef ENABLE_LOGGING_AND_PROFILING ProducerHeapProfile::RecordJSObjectAllocation(result); #endif return result; } MaybeObject* Heap::InitializeFunction(JSFunction* function, SharedFunctionInfo* shared, Object* prototype) { ASSERT(!prototype->IsMap()); function->initialize_properties(); function->initialize_elements(); function->set_shared(shared); function->set_code(shared->code()); function->set_prototype_or_initial_map(prototype); function->set_context(undefined_value()); function->set_literals(empty_fixed_array()); function->set_next_function_link(undefined_value()); return function; } MaybeObject* Heap::AllocateFunctionPrototype(JSFunction* function) { // Allocate the prototype. Make sure to use the object function // from the function's context, since the function can be from a // different context. JSFunction* object_function = function->context()->global_context()->object_function(); Object* prototype; { MaybeObject* maybe_prototype = AllocateJSObject(object_function); if (!maybe_prototype->ToObject(&prototype)) return maybe_prototype; } // When creating the prototype for the function we must set its // constructor to the function. Object* result; { MaybeObject* maybe_result = JSObject::cast(prototype)->SetProperty(constructor_symbol(), function, DONT_ENUM); if (!maybe_result->ToObject(&result)) return maybe_result; } return prototype; } MaybeObject* Heap::AllocateFunction(Map* function_map, SharedFunctionInfo* shared, Object* prototype, PretenureFlag pretenure) { AllocationSpace space = (pretenure == TENURED) ? OLD_POINTER_SPACE : NEW_SPACE; Object* result; { MaybeObject* maybe_result = Allocate(function_map, space); if (!maybe_result->ToObject(&result)) return maybe_result; } return InitializeFunction(JSFunction::cast(result), shared, prototype); } MaybeObject* Heap::AllocateArgumentsObject(Object* callee, int length) { // To get fast allocation and map sharing for arguments objects we // allocate them based on an arguments boilerplate. // This calls Copy directly rather than using Heap::AllocateRaw so we // duplicate the check here. ASSERT(allocation_allowed_ && gc_state_ == NOT_IN_GC); JSObject* boilerplate = Top::context()->global_context()->arguments_boilerplate(); // Check that the size of the boilerplate matches our // expectations. The ArgumentsAccessStub::GenerateNewObject relies // on the size being a known constant. ASSERT(kArgumentsObjectSize == boilerplate->map()->instance_size()); // Do the allocation. Object* result; { MaybeObject* maybe_result = AllocateRaw(kArgumentsObjectSize, NEW_SPACE, OLD_POINTER_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } // Copy the content. The arguments boilerplate doesn't have any // fields that point to new space so it's safe to skip the write // barrier here. CopyBlock(HeapObject::cast(result)->address(), boilerplate->address(), kArgumentsObjectSize); // Set the two properties. JSObject::cast(result)->InObjectPropertyAtPut(arguments_callee_index, callee); JSObject::cast(result)->InObjectPropertyAtPut(arguments_length_index, Smi::FromInt(length), SKIP_WRITE_BARRIER); // Check the state of the object ASSERT(JSObject::cast(result)->HasFastProperties()); ASSERT(JSObject::cast(result)->HasFastElements()); return result; } static bool HasDuplicates(DescriptorArray* descriptors) { int count = descriptors->number_of_descriptors(); if (count > 1) { String* prev_key = descriptors->GetKey(0); for (int i = 1; i != count; i++) { String* current_key = descriptors->GetKey(i); if (prev_key == current_key) return true; prev_key = current_key; } } return false; } MaybeObject* Heap::AllocateInitialMap(JSFunction* fun) { ASSERT(!fun->has_initial_map()); // First create a new map with the size and number of in-object properties // suggested by the function. int instance_size = fun->shared()->CalculateInstanceSize(); int in_object_properties = fun->shared()->CalculateInObjectProperties(); Object* map_obj; { MaybeObject* maybe_map_obj = Heap::AllocateMap(JS_OBJECT_TYPE, instance_size); if (!maybe_map_obj->ToObject(&map_obj)) return maybe_map_obj; } // Fetch or allocate prototype. Object* prototype; if (fun->has_instance_prototype()) { prototype = fun->instance_prototype(); } else { { MaybeObject* maybe_prototype = AllocateFunctionPrototype(fun); if (!maybe_prototype->ToObject(&prototype)) return maybe_prototype; } } Map* map = Map::cast(map_obj); map->set_inobject_properties(in_object_properties); map->set_unused_property_fields(in_object_properties); map->set_prototype(prototype); ASSERT(map->has_fast_elements()); // If the function has only simple this property assignments add // field descriptors for these to the initial map as the object // cannot be constructed without having these properties. Guard by // the inline_new flag so we only change the map if we generate a // specialized construct stub. ASSERT(in_object_properties <= Map::kMaxPreAllocatedPropertyFields); if (fun->shared()->CanGenerateInlineConstructor(prototype)) { int count = fun->shared()->this_property_assignments_count(); if (count > in_object_properties) { // Inline constructor can only handle inobject properties. fun->shared()->ForbidInlineConstructor(); } else { Object* descriptors_obj; { MaybeObject* maybe_descriptors_obj = DescriptorArray::Allocate(count); if (!maybe_descriptors_obj->ToObject(&descriptors_obj)) { return maybe_descriptors_obj; } } DescriptorArray* descriptors = DescriptorArray::cast(descriptors_obj); for (int i = 0; i < count; i++) { String* name = fun->shared()->GetThisPropertyAssignmentName(i); ASSERT(name->IsSymbol()); FieldDescriptor field(name, i, NONE); field.SetEnumerationIndex(i); descriptors->Set(i, &field); } descriptors->SetNextEnumerationIndex(count); descriptors->SortUnchecked(); // The descriptors may contain duplicates because the compiler does not // guarantee the uniqueness of property names (it would have required // quadratic time). Once the descriptors are sorted we can check for // duplicates in linear time. if (HasDuplicates(descriptors)) { fun->shared()->ForbidInlineConstructor(); } else { map->set_instance_descriptors(descriptors); map->set_pre_allocated_property_fields(count); map->set_unused_property_fields(in_object_properties - count); } } } fun->shared()->StartInobjectSlackTracking(map); return map; } void Heap::InitializeJSObjectFromMap(JSObject* obj, FixedArray* properties, Map* map) { obj->set_properties(properties); obj->initialize_elements(); // TODO(1240798): Initialize the object's body using valid initial values // according to the object's initial map. For example, if the map's // instance type is JS_ARRAY_TYPE, the length field should be initialized // to a number (eg, Smi::FromInt(0)) and the elements initialized to a // fixed array (eg, Heap::empty_fixed_array()). Currently, the object // verification code has to cope with (temporarily) invalid objects. See // for example, JSArray::JSArrayVerify). Object* filler; // We cannot always fill with one_pointer_filler_map because objects // created from API functions expect their internal fields to be initialized // with undefined_value. if (map->constructor()->IsJSFunction() && JSFunction::cast(map->constructor())->shared()-> IsInobjectSlackTrackingInProgress()) { // We might want to shrink the object later. ASSERT(obj->GetInternalFieldCount() == 0); filler = Heap::one_pointer_filler_map(); } else { filler = Heap::undefined_value(); } obj->InitializeBody(map->instance_size(), filler); } MaybeObject* Heap::AllocateJSObjectFromMap(Map* map, PretenureFlag pretenure) { // JSFunctions should be allocated using AllocateFunction to be // properly initialized. ASSERT(map->instance_type() != JS_FUNCTION_TYPE); // Both types of global objects should be allocated using // AllocateGlobalObject to be properly initialized. ASSERT(map->instance_type() != JS_GLOBAL_OBJECT_TYPE); ASSERT(map->instance_type() != JS_BUILTINS_OBJECT_TYPE); // Allocate the backing storage for the properties. int prop_size = map->pre_allocated_property_fields() + map->unused_property_fields() - map->inobject_properties(); ASSERT(prop_size >= 0); Object* properties; { MaybeObject* maybe_properties = AllocateFixedArray(prop_size, pretenure); if (!maybe_properties->ToObject(&properties)) return maybe_properties; } // Allocate the JSObject. AllocationSpace space = (pretenure == TENURED) ? OLD_POINTER_SPACE : NEW_SPACE; if (map->instance_size() > MaxObjectSizeInPagedSpace()) space = LO_SPACE; Object* obj; { MaybeObject* maybe_obj = Allocate(map, space); if (!maybe_obj->ToObject(&obj)) return maybe_obj; } // Initialize the JSObject. InitializeJSObjectFromMap(JSObject::cast(obj), FixedArray::cast(properties), map); ASSERT(JSObject::cast(obj)->HasFastElements()); return obj; } MaybeObject* Heap::AllocateJSObject(JSFunction* constructor, PretenureFlag pretenure) { // Allocate the initial map if absent. if (!constructor->has_initial_map()) { Object* initial_map; { MaybeObject* maybe_initial_map = AllocateInitialMap(constructor); if (!maybe_initial_map->ToObject(&initial_map)) return maybe_initial_map; } constructor->set_initial_map(Map::cast(initial_map)); Map::cast(initial_map)->set_constructor(constructor); } // Allocate the object based on the constructors initial map. MaybeObject* result = AllocateJSObjectFromMap(constructor->initial_map(), pretenure); #ifdef DEBUG // Make sure result is NOT a global object if valid. Object* non_failure; ASSERT(!result->ToObject(&non_failure) || !non_failure->IsGlobalObject()); #endif return result; } MaybeObject* Heap::AllocateGlobalObject(JSFunction* constructor) { ASSERT(constructor->has_initial_map()); Map* map = constructor->initial_map(); // Make sure no field properties are described in the initial map. // This guarantees us that normalizing the properties does not // require us to change property values to JSGlobalPropertyCells. ASSERT(map->NextFreePropertyIndex() == 0); // Make sure we don't have a ton of pre-allocated slots in the // global objects. They will be unused once we normalize the object. ASSERT(map->unused_property_fields() == 0); ASSERT(map->inobject_properties() == 0); // Initial size of the backing store to avoid resize of the storage during // bootstrapping. The size differs between the JS global object ad the // builtins object. int initial_size = map->instance_type() == JS_GLOBAL_OBJECT_TYPE ? 64 : 512; // Allocate a dictionary object for backing storage. Object* obj; { MaybeObject* maybe_obj = StringDictionary::Allocate( map->NumberOfDescribedProperties() * 2 + initial_size); if (!maybe_obj->ToObject(&obj)) return maybe_obj; } StringDictionary* dictionary = StringDictionary::cast(obj); // The global object might be created from an object template with accessors. // Fill these accessors into the dictionary. DescriptorArray* descs = map->instance_descriptors(); for (int i = 0; i < descs->number_of_descriptors(); i++) { PropertyDetails details = descs->GetDetails(i); ASSERT(details.type() == CALLBACKS); // Only accessors are expected. PropertyDetails d = PropertyDetails(details.attributes(), CALLBACKS, details.index()); Object* value = descs->GetCallbacksObject(i); { MaybeObject* maybe_value = Heap::AllocateJSGlobalPropertyCell(value); if (!maybe_value->ToObject(&value)) return maybe_value; } Object* result; { MaybeObject* maybe_result = dictionary->Add(descs->GetKey(i), value, d); if (!maybe_result->ToObject(&result)) return maybe_result; } dictionary = StringDictionary::cast(result); } // Allocate the global object and initialize it with the backing store. { MaybeObject* maybe_obj = Allocate(map, OLD_POINTER_SPACE); if (!maybe_obj->ToObject(&obj)) return maybe_obj; } JSObject* global = JSObject::cast(obj); InitializeJSObjectFromMap(global, dictionary, map); // Create a new map for the global object. { MaybeObject* maybe_obj = map->CopyDropDescriptors(); if (!maybe_obj->ToObject(&obj)) return maybe_obj; } Map* new_map = Map::cast(obj); // Setup the global object as a normalized object. global->set_map(new_map); global->map()->set_instance_descriptors(Heap::empty_descriptor_array()); global->set_properties(dictionary); // Make sure result is a global object with properties in dictionary. ASSERT(global->IsGlobalObject()); ASSERT(!global->HasFastProperties()); return global; } MaybeObject* Heap::CopyJSObject(JSObject* source) { // Never used to copy functions. If functions need to be copied we // have to be careful to clear the literals array. ASSERT(!source->IsJSFunction()); // Make the clone. Map* map = source->map(); int object_size = map->instance_size(); Object* clone; // If we're forced to always allocate, we use the general allocation // functions which may leave us with an object in old space. if (always_allocate()) { { MaybeObject* maybe_clone = AllocateRaw(object_size, NEW_SPACE, OLD_POINTER_SPACE); if (!maybe_clone->ToObject(&clone)) return maybe_clone; } Address clone_address = HeapObject::cast(clone)->address(); CopyBlock(clone_address, source->address(), object_size); // Update write barrier for all fields that lie beyond the header. RecordWrites(clone_address, JSObject::kHeaderSize, (object_size - JSObject::kHeaderSize) / kPointerSize); } else { { MaybeObject* maybe_clone = new_space_.AllocateRaw(object_size); if (!maybe_clone->ToObject(&clone)) return maybe_clone; } ASSERT(Heap::InNewSpace(clone)); // Since we know the clone is allocated in new space, we can copy // the contents without worrying about updating the write barrier. CopyBlock(HeapObject::cast(clone)->address(), source->address(), object_size); } FixedArray* elements = FixedArray::cast(source->elements()); FixedArray* properties = FixedArray::cast(source->properties()); // Update elements if necessary. if (elements->length() > 0) { Object* elem; { MaybeObject* maybe_elem = (elements->map() == fixed_cow_array_map()) ? elements : CopyFixedArray(elements); if (!maybe_elem->ToObject(&elem)) return maybe_elem; } JSObject::cast(clone)->set_elements(FixedArray::cast(elem)); } // Update properties if necessary. if (properties->length() > 0) { Object* prop; { MaybeObject* maybe_prop = CopyFixedArray(properties); if (!maybe_prop->ToObject(&prop)) return maybe_prop; } JSObject::cast(clone)->set_properties(FixedArray::cast(prop)); } // Return the new clone. #ifdef ENABLE_LOGGING_AND_PROFILING ProducerHeapProfile::RecordJSObjectAllocation(clone); #endif return clone; } MaybeObject* Heap::ReinitializeJSGlobalProxy(JSFunction* constructor, JSGlobalProxy* object) { ASSERT(constructor->has_initial_map()); Map* map = constructor->initial_map(); // Check that the already allocated object has the same size and type as // objects allocated using the constructor. ASSERT(map->instance_size() == object->map()->instance_size()); ASSERT(map->instance_type() == object->map()->instance_type()); // Allocate the backing storage for the properties. int prop_size = map->unused_property_fields() - map->inobject_properties(); Object* properties; { MaybeObject* maybe_properties = AllocateFixedArray(prop_size, TENURED); if (!maybe_properties->ToObject(&properties)) return maybe_properties; } // Reset the map for the object. object->set_map(constructor->initial_map()); // Reinitialize the object from the constructor map. InitializeJSObjectFromMap(object, FixedArray::cast(properties), map); return object; } MaybeObject* Heap::AllocateStringFromAscii(Vector string, PretenureFlag pretenure) { Object* result; { MaybeObject* maybe_result = AllocateRawAsciiString(string.length(), pretenure); if (!maybe_result->ToObject(&result)) return maybe_result; } // Copy the characters into the new object. SeqAsciiString* string_result = SeqAsciiString::cast(result); for (int i = 0; i < string.length(); i++) { string_result->SeqAsciiStringSet(i, string[i]); } return result; } MaybeObject* Heap::AllocateStringFromUtf8Slow(Vector string, PretenureFlag pretenure) { // V8 only supports characters in the Basic Multilingual Plane. const uc32 kMaxSupportedChar = 0xFFFF; // Count the number of characters in the UTF-8 string and check if // it is an ASCII string. Access decoder(ScannerConstants::utf8_decoder()); decoder->Reset(string.start(), string.length()); int chars = 0; while (decoder->has_more()) { decoder->GetNext(); chars++; } Object* result; { MaybeObject* maybe_result = AllocateRawTwoByteString(chars, pretenure); if (!maybe_result->ToObject(&result)) return maybe_result; } // Convert and copy the characters into the new object. String* string_result = String::cast(result); decoder->Reset(string.start(), string.length()); for (int i = 0; i < chars; i++) { uc32 r = decoder->GetNext(); if (r > kMaxSupportedChar) { r = unibrow::Utf8::kBadChar; } string_result->Set(i, r); } return result; } MaybeObject* Heap::AllocateStringFromTwoByte(Vector string, PretenureFlag pretenure) { // Check if the string is an ASCII string. MaybeObject* maybe_result; if (String::IsAscii(string.start(), string.length())) { maybe_result = AllocateRawAsciiString(string.length(), pretenure); } else { // It's not an ASCII string. maybe_result = AllocateRawTwoByteString(string.length(), pretenure); } Object* result; if (!maybe_result->ToObject(&result)) return maybe_result; // Copy the characters into the new object, which may be either ASCII or // UTF-16. String* string_result = String::cast(result); for (int i = 0; i < string.length(); i++) { string_result->Set(i, string[i]); } return result; } Map* Heap::SymbolMapForString(String* string) { // If the string is in new space it cannot be used as a symbol. if (InNewSpace(string)) return NULL; // Find the corresponding symbol map for strings. Map* map = string->map(); if (map == ascii_string_map()) return ascii_symbol_map(); if (map == string_map()) return symbol_map(); if (map == cons_string_map()) return cons_symbol_map(); if (map == cons_ascii_string_map()) return cons_ascii_symbol_map(); if (map == external_string_map()) return external_symbol_map(); if (map == external_ascii_string_map()) return external_ascii_symbol_map(); if (map == external_string_with_ascii_data_map()) { return external_symbol_with_ascii_data_map(); } // No match found. return NULL; } MaybeObject* Heap::AllocateInternalSymbol(unibrow::CharacterStream* buffer, int chars, uint32_t hash_field) { ASSERT(chars >= 0); // Ensure the chars matches the number of characters in the buffer. ASSERT(static_cast(chars) == buffer->Length()); // Determine whether the string is ascii. bool is_ascii = true; while (buffer->has_more()) { if (buffer->GetNext() > unibrow::Utf8::kMaxOneByteChar) { is_ascii = false; break; } } buffer->Rewind(); // Compute map and object size. int size; Map* map; if (is_ascii) { if (chars > SeqAsciiString::kMaxLength) { return Failure::OutOfMemoryException(); } map = ascii_symbol_map(); size = SeqAsciiString::SizeFor(chars); } else { if (chars > SeqTwoByteString::kMaxLength) { return Failure::OutOfMemoryException(); } map = symbol_map(); size = SeqTwoByteString::SizeFor(chars); } // Allocate string. Object* result; { MaybeObject* maybe_result = (size > MaxObjectSizeInPagedSpace()) ? lo_space_->AllocateRaw(size) : old_data_space_->AllocateRaw(size); if (!maybe_result->ToObject(&result)) return maybe_result; } reinterpret_cast(result)->set_map(map); // Set length and hash fields of the allocated string. String* answer = String::cast(result); answer->set_length(chars); answer->set_hash_field(hash_field); ASSERT_EQ(size, answer->Size()); // Fill in the characters. for (int i = 0; i < chars; i++) { answer->Set(i, buffer->GetNext()); } return answer; } MaybeObject* Heap::AllocateRawAsciiString(int length, PretenureFlag pretenure) { if (length < 0 || length > SeqAsciiString::kMaxLength) { return Failure::OutOfMemoryException(); } int size = SeqAsciiString::SizeFor(length); ASSERT(size <= SeqAsciiString::kMaxSize); AllocationSpace space = (pretenure == TENURED) ? OLD_DATA_SPACE : NEW_SPACE; AllocationSpace retry_space = OLD_DATA_SPACE; if (space == NEW_SPACE) { if (size > kMaxObjectSizeInNewSpace) { // Allocate in large object space, retry space will be ignored. space = LO_SPACE; } else if (size > MaxObjectSizeInPagedSpace()) { // Allocate in new space, retry in large object space. retry_space = LO_SPACE; } } else if (space == OLD_DATA_SPACE && size > MaxObjectSizeInPagedSpace()) { space = LO_SPACE; } Object* result; { MaybeObject* maybe_result = AllocateRaw(size, space, retry_space); if (!maybe_result->ToObject(&result)) return maybe_result; } // Partially initialize the object. HeapObject::cast(result)->set_map(ascii_string_map()); String::cast(result)->set_length(length); String::cast(result)->set_hash_field(String::kEmptyHashField); ASSERT_EQ(size, HeapObject::cast(result)->Size()); return result; } MaybeObject* Heap::AllocateRawTwoByteString(int length, PretenureFlag pretenure) { if (length < 0 || length > SeqTwoByteString::kMaxLength) { return Failure::OutOfMemoryException(); } int size = SeqTwoByteString::SizeFor(length); ASSERT(size <= SeqTwoByteString::kMaxSize); AllocationSpace space = (pretenure == TENURED) ? OLD_DATA_SPACE : NEW_SPACE; AllocationSpace retry_space = OLD_DATA_SPACE; if (space == NEW_SPACE) { if (size > kMaxObjectSizeInNewSpace) { // Allocate in large object space, retry space will be ignored. space = LO_SPACE; } else if (size > MaxObjectSizeInPagedSpace()) { // Allocate in new space, retry in large object space. retry_space = LO_SPACE; } } else if (space == OLD_DATA_SPACE && size > MaxObjectSizeInPagedSpace()) { space = LO_SPACE; } Object* result; { MaybeObject* maybe_result = AllocateRaw(size, space, retry_space); if (!maybe_result->ToObject(&result)) return maybe_result; } // Partially initialize the object. HeapObject::cast(result)->set_map(string_map()); String::cast(result)->set_length(length); String::cast(result)->set_hash_field(String::kEmptyHashField); ASSERT_EQ(size, HeapObject::cast(result)->Size()); return result; } MaybeObject* Heap::AllocateEmptyFixedArray() { int size = FixedArray::SizeFor(0); Object* result; { MaybeObject* maybe_result = AllocateRaw(size, OLD_DATA_SPACE, OLD_DATA_SPACE); if (!maybe_result->ToObject(&result)) return maybe_result; } // Initialize the object. reinterpret_cast(result)->set_map(fixed_array_map()); reinterpret_cast(result)->set_length(0); return result; } MaybeObject* Heap::AllocateRawFixedArray(int length) { if (length < 0 || length > FixedArray::kMaxLength) { return Failure::OutOfMemoryException(); } ASSERT(length > 0); // Use the general function if we're forced to always allocate. if (always_allocate()) return AllocateFixedArray(length, TENURED); // Allocate the raw data for a fixed array. int size = FixedArray::SizeFor(length); return size <= kMaxObjectSizeInNewSpace ? new_space_.AllocateRaw(size) : lo_space_->AllocateRawFixedArray(size); } MaybeObject* Heap::CopyFixedArrayWithMap(FixedArray* src, Map* map) { int len = src->length(); Object* obj; { MaybeObject* maybe_obj = AllocateRawFixedArray(len); if (!maybe_obj->ToObject(&obj)) return maybe_obj; } if (Heap::InNewSpace(obj)) { HeapObject* dst = HeapObject::cast(obj); dst->set_map(map); CopyBlock(dst->address() + kPointerSize, src->address() + kPointerSize, FixedArray::SizeFor(len) - kPointerSize); return obj; } HeapObject::cast(obj)->set_map(map); FixedArray* result = FixedArray::cast(obj); result->set_length(len); // Copy the content AssertNoAllocation no_gc; WriteBarrierMode mode = result->GetWriteBarrierMode(no_gc); for (int i = 0; i < len; i++) result->set(i, src->get(i), mode); return result; } MaybeObject* Heap::AllocateFixedArray(int length) { ASSERT(length >= 0); if (length == 0) return empty_fixed_array(); Object* result; { MaybeObject* maybe_result = AllocateRawFixedArray(length); if (!maybe_result->ToObject(&result)) return maybe_result; } // Initialize header. FixedArray* array = reinterpret_cast(result); array->set_map(fixed_array_map()); array->set_length(length); // Initialize body. ASSERT(!Heap::InNewSpace(undefined_value())); MemsetPointer(array->data_start(), undefined_value(), length); return result; } MaybeObject* Heap::AllocateRawFixedArray(int length, PretenureFlag pretenure) { if (length < 0 || length > FixedArray::kMaxLength) { return Failure::OutOfMemoryException(); } AllocationSpace space = (pretenure == TENURED) ? OLD_POINTER_SPACE : NEW_SPACE; int size = FixedArray::SizeFor(length); if (space == NEW_SPACE && size > kMaxObjectSizeInNewSpace) { // Too big for new space. space = LO_SPACE; } else if (space == OLD_POINTER_SPACE && size > MaxObjectSizeInPagedSpace()) { // Too big for old pointer space. space = LO_SPACE; } AllocationSpace retry_space = (size <= MaxObjectSizeInPagedSpace()) ? OLD_POINTER_SPACE : LO_SPACE; return AllocateRaw(size, space, retry_space); } MUST_USE_RESULT static MaybeObject* AllocateFixedArrayWithFiller( int length, PretenureFlag pretenure, Object* filler) { ASSERT(length >= 0); ASSERT(Heap::empty_fixed_array()->IsFixedArray()); if (length == 0) return Heap::empty_fixed_array(); ASSERT(!Heap::InNewSpace(filler)); Object* result; { MaybeObject* maybe_result = Heap::AllocateRawFixedArray(length, pretenure); if (!maybe_result->ToObject(&result)) return maybe_result; } HeapObject::cast(result)->set_map(Heap::fixed_array_map()); FixedArray* array = FixedArray::cast(result); array->set_length(length); MemsetPointer(array->data_start(), filler, length); return array; } MaybeObject* Heap::AllocateFixedArray(int length, PretenureFlag pretenure) { return AllocateFixedArrayWithFiller(length, pretenure, undefined_value()); } MaybeObject* Heap::AllocateFixedArrayWithHoles(int length, PretenureFlag pretenure) { return AllocateFixedArrayWithFiller(length, pretenure, the_hole_value()); } MaybeObject* Heap::AllocateUninitializedFixedArray(int length) { if (length == 0) return empty_fixed_array(); Object* obj; { MaybeObject* maybe_obj = AllocateRawFixedArray(length); if (!maybe_obj->ToObject(&obj)) return maybe_obj; } reinterpret_cast(obj)->set_map(fixed_array_map()); FixedArray::cast(obj)->set_length(length); return obj; } MaybeObject* Heap::AllocateHashTable(int length, PretenureFlag pretenure) { Object* result; { MaybeObject* maybe_result = Heap::AllocateFixedArray(length, pretenure); if (!maybe_result->ToObject(&result)) return maybe_result; } reinterpret_cast(result)->set_map(hash_table_map()); ASSERT(result->IsHashTable()); return result; } MaybeObject* Heap::AllocateGlobalContext() { Object* result; { MaybeObject* maybe_result = Heap::AllocateFixedArray(Context::GLOBAL_CONTEXT_SLOTS); if (!maybe_result->ToObject(&result)) return maybe_result; } Context* context = reinterpret_cast(result); context->set_map(global_context_map()); ASSERT(context->IsGlobalContext()); ASSERT(result->IsContext()); return result; } MaybeObject* Heap::AllocateFunctionContext(int length, JSFunction* function) { ASSERT(length >= Context::MIN_CONTEXT_SLOTS); Object* result; { MaybeObject* maybe_result = Heap::AllocateFixedArray(length); if (!maybe_result->ToObject(&result)) return maybe_result; } Context* context = reinterpret_cast(result); context->set_map(context_map()); context->set_closure(function); context->set_fcontext(context); context->set_previous(NULL); context->set_extension(NULL); context->set_global(function->context()->global()); ASSERT(!context->IsGlobalContext()); ASSERT(context->is_function_context()); ASSERT(result->IsContext()); return result; } MaybeObject* Heap::AllocateWithContext(Context* previous, JSObject* extension, bool is_catch_context) { Object* result; { MaybeObject* maybe_result = Heap::AllocateFixedArray(Context::MIN_CONTEXT_SLOTS); if (!maybe_result->ToObject(&result)) return maybe_result; } Context* context = reinterpret_cast(result); context->set_map(is_catch_context ? catch_context_map() : context_map()); context->set_closure(previous->closure()); context->set_fcontext(previous->fcontext()); context->set_previous(previous); context->set_extension(extension); context->set_global(previous->global()); ASSERT(!context->IsGlobalContext()); ASSERT(!context->is_function_context()); ASSERT(result->IsContext()); return result; } MaybeObject* Heap::AllocateStruct(InstanceType type) { Map* map; switch (type) { #define MAKE_CASE(NAME, Name, name) case NAME##_TYPE: map = name##_map(); break; STRUCT_LIST(MAKE_CASE) #undef MAKE_CASE default: UNREACHABLE(); return Failure::InternalError(); } int size = map->instance_size(); AllocationSpace space = (size > MaxObjectSizeInPagedSpace()) ? LO_SPACE : OLD_POINTER_SPACE; Object* result; { MaybeObject* maybe_result = Heap::Allocate(map, space); if (!maybe_result->ToObject(&result)) return maybe_result; } Struct::cast(result)->InitializeBody(size); return result; } bool Heap::IdleNotification() { static const int kIdlesBeforeScavenge = 4; static const int kIdlesBeforeMarkSweep = 7; static const int kIdlesBeforeMarkCompact = 8; static const int kMaxIdleCount = kIdlesBeforeMarkCompact + 1; static const int kGCsBetweenCleanup = 4; static int number_idle_notifications = 0; static int last_gc_count = gc_count_; bool uncommit = true; bool finished = false; // Reset the number of idle notifications received when a number of // GCs have taken place. This allows another round of cleanup based // on idle notifications if enough work has been carried out to // provoke a number of garbage collections. if (gc_count_ < last_gc_count + kGCsBetweenCleanup) { number_idle_notifications = Min(number_idle_notifications + 1, kMaxIdleCount); } else { number_idle_notifications = 0; last_gc_count = gc_count_; } if (number_idle_notifications == kIdlesBeforeScavenge) { if (contexts_disposed_ > 0) { HistogramTimerScope scope(&Counters::gc_context); CollectAllGarbage(false); } else { CollectGarbage(NEW_SPACE); } new_space_.Shrink(); last_gc_count = gc_count_; } else if (number_idle_notifications == kIdlesBeforeMarkSweep) { // Before doing the mark-sweep collections we clear the // compilation cache to avoid hanging on to source code and // generated code for cached functions. CompilationCache::Clear(); CollectAllGarbage(false); new_space_.Shrink(); last_gc_count = gc_count_; } else if (number_idle_notifications == kIdlesBeforeMarkCompact) { CollectAllGarbage(true); new_space_.Shrink(); last_gc_count = gc_count_; finished = true; } else if (contexts_disposed_ > 0) { if (FLAG_expose_gc) { contexts_disposed_ = 0; } else { HistogramTimerScope scope(&Counters::gc_context); CollectAllGarbage(false); last_gc_count = gc_count_; } // If this is the first idle notification, we reset the // notification count to avoid letting idle notifications for // context disposal garbage collections start a potentially too // aggressive idle GC cycle. if (number_idle_notifications <= 1) { number_idle_notifications = 0; uncommit = false; } } else if (number_idle_notifications > kIdlesBeforeMarkCompact) { // If we have received more than kIdlesBeforeMarkCompact idle // notifications we do not perform any cleanup because we don't // expect to gain much by doing so. finished = true; } // Make sure that we have no pending context disposals and // conditionally uncommit from space. ASSERT(contexts_disposed_ == 0); if (uncommit) Heap::UncommitFromSpace(); return finished; } #ifdef DEBUG void Heap::Print() { if (!HasBeenSetup()) return; Top::PrintStack(); AllSpaces spaces; for (Space* space = spaces.next(); space != NULL; space = spaces.next()) space->Print(); } void Heap::ReportCodeStatistics(const char* title) { PrintF(">>>>>> Code Stats (%s) >>>>>>\n", title); PagedSpace::ResetCodeStatistics(); // We do not look for code in new space, map space, or old space. If code // somehow ends up in those spaces, we would miss it here. code_space_->CollectCodeStatistics(); lo_space_->CollectCodeStatistics(); PagedSpace::ReportCodeStatistics(); } // This function expects that NewSpace's allocated objects histogram is // populated (via a call to CollectStatistics or else as a side effect of a // just-completed scavenge collection). void Heap::ReportHeapStatistics(const char* title) { USE(title); PrintF(">>>>>> =============== %s (%d) =============== >>>>>>\n", title, gc_count_); PrintF("mark-compact GC : %d\n", mc_count_); PrintF("old_gen_promotion_limit_ %" V8_PTR_PREFIX "d\n", old_gen_promotion_limit_); PrintF("old_gen_allocation_limit_ %" V8_PTR_PREFIX "d\n", old_gen_allocation_limit_); PrintF("\n"); PrintF("Number of handles : %d\n", HandleScope::NumberOfHandles()); GlobalHandles::PrintStats(); PrintF("\n"); PrintF("Heap statistics : "); MemoryAllocator::ReportStatistics(); PrintF("To space : "); new_space_.ReportStatistics(); PrintF("Old pointer space : "); old_pointer_space_->ReportStatistics(); PrintF("Old data space : "); old_data_space_->ReportStatistics(); PrintF("Code space : "); code_space_->ReportStatistics(); PrintF("Map space : "); map_space_->ReportStatistics(); PrintF("Cell space : "); cell_space_->ReportStatistics(); PrintF("Large object space : "); lo_space_->ReportStatistics(); PrintF(">>>>>> ========================================= >>>>>>\n"); } #endif // DEBUG bool Heap::Contains(HeapObject* value) { return Contains(value->address()); } bool Heap::Contains(Address addr) { if (OS::IsOutsideAllocatedSpace(addr)) return false; return HasBeenSetup() && (new_space_.ToSpaceContains(addr) || old_pointer_space_->Contains(addr) || old_data_space_->Contains(addr) || code_space_->Contains(addr) || map_space_->Contains(addr) || cell_space_->Contains(addr) || lo_space_->SlowContains(addr)); } bool Heap::InSpace(HeapObject* value, AllocationSpace space) { return InSpace(value->address(), space); } bool Heap::InSpace(Address addr, AllocationSpace space) { if (OS::IsOutsideAllocatedSpace(addr)) return false; if (!HasBeenSetup()) return false; switch (space) { case NEW_SPACE: return new_space_.ToSpaceContains(addr); case OLD_POINTER_SPACE: return old_pointer_space_->Contains(addr); case OLD_DATA_SPACE: return old_data_space_->Contains(addr); case CODE_SPACE: return code_space_->Contains(addr); case MAP_SPACE: return map_space_->Contains(addr); case CELL_SPACE: return cell_space_->Contains(addr); case LO_SPACE: return lo_space_->SlowContains(addr); } return false; } #ifdef DEBUG static void DummyScavengePointer(HeapObject** p) { } static void VerifyPointersUnderWatermark( PagedSpace* space, DirtyRegionCallback visit_dirty_region) { PageIterator it(space, PageIterator::PAGES_IN_USE); while (it.has_next()) { Page* page = it.next(); Address start = page->ObjectAreaStart(); Address end = page->AllocationWatermark(); Heap::IterateDirtyRegions(Page::kAllRegionsDirtyMarks, start, end, visit_dirty_region, &DummyScavengePointer); } } static void VerifyPointersUnderWatermark(LargeObjectSpace* space) { LargeObjectIterator it(space); for (HeapObject* object = it.next(); object != NULL; object = it.next()) { if (object->IsFixedArray()) { Address slot_address = object->address(); Address end = object->address() + object->Size(); while (slot_address < end) { HeapObject** slot = reinterpret_cast(slot_address); // When we are not in GC the Heap::InNewSpace() predicate // checks that pointers which satisfy predicate point into // the active semispace. Heap::InNewSpace(*slot); slot_address += kPointerSize; } } } } void Heap::Verify() { ASSERT(HasBeenSetup()); VerifyPointersVisitor visitor; IterateRoots(&visitor, VISIT_ONLY_STRONG); new_space_.Verify(); VerifyPointersAndDirtyRegionsVisitor dirty_regions_visitor; old_pointer_space_->Verify(&dirty_regions_visitor); map_space_->Verify(&dirty_regions_visitor); VerifyPointersUnderWatermark(old_pointer_space_, &IteratePointersInDirtyRegion); VerifyPointersUnderWatermark(map_space_, &IteratePointersInDirtyMapsRegion); VerifyPointersUnderWatermark(lo_space_); VerifyPageWatermarkValidity(old_pointer_space_, ALL_INVALID); VerifyPageWatermarkValidity(map_space_, ALL_INVALID); VerifyPointersVisitor no_dirty_regions_visitor; old_data_space_->Verify(&no_dirty_regions_visitor); code_space_->Verify(&no_dirty_regions_visitor); cell_space_->Verify(&no_dirty_regions_visitor); lo_space_->Verify(); } #endif // DEBUG MaybeObject* Heap::LookupSymbol(Vector string) { Object* symbol = NULL; Object* new_table; { MaybeObject* maybe_new_table = symbol_table()->LookupSymbol(string, &symbol); if (!maybe_new_table->ToObject(&new_table)) return maybe_new_table; } // Can't use set_symbol_table because SymbolTable::cast knows that // SymbolTable is a singleton and checks for identity. roots_[kSymbolTableRootIndex] = new_table; ASSERT(symbol != NULL); return symbol; } MaybeObject* Heap::LookupAsciiSymbol(Vector string) { Object* symbol = NULL; Object* new_table; { MaybeObject* maybe_new_table = symbol_table()->LookupAsciiSymbol(string, &symbol); if (!maybe_new_table->ToObject(&new_table)) return maybe_new_table; } // Can't use set_symbol_table because SymbolTable::cast knows that // SymbolTable is a singleton and checks for identity. roots_[kSymbolTableRootIndex] = new_table; ASSERT(symbol != NULL); return symbol; } MaybeObject* Heap::LookupTwoByteSymbol(Vector string) { Object* symbol = NULL; Object* new_table; { MaybeObject* maybe_new_table = symbol_table()->LookupTwoByteSymbol(string, &symbol); if (!maybe_new_table->ToObject(&new_table)) return maybe_new_table; } // Can't use set_symbol_table because SymbolTable::cast knows that // SymbolTable is a singleton and checks for identity. roots_[kSymbolTableRootIndex] = new_table; ASSERT(symbol != NULL); return symbol; } MaybeObject* Heap::LookupSymbol(String* string) { if (string->IsSymbol()) return string; Object* symbol = NULL; Object* new_table; { MaybeObject* maybe_new_table = symbol_table()->LookupString(string, &symbol); if (!maybe_new_table->ToObject(&new_table)) return maybe_new_table; } // Can't use set_symbol_table because SymbolTable::cast knows that // SymbolTable is a singleton and checks for identity. roots_[kSymbolTableRootIndex] = new_table; ASSERT(symbol != NULL); return symbol; } bool Heap::LookupSymbolIfExists(String* string, String** symbol) { if (string->IsSymbol()) { *symbol = string; return true; } return symbol_table()->LookupSymbolIfExists(string, symbol); } #ifdef DEBUG void Heap::ZapFromSpace() { ASSERT(reinterpret_cast(kFromSpaceZapValue)->IsHeapObject()); for (Address a = new_space_.FromSpaceLow(); a < new_space_.FromSpaceHigh(); a += kPointerSize) { Memory::Address_at(a) = kFromSpaceZapValue; } } #endif // DEBUG bool Heap::IteratePointersInDirtyRegion(Address start, Address end, ObjectSlotCallback copy_object_func) { Address slot_address = start; bool pointers_to_new_space_found = false; while (slot_address < end) { Object** slot = reinterpret_cast(slot_address); if (Heap::InNewSpace(*slot)) { ASSERT((*slot)->IsHeapObject()); copy_object_func(reinterpret_cast(slot)); if (Heap::InNewSpace(*slot)) { ASSERT((*slot)->IsHeapObject()); pointers_to_new_space_found = true; } } slot_address += kPointerSize; } return pointers_to_new_space_found; } // Compute start address of the first map following given addr. static inline Address MapStartAlign(Address addr) { Address page = Page::FromAddress(addr)->ObjectAreaStart(); return page + (((addr - page) + (Map::kSize - 1)) / Map::kSize * Map::kSize); } // Compute end address of the first map preceding given addr. static inline Address MapEndAlign(Address addr) { Address page = Page::FromAllocationTop(addr)->ObjectAreaStart(); return page + ((addr - page) / Map::kSize * Map::kSize); } static bool IteratePointersInDirtyMaps(Address start, Address end, ObjectSlotCallback copy_object_func) { ASSERT(MapStartAlign(start) == start); ASSERT(MapEndAlign(end) == end); Address map_address = start; bool pointers_to_new_space_found = false; while (map_address < end) { ASSERT(!Heap::InNewSpace(Memory::Object_at(map_address))); ASSERT(Memory::Object_at(map_address)->IsMap()); Address pointer_fields_start = map_address + Map::kPointerFieldsBeginOffset; Address pointer_fields_end = map_address + Map::kPointerFieldsEndOffset; if (Heap::IteratePointersInDirtyRegion(pointer_fields_start, pointer_fields_end, copy_object_func)) { pointers_to_new_space_found = true; } map_address += Map::kSize; } return pointers_to_new_space_found; } bool Heap::IteratePointersInDirtyMapsRegion( Address start, Address end, ObjectSlotCallback copy_object_func) { Address map_aligned_start = MapStartAlign(start); Address map_aligned_end = MapEndAlign(end); bool contains_pointers_to_new_space = false; if (map_aligned_start != start) { Address prev_map = map_aligned_start - Map::kSize; ASSERT(Memory::Object_at(prev_map)->IsMap()); Address pointer_fields_start = Max(start, prev_map + Map::kPointerFieldsBeginOffset); Address pointer_fields_end = Min(prev_map + Map::kPointerFieldsEndOffset, end); contains_pointers_to_new_space = IteratePointersInDirtyRegion(pointer_fields_start, pointer_fields_end, copy_object_func) || contains_pointers_to_new_space; } contains_pointers_to_new_space = IteratePointersInDirtyMaps(map_aligned_start, map_aligned_end, copy_object_func) || contains_pointers_to_new_space; if (map_aligned_end != end) { ASSERT(Memory::Object_at(map_aligned_end)->IsMap()); Address pointer_fields_start = map_aligned_end + Map::kPointerFieldsBeginOffset; Address pointer_fields_end = Min(end, map_aligned_end + Map::kPointerFieldsEndOffset); contains_pointers_to_new_space = IteratePointersInDirtyRegion(pointer_fields_start, pointer_fields_end, copy_object_func) || contains_pointers_to_new_space; } return contains_pointers_to_new_space; } void Heap::IterateAndMarkPointersToFromSpace(Address start, Address end, ObjectSlotCallback callback) { Address slot_address = start; Page* page = Page::FromAddress(start); uint32_t marks = page->GetRegionMarks(); while (slot_address < end) { Object** slot = reinterpret_cast(slot_address); if (Heap::InFromSpace(*slot)) { ASSERT((*slot)->IsHeapObject()); callback(reinterpret_cast(slot)); if (Heap::InNewSpace(*slot)) { ASSERT((*slot)->IsHeapObject()); marks |= page->GetRegionMaskForAddress(slot_address); } } slot_address += kPointerSize; } page->SetRegionMarks(marks); } uint32_t Heap::IterateDirtyRegions( uint32_t marks, Address area_start, Address area_end, DirtyRegionCallback visit_dirty_region, ObjectSlotCallback copy_object_func) { uint32_t newmarks = 0; uint32_t mask = 1; if (area_start >= area_end) { return newmarks; } Address region_start = area_start; // area_start does not necessarily coincide with start of the first region. // Thus to calculate the beginning of the next region we have to align // area_start by Page::kRegionSize. Address second_region = reinterpret_cast
( reinterpret_cast(area_start + Page::kRegionSize) & ~Page::kRegionAlignmentMask); // Next region might be beyond area_end. Address region_end = Min(second_region, area_end); if (marks & mask) { if (visit_dirty_region(region_start, region_end, copy_object_func)) { newmarks |= mask; } } mask <<= 1; // Iterate subsequent regions which fully lay inside [area_start, area_end[. region_start = region_end; region_end = region_start + Page::kRegionSize; while (region_end <= area_end) { if (marks & mask) { if (visit_dirty_region(region_start, region_end, copy_object_func)) { newmarks |= mask; } } region_start = region_end; region_end = region_start + Page::kRegionSize; mask <<= 1; } if (region_start != area_end) { // A small piece of area left uniterated because area_end does not coincide // with region end. Check whether region covering last part of area is // dirty. if (marks & mask) { if (visit_dirty_region(region_start, area_end, copy_object_func)) { newmarks |= mask; } } } return newmarks; } void Heap::IterateDirtyRegions( PagedSpace* space, DirtyRegionCallback visit_dirty_region, ObjectSlotCallback copy_object_func, ExpectedPageWatermarkState expected_page_watermark_state) { PageIterator it(space, PageIterator::PAGES_IN_USE); while (it.has_next()) { Page* page = it.next(); uint32_t marks = page->GetRegionMarks(); if (marks != Page::kAllRegionsCleanMarks) { Address start = page->ObjectAreaStart(); // Do not try to visit pointers beyond page allocation watermark. // Page can contain garbage pointers there. Address end; if ((expected_page_watermark_state == WATERMARK_SHOULD_BE_VALID) || page->IsWatermarkValid()) { end = page->AllocationWatermark(); } else { end = page->CachedAllocationWatermark(); } ASSERT(space == old_pointer_space_ || (space == map_space_ && ((page->ObjectAreaStart() - end) % Map::kSize == 0))); page->SetRegionMarks(IterateDirtyRegions(marks, start, end, visit_dirty_region, copy_object_func)); } // Mark page watermark as invalid to maintain watermark validity invariant. // See Page::FlipMeaningOfInvalidatedWatermarkFlag() for details. page->InvalidateWatermark(true); } } void Heap::IterateRoots(ObjectVisitor* v, VisitMode mode) { IterateStrongRoots(v, mode); IterateWeakRoots(v, mode); } void Heap::IterateWeakRoots(ObjectVisitor* v, VisitMode mode) { v->VisitPointer(reinterpret_cast(&roots_[kSymbolTableRootIndex])); v->Synchronize("symbol_table"); if (mode != VISIT_ALL_IN_SCAVENGE) { // Scavenge collections have special processing for this. ExternalStringTable::Iterate(v); } v->Synchronize("external_string_table"); } void Heap::IterateStrongRoots(ObjectVisitor* v, VisitMode mode) { v->VisitPointers(&roots_[0], &roots_[kStrongRootListLength]); v->Synchronize("strong_root_list"); v->VisitPointer(BitCast(&hidden_symbol_)); v->Synchronize("symbol"); Bootstrapper::Iterate(v); v->Synchronize("bootstrapper"); Top::Iterate(v); v->Synchronize("top"); Relocatable::Iterate(v); v->Synchronize("relocatable"); #ifdef ENABLE_DEBUGGER_SUPPORT Debug::Iterate(v); #endif v->Synchronize("debug"); CompilationCache::Iterate(v); v->Synchronize("compilationcache"); // Iterate over local handles in handle scopes. HandleScopeImplementer::Iterate(v); v->Synchronize("handlescope"); // Iterate over the builtin code objects and code stubs in the // heap. Note that it is not necessary to iterate over code objects // on scavenge collections. if (mode != VISIT_ALL_IN_SCAVENGE) { Builtins::IterateBuiltins(v); } v->Synchronize("builtins"); // Iterate over global handles. if (mode == VISIT_ONLY_STRONG) { GlobalHandles::IterateStrongRoots(v); } else { GlobalHandles::IterateAllRoots(v); } v->Synchronize("globalhandles"); // Iterate over pointers being held by inactive threads. ThreadManager::Iterate(v); v->Synchronize("threadmanager"); // Iterate over the pointers the Serialization/Deserialization code is // holding. // During garbage collection this keeps the partial snapshot cache alive. // During deserialization of the startup snapshot this creates the partial // snapshot cache and deserializes the objects it refers to. During // serialization this does nothing, since the partial snapshot cache is // empty. However the next thing we do is create the partial snapshot, // filling up the partial snapshot cache with objects it needs as we go. SerializerDeserializer::Iterate(v); // We don't do a v->Synchronize call here, because in debug mode that will // output a flag to the snapshot. However at this point the serializer and // deserializer are deliberately a little unsynchronized (see above) so the // checking of the sync flag in the snapshot would fail. } // Flag is set when the heap has been configured. The heap can be repeatedly // configured through the API until it is setup. static bool heap_configured = false; // TODO(1236194): Since the heap size is configurable on the command line // and through the API, we should gracefully handle the case that the heap // size is not big enough to fit all the initial objects. bool Heap::ConfigureHeap(int max_semispace_size, int max_old_gen_size, int max_executable_size) { if (HasBeenSetup()) return false; if (max_semispace_size > 0) max_semispace_size_ = max_semispace_size; if (Snapshot::IsEnabled()) { // If we are using a snapshot we always reserve the default amount // of memory for each semispace because code in the snapshot has // write-barrier code that relies on the size and alignment of new // space. We therefore cannot use a larger max semispace size // than the default reserved semispace size. if (max_semispace_size_ > reserved_semispace_size_) { max_semispace_size_ = reserved_semispace_size_; } } else { // If we are not using snapshots we reserve space for the actual // max semispace size. reserved_semispace_size_ = max_semispace_size_; } if (max_old_gen_size > 0) max_old_generation_size_ = max_old_gen_size; if (max_executable_size > 0) { max_executable_size_ = RoundUp(max_executable_size, Page::kPageSize); } // The max executable size must be less than or equal to the max old // generation size. if (max_executable_size_ > max_old_generation_size_) { max_executable_size_ = max_old_generation_size_; } // The new space size must be a power of two to support single-bit testing // for containment. max_semispace_size_ = RoundUpToPowerOf2(max_semispace_size_); reserved_semispace_size_ = RoundUpToPowerOf2(reserved_semispace_size_); initial_semispace_size_ = Min(initial_semispace_size_, max_semispace_size_); external_allocation_limit_ = 10 * max_semispace_size_; // The old generation is paged. max_old_generation_size_ = RoundUp(max_old_generation_size_, Page::kPageSize); heap_configured = true; return true; } bool Heap::ConfigureHeapDefault() { return ConfigureHeap(FLAG_max_new_space_size / 2 * KB, FLAG_max_old_space_size * MB, FLAG_max_executable_size * MB); } void Heap::RecordStats(HeapStats* stats, bool take_snapshot) { *stats->start_marker = HeapStats::kStartMarker; *stats->end_marker = HeapStats::kEndMarker; *stats->new_space_size = new_space_.SizeAsInt(); *stats->new_space_capacity = static_cast(new_space_.Capacity()); *stats->old_pointer_space_size = old_pointer_space_->Size(); *stats->old_pointer_space_capacity = old_pointer_space_->Capacity(); *stats->old_data_space_size = old_data_space_->Size(); *stats->old_data_space_capacity = old_data_space_->Capacity(); *stats->code_space_size = code_space_->Size(); *stats->code_space_capacity = code_space_->Capacity(); *stats->map_space_size = map_space_->Size(); *stats->map_space_capacity = map_space_->Capacity(); *stats->cell_space_size = cell_space_->Size(); *stats->cell_space_capacity = cell_space_->Capacity(); *stats->lo_space_size = lo_space_->Size(); GlobalHandles::RecordStats(stats); *stats->memory_allocator_size = MemoryAllocator::Size(); *stats->memory_allocator_capacity = MemoryAllocator::Size() + MemoryAllocator::Available(); *stats->os_error = OS::GetLastError(); if (take_snapshot) { HeapIterator iterator(HeapIterator::kFilterFreeListNodes); for (HeapObject* obj = iterator.next(); obj != NULL; obj = iterator.next()) { InstanceType type = obj->map()->instance_type(); ASSERT(0 <= type && type <= LAST_TYPE); stats->objects_per_type[type]++; stats->size_per_type[type] += obj->Size(); } } } intptr_t Heap::PromotedSpaceSize() { return old_pointer_space_->Size() + old_data_space_->Size() + code_space_->Size() + map_space_->Size() + cell_space_->Size() + lo_space_->Size(); } int Heap::PromotedExternalMemorySize() { if (amount_of_external_allocated_memory_ <= amount_of_external_allocated_memory_at_last_global_gc_) return 0; return amount_of_external_allocated_memory_ - amount_of_external_allocated_memory_at_last_global_gc_; } bool Heap::Setup(bool create_heap_objects) { // Initialize heap spaces and initial maps and objects. Whenever something // goes wrong, just return false. The caller should check the results and // call Heap::TearDown() to release allocated memory. // // If the heap is not yet configured (eg, through the API), configure it. // Configuration is based on the flags new-space-size (really the semispace // size) and old-space-size if set or the initial values of semispace_size_ // and old_generation_size_ otherwise. if (!heap_configured) { if (!ConfigureHeapDefault()) return false; } ScavengingVisitor::Initialize(); NewSpaceScavenger::Initialize(); MarkCompactCollector::Initialize(); MarkMapPointersAsEncoded(false); // Setup memory allocator and reserve a chunk of memory for new // space. The chunk is double the size of the requested reserved // new space size to ensure that we can find a pair of semispaces that // are contiguous and aligned to their size. if (!MemoryAllocator::Setup(MaxReserved(), MaxExecutableSize())) return false; void* chunk = MemoryAllocator::ReserveInitialChunk(4 * reserved_semispace_size_); if (chunk == NULL) return false; // Align the pair of semispaces to their size, which must be a power // of 2. Address new_space_start = RoundUp(reinterpret_cast(chunk), 2 * reserved_semispace_size_); if (!new_space_.Setup(new_space_start, 2 * reserved_semispace_size_)) { return false; } // Initialize old pointer space. old_pointer_space_ = new OldSpace(max_old_generation_size_, OLD_POINTER_SPACE, NOT_EXECUTABLE); if (old_pointer_space_ == NULL) return false; if (!old_pointer_space_->Setup(NULL, 0)) return false; // Initialize old data space. old_data_space_ = new OldSpace(max_old_generation_size_, OLD_DATA_SPACE, NOT_EXECUTABLE); if (old_data_space_ == NULL) return false; if (!old_data_space_->Setup(NULL, 0)) return false; // Initialize the code space, set its maximum capacity to the old // generation size. It needs executable memory. // On 64-bit platform(s), we put all code objects in a 2 GB range of // virtual address space, so that they can call each other with near calls. if (code_range_size_ > 0) { if (!CodeRange::Setup(code_range_size_)) { return false; } } code_space_ = new OldSpace(max_old_generation_size_, CODE_SPACE, EXECUTABLE); if (code_space_ == NULL) return false; if (!code_space_->Setup(NULL, 0)) return false; // Initialize map space. map_space_ = new MapSpace(FLAG_use_big_map_space ? max_old_generation_size_ : MapSpace::kMaxMapPageIndex * Page::kPageSize, FLAG_max_map_space_pages, MAP_SPACE); if (map_space_ == NULL) return false; if (!map_space_->Setup(NULL, 0)) return false; // Initialize global property cell space. cell_space_ = new CellSpace(max_old_generation_size_, CELL_SPACE); if (cell_space_ == NULL) return false; if (!cell_space_->Setup(NULL, 0)) return false; // The large object code space may contain code or data. We set the memory // to be non-executable here for safety, but this means we need to enable it // explicitly when allocating large code objects. lo_space_ = new LargeObjectSpace(LO_SPACE); if (lo_space_ == NULL) return false; if (!lo_space_->Setup()) return false; if (create_heap_objects) { // Create initial maps. if (!CreateInitialMaps()) return false; if (!CreateApiObjects()) return false; // Create initial objects if (!CreateInitialObjects()) return false; global_contexts_list_ = undefined_value(); } LOG(IntPtrTEvent("heap-capacity", Capacity())); LOG(IntPtrTEvent("heap-available", Available())); #ifdef ENABLE_LOGGING_AND_PROFILING // This should be called only after initial objects have been created. ProducerHeapProfile::Setup(); #endif return true; } void Heap::SetStackLimits() { // On 64 bit machines, pointers are generally out of range of Smis. We write // something that looks like an out of range Smi to the GC. // Set up the special root array entries containing the stack limits. // These are actually addresses, but the tag makes the GC ignore it. roots_[kStackLimitRootIndex] = reinterpret_cast( (StackGuard::jslimit() & ~kSmiTagMask) | kSmiTag); roots_[kRealStackLimitRootIndex] = reinterpret_cast( (StackGuard::real_jslimit() & ~kSmiTagMask) | kSmiTag); } void Heap::TearDown() { if (FLAG_print_cumulative_gc_stat) { PrintF("\n\n"); PrintF("gc_count=%d ", gc_count_); PrintF("mark_sweep_count=%d ", ms_count_); PrintF("mark_compact_count=%d ", mc_count_); PrintF("max_gc_pause=%d ", GCTracer::get_max_gc_pause()); PrintF("min_in_mutator=%d ", GCTracer::get_min_in_mutator()); PrintF("max_alive_after_gc=%" V8_PTR_PREFIX "d ", GCTracer::get_max_alive_after_gc()); PrintF("\n\n"); } GlobalHandles::TearDown(); ExternalStringTable::TearDown(); new_space_.TearDown(); if (old_pointer_space_ != NULL) { old_pointer_space_->TearDown(); delete old_pointer_space_; old_pointer_space_ = NULL; } if (old_data_space_ != NULL) { old_data_space_->TearDown(); delete old_data_space_; old_data_space_ = NULL; } if (code_space_ != NULL) { code_space_->TearDown(); delete code_space_; code_space_ = NULL; } if (map_space_ != NULL) { map_space_->TearDown(); delete map_space_; map_space_ = NULL; } if (cell_space_ != NULL) { cell_space_->TearDown(); delete cell_space_; cell_space_ = NULL; } if (lo_space_ != NULL) { lo_space_->TearDown(); delete lo_space_; lo_space_ = NULL; } MemoryAllocator::TearDown(); } void Heap::Shrink() { // Try to shrink all paged spaces. PagedSpaces spaces; for (PagedSpace* space = spaces.next(); space != NULL; space = spaces.next()) space->Shrink(); } #ifdef ENABLE_HEAP_PROTECTION void Heap::Protect() { if (HasBeenSetup()) { AllSpaces spaces; for (Space* space = spaces.next(); space != NULL; space = spaces.next()) space->Protect(); } } void Heap::Unprotect() { if (HasBeenSetup()) { AllSpaces spaces; for (Space* space = spaces.next(); space != NULL; space = spaces.next()) space->Unprotect(); } } #endif void Heap::AddGCPrologueCallback(GCPrologueCallback callback, GCType gc_type) { ASSERT(callback != NULL); GCPrologueCallbackPair pair(callback, gc_type); ASSERT(!gc_prologue_callbacks_.Contains(pair)); return gc_prologue_callbacks_.Add(pair); } void Heap::RemoveGCPrologueCallback(GCPrologueCallback callback) { ASSERT(callback != NULL); for (int i = 0; i < gc_prologue_callbacks_.length(); ++i) { if (gc_prologue_callbacks_[i].callback == callback) { gc_prologue_callbacks_.Remove(i); return; } } UNREACHABLE(); } void Heap::AddGCEpilogueCallback(GCEpilogueCallback callback, GCType gc_type) { ASSERT(callback != NULL); GCEpilogueCallbackPair pair(callback, gc_type); ASSERT(!gc_epilogue_callbacks_.Contains(pair)); return gc_epilogue_callbacks_.Add(pair); } void Heap::RemoveGCEpilogueCallback(GCEpilogueCallback callback) { ASSERT(callback != NULL); for (int i = 0; i < gc_epilogue_callbacks_.length(); ++i) { if (gc_epilogue_callbacks_[i].callback == callback) { gc_epilogue_callbacks_.Remove(i); return; } } UNREACHABLE(); } #ifdef DEBUG class PrintHandleVisitor: public ObjectVisitor { public: void VisitPointers(Object** start, Object** end) { for (Object** p = start; p < end; p++) PrintF(" handle %p to %p\n", reinterpret_cast(p), reinterpret_cast(*p)); } }; void Heap::PrintHandles() { PrintF("Handles:\n"); PrintHandleVisitor v; HandleScopeImplementer::Iterate(&v); } #endif Space* AllSpaces::next() { switch (counter_++) { case NEW_SPACE: return Heap::new_space(); case OLD_POINTER_SPACE: return Heap::old_pointer_space(); case OLD_DATA_SPACE: return Heap::old_data_space(); case CODE_SPACE: return Heap::code_space(); case MAP_SPACE: return Heap::map_space(); case CELL_SPACE: return Heap::cell_space(); case LO_SPACE: return Heap::lo_space(); default: return NULL; } } PagedSpace* PagedSpaces::next() { switch (counter_++) { case OLD_POINTER_SPACE: return Heap::old_pointer_space(); case OLD_DATA_SPACE: return Heap::old_data_space(); case CODE_SPACE: return Heap::code_space(); case MAP_SPACE: return Heap::map_space(); case CELL_SPACE: return Heap::cell_space(); default: return NULL; } } OldSpace* OldSpaces::next() { switch (counter_++) { case OLD_POINTER_SPACE: return Heap::old_pointer_space(); case OLD_DATA_SPACE: return Heap::old_data_space(); case CODE_SPACE: return Heap::code_space(); default: return NULL; } } SpaceIterator::SpaceIterator() : current_space_(FIRST_SPACE), iterator_(NULL), size_func_(NULL) { } SpaceIterator::SpaceIterator(HeapObjectCallback size_func) : current_space_(FIRST_SPACE), iterator_(NULL), size_func_(size_func) { } SpaceIterator::~SpaceIterator() { // Delete active iterator if any. delete iterator_; } bool SpaceIterator::has_next() { // Iterate until no more spaces. return current_space_ != LAST_SPACE; } ObjectIterator* SpaceIterator::next() { if (iterator_ != NULL) { delete iterator_; iterator_ = NULL; // Move to the next space current_space_++; if (current_space_ > LAST_SPACE) { return NULL; } } // Return iterator for the new current space. return CreateIterator(); } // Create an iterator for the space to iterate. ObjectIterator* SpaceIterator::CreateIterator() { ASSERT(iterator_ == NULL); switch (current_space_) { case NEW_SPACE: iterator_ = new SemiSpaceIterator(Heap::new_space(), size_func_); break; case OLD_POINTER_SPACE: iterator_ = new HeapObjectIterator(Heap::old_pointer_space(), size_func_); break; case OLD_DATA_SPACE: iterator_ = new HeapObjectIterator(Heap::old_data_space(), size_func_); break; case CODE_SPACE: iterator_ = new HeapObjectIterator(Heap::code_space(), size_func_); break; case MAP_SPACE: iterator_ = new HeapObjectIterator(Heap::map_space(), size_func_); break; case CELL_SPACE: iterator_ = new HeapObjectIterator(Heap::cell_space(), size_func_); break; case LO_SPACE: iterator_ = new LargeObjectIterator(Heap::lo_space(), size_func_); break; } // Return the newly allocated iterator; ASSERT(iterator_ != NULL); return iterator_; } class HeapObjectsFilter { public: virtual ~HeapObjectsFilter() {} virtual bool SkipObject(HeapObject* object) = 0; }; class FreeListNodesFilter : public HeapObjectsFilter { public: FreeListNodesFilter() { MarkFreeListNodes(); } bool SkipObject(HeapObject* object) { if (object->IsMarked()) { object->ClearMark(); return true; } else { return false; } } private: void MarkFreeListNodes() { Heap::old_pointer_space()->MarkFreeListNodes(); Heap::old_data_space()->MarkFreeListNodes(); MarkCodeSpaceFreeListNodes(); Heap::map_space()->MarkFreeListNodes(); Heap::cell_space()->MarkFreeListNodes(); } void MarkCodeSpaceFreeListNodes() { // For code space, using FreeListNode::IsFreeListNode is OK. HeapObjectIterator iter(Heap::code_space()); for (HeapObject* obj = iter.next_object(); obj != NULL; obj = iter.next_object()) { if (FreeListNode::IsFreeListNode(obj)) obj->SetMark(); } } AssertNoAllocation no_alloc; }; class UnreachableObjectsFilter : public HeapObjectsFilter { public: UnreachableObjectsFilter() { MarkUnreachableObjects(); } bool SkipObject(HeapObject* object) { if (object->IsMarked()) { object->ClearMark(); return true; } else { return false; } } private: class UnmarkingVisitor : public ObjectVisitor { public: UnmarkingVisitor() : list_(10) {} void VisitPointers(Object** start, Object** end) { for (Object** p = start; p < end; p++) { if (!(*p)->IsHeapObject()) continue; HeapObject* obj = HeapObject::cast(*p); if (obj->IsMarked()) { obj->ClearMark(); list_.Add(obj); } } } bool can_process() { return !list_.is_empty(); } void ProcessNext() { HeapObject* obj = list_.RemoveLast(); obj->Iterate(this); } private: List list_; }; void MarkUnreachableObjects() { HeapIterator iterator; for (HeapObject* obj = iterator.next(); obj != NULL; obj = iterator.next()) { obj->SetMark(); } UnmarkingVisitor visitor; Heap::IterateRoots(&visitor, VISIT_ALL); while (visitor.can_process()) visitor.ProcessNext(); } AssertNoAllocation no_alloc; }; HeapIterator::HeapIterator() : filtering_(HeapIterator::kNoFiltering), filter_(NULL) { Init(); } HeapIterator::HeapIterator(HeapIterator::HeapObjectsFiltering filtering) : filtering_(filtering), filter_(NULL) { Init(); } HeapIterator::~HeapIterator() { Shutdown(); } void HeapIterator::Init() { // Start the iteration. space_iterator_ = filtering_ == kNoFiltering ? new SpaceIterator : new SpaceIterator(MarkCompactCollector::SizeOfMarkedObject); switch (filtering_) { case kFilterFreeListNodes: filter_ = new FreeListNodesFilter; break; case kFilterUnreachable: filter_ = new UnreachableObjectsFilter; break; default: break; } object_iterator_ = space_iterator_->next(); } void HeapIterator::Shutdown() { #ifdef DEBUG // Assert that in filtering mode we have iterated through all // objects. Otherwise, heap will be left in an inconsistent state. if (filtering_ != kNoFiltering) { ASSERT(object_iterator_ == NULL); } #endif // Make sure the last iterator is deallocated. delete space_iterator_; space_iterator_ = NULL; object_iterator_ = NULL; delete filter_; filter_ = NULL; } HeapObject* HeapIterator::next() { if (filter_ == NULL) return NextObject(); HeapObject* obj = NextObject(); while (obj != NULL && filter_->SkipObject(obj)) obj = NextObject(); return obj; } HeapObject* HeapIterator::NextObject() { // No iterator means we are done. if (object_iterator_ == NULL) return NULL; if (HeapObject* obj = object_iterator_->next_object()) { // If the current iterator has more objects we are fine. return obj; } else { // Go though the spaces looking for one that has objects. while (space_iterator_->has_next()) { object_iterator_ = space_iterator_->next(); if (HeapObject* obj = object_iterator_->next_object()) { return obj; } } } // Done with the last space. object_iterator_ = NULL; return NULL; } void HeapIterator::reset() { // Restart the iterator. Shutdown(); Init(); } #ifdef DEBUG static bool search_for_any_global; static Object* search_target; static bool found_target; static List object_stack(20); // Tags 0, 1, and 3 are used. Use 2 for marking visited HeapObject. static const int kMarkTag = 2; static void MarkObjectRecursively(Object** p); class MarkObjectVisitor : public ObjectVisitor { public: void VisitPointers(Object** start, Object** end) { // Copy all HeapObject pointers in [start, end) for (Object** p = start; p < end; p++) { if ((*p)->IsHeapObject()) MarkObjectRecursively(p); } } }; static MarkObjectVisitor mark_visitor; static void MarkObjectRecursively(Object** p) { if (!(*p)->IsHeapObject()) return; HeapObject* obj = HeapObject::cast(*p); Object* map = obj->map(); if (!map->IsHeapObject()) return; // visited before if (found_target) return; // stop if target found object_stack.Add(obj); if ((search_for_any_global && obj->IsJSGlobalObject()) || (!search_for_any_global && (obj == search_target))) { found_target = true; return; } // not visited yet Map* map_p = reinterpret_cast(HeapObject::cast(map)); Address map_addr = map_p->address(); obj->set_map(reinterpret_cast(map_addr + kMarkTag)); MarkObjectRecursively(&map); obj->IterateBody(map_p->instance_type(), obj->SizeFromMap(map_p), &mark_visitor); if (!found_target) // don't pop if found the target object_stack.RemoveLast(); } static void UnmarkObjectRecursively(Object** p); class UnmarkObjectVisitor : public ObjectVisitor { public: void VisitPointers(Object** start, Object** end) { // Copy all HeapObject pointers in [start, end) for (Object** p = start; p < end; p++) { if ((*p)->IsHeapObject()) UnmarkObjectRecursively(p); } } }; static UnmarkObjectVisitor unmark_visitor; static void UnmarkObjectRecursively(Object** p) { if (!(*p)->IsHeapObject()) return; HeapObject* obj = HeapObject::cast(*p); Object* map = obj->map(); if (map->IsHeapObject()) return; // unmarked already Address map_addr = reinterpret_cast
(map); map_addr -= kMarkTag; ASSERT_TAG_ALIGNED(map_addr); HeapObject* map_p = HeapObject::FromAddress(map_addr); obj->set_map(reinterpret_cast(map_p)); UnmarkObjectRecursively(reinterpret_cast(&map_p)); obj->IterateBody(Map::cast(map_p)->instance_type(), obj->SizeFromMap(Map::cast(map_p)), &unmark_visitor); } static void MarkRootObjectRecursively(Object** root) { if (search_for_any_global) { ASSERT(search_target == NULL); } else { ASSERT(search_target->IsHeapObject()); } found_target = false; object_stack.Clear(); MarkObjectRecursively(root); UnmarkObjectRecursively(root); if (found_target) { PrintF("=====================================\n"); PrintF("==== Path to object ====\n"); PrintF("=====================================\n\n"); ASSERT(!object_stack.is_empty()); for (int i = 0; i < object_stack.length(); i++) { if (i > 0) PrintF("\n |\n |\n V\n\n"); Object* obj = object_stack[i]; obj->Print(); } PrintF("=====================================\n"); } } // Helper class for visiting HeapObjects recursively. class MarkRootVisitor: public ObjectVisitor { public: void VisitPointers(Object** start, Object** end) { // Visit all HeapObject pointers in [start, end) for (Object** p = start; p < end; p++) { if ((*p)->IsHeapObject()) MarkRootObjectRecursively(p); } } }; // Triggers a depth-first traversal of reachable objects from roots // and finds a path to a specific heap object and prints it. void Heap::TracePathToObject(Object* target) { search_target = target; search_for_any_global = false; MarkRootVisitor root_visitor; IterateRoots(&root_visitor, VISIT_ONLY_STRONG); } // Triggers a depth-first traversal of reachable objects from roots // and finds a path to any global object and prints it. Useful for // determining the source for leaks of global objects. void Heap::TracePathToGlobal() { search_target = NULL; search_for_any_global = true; MarkRootVisitor root_visitor; IterateRoots(&root_visitor, VISIT_ONLY_STRONG); } #endif static intptr_t CountTotalHolesSize() { intptr_t holes_size = 0; OldSpaces spaces; for (OldSpace* space = spaces.next(); space != NULL; space = spaces.next()) { holes_size += space->Waste() + space->AvailableFree(); } return holes_size; } GCTracer::GCTracer() : start_time_(0.0), start_size_(0), gc_count_(0), full_gc_count_(0), is_compacting_(false), marked_count_(0), allocated_since_last_gc_(0), spent_in_mutator_(0), promoted_objects_size_(0) { // These two fields reflect the state of the previous full collection. // Set them before they are changed by the collector. previous_has_compacted_ = MarkCompactCollector::HasCompacted(); previous_marked_count_ = MarkCompactCollector::previous_marked_count(); if (!FLAG_trace_gc && !FLAG_print_cumulative_gc_stat) return; start_time_ = OS::TimeCurrentMillis(); start_size_ = Heap::SizeOfObjects(); for (int i = 0; i < Scope::kNumberOfScopes; i++) { scopes_[i] = 0; } in_free_list_or_wasted_before_gc_ = CountTotalHolesSize(); allocated_since_last_gc_ = Heap::SizeOfObjects() - alive_after_last_gc_; if (last_gc_end_timestamp_ > 0) { spent_in_mutator_ = Max(start_time_ - last_gc_end_timestamp_, 0.0); } } GCTracer::~GCTracer() { // Printf ONE line iff flag is set. if (!FLAG_trace_gc && !FLAG_print_cumulative_gc_stat) return; bool first_gc = (last_gc_end_timestamp_ == 0); alive_after_last_gc_ = Heap::SizeOfObjects(); last_gc_end_timestamp_ = OS::TimeCurrentMillis(); int time = static_cast(last_gc_end_timestamp_ - start_time_); // Update cumulative GC statistics if required. if (FLAG_print_cumulative_gc_stat) { max_gc_pause_ = Max(max_gc_pause_, time); max_alive_after_gc_ = Max(max_alive_after_gc_, alive_after_last_gc_); if (!first_gc) { min_in_mutator_ = Min(min_in_mutator_, static_cast(spent_in_mutator_)); } } if (!FLAG_trace_gc_nvp) { int external_time = static_cast(scopes_[Scope::EXTERNAL]); PrintF("%s %.1f -> %.1f MB, ", CollectorString(), static_cast(start_size_) / MB, SizeOfHeapObjects()); if (external_time > 0) PrintF("%d / ", external_time); PrintF("%d ms.\n", time); } else { PrintF("pause=%d ", time); PrintF("mutator=%d ", static_cast(spent_in_mutator_)); PrintF("gc="); switch (collector_) { case SCAVENGER: PrintF("s"); break; case MARK_COMPACTOR: PrintF(MarkCompactCollector::HasCompacted() ? "mc" : "ms"); break; default: UNREACHABLE(); } PrintF(" "); PrintF("external=%d ", static_cast(scopes_[Scope::EXTERNAL])); PrintF("mark=%d ", static_cast(scopes_[Scope::MC_MARK])); PrintF("sweep=%d ", static_cast(scopes_[Scope::MC_SWEEP])); PrintF("sweepns=%d ", static_cast(scopes_[Scope::MC_SWEEP_NEWSPACE])); PrintF("compact=%d ", static_cast(scopes_[Scope::MC_COMPACT])); PrintF("total_size_before=%" V8_PTR_PREFIX "d ", start_size_); PrintF("total_size_after=%" V8_PTR_PREFIX "d ", Heap::SizeOfObjects()); PrintF("holes_size_before=%" V8_PTR_PREFIX "d ", in_free_list_or_wasted_before_gc_); PrintF("holes_size_after=%" V8_PTR_PREFIX "d ", CountTotalHolesSize()); PrintF("allocated=%" V8_PTR_PREFIX "d ", allocated_since_last_gc_); PrintF("promoted=%" V8_PTR_PREFIX "d ", promoted_objects_size_); PrintF("\n"); } #if defined(ENABLE_LOGGING_AND_PROFILING) Heap::PrintShortHeapStatistics(); #endif } const char* GCTracer::CollectorString() { switch (collector_) { case SCAVENGER: return "Scavenge"; case MARK_COMPACTOR: return MarkCompactCollector::HasCompacted() ? "Mark-compact" : "Mark-sweep"; } return "Unknown GC"; } int KeyedLookupCache::Hash(Map* map, String* name) { // Uses only lower 32 bits if pointers are larger. uintptr_t addr_hash = static_cast(reinterpret_cast(map)) >> kMapHashShift; return static_cast((addr_hash ^ name->Hash()) & kCapacityMask); } int KeyedLookupCache::Lookup(Map* map, String* name) { int index = Hash(map, name); Key& key = keys_[index]; if ((key.map == map) && key.name->Equals(name)) { return field_offsets_[index]; } return -1; } void KeyedLookupCache::Update(Map* map, String* name, int field_offset) { String* symbol; if (Heap::LookupSymbolIfExists(name, &symbol)) { int index = Hash(map, symbol); Key& key = keys_[index]; key.map = map; key.name = symbol; field_offsets_[index] = field_offset; } } void KeyedLookupCache::Clear() { for (int index = 0; index < kLength; index++) keys_[index].map = NULL; } KeyedLookupCache::Key KeyedLookupCache::keys_[KeyedLookupCache::kLength]; int KeyedLookupCache::field_offsets_[KeyedLookupCache::kLength]; void DescriptorLookupCache::Clear() { for (int index = 0; index < kLength; index++) keys_[index].array = NULL; } DescriptorLookupCache::Key DescriptorLookupCache::keys_[DescriptorLookupCache::kLength]; int DescriptorLookupCache::results_[DescriptorLookupCache::kLength]; #ifdef DEBUG void Heap::GarbageCollectionGreedyCheck() { ASSERT(FLAG_gc_greedy); if (Bootstrapper::IsActive()) return; if (disallow_allocation_failure()) return; CollectGarbage(NEW_SPACE); } #endif TranscendentalCache::TranscendentalCache(TranscendentalCache::Type t) : type_(t) { uint32_t in0 = 0xffffffffu; // Bit-pattern for a NaN that isn't uint32_t in1 = 0xffffffffu; // generated by the FPU. for (int i = 0; i < kCacheSize; i++) { elements_[i].in[0] = in0; elements_[i].in[1] = in1; elements_[i].output = NULL; } } TranscendentalCache* TranscendentalCache::caches_[kNumberOfCaches]; void TranscendentalCache::Clear() { for (int i = 0; i < kNumberOfCaches; i++) { if (caches_[i] != NULL) { delete caches_[i]; caches_[i] = NULL; } } } void ExternalStringTable::CleanUp() { int last = 0; for (int i = 0; i < new_space_strings_.length(); ++i) { if (new_space_strings_[i] == Heap::raw_unchecked_null_value()) continue; if (Heap::InNewSpace(new_space_strings_[i])) { new_space_strings_[last++] = new_space_strings_[i]; } else { old_space_strings_.Add(new_space_strings_[i]); } } new_space_strings_.Rewind(last); last = 0; for (int i = 0; i < old_space_strings_.length(); ++i) { if (old_space_strings_[i] == Heap::raw_unchecked_null_value()) continue; ASSERT(!Heap::InNewSpace(old_space_strings_[i])); old_space_strings_[last++] = old_space_strings_[i]; } old_space_strings_.Rewind(last); Verify(); } void ExternalStringTable::TearDown() { new_space_strings_.Free(); old_space_strings_.Free(); } List ExternalStringTable::new_space_strings_; List ExternalStringTable::old_space_strings_; } } // namespace v8::internal