diff options
Diffstat (limited to 'chromium/v8/src/compiler/js-heap-broker.cc')
| -rw-r--r-- | chromium/v8/src/compiler/js-heap-broker.cc | 1321 |
1 files changed, 961 insertions, 360 deletions
diff --git a/chromium/v8/src/compiler/js-heap-broker.cc b/chromium/v8/src/compiler/js-heap-broker.cc index c79c793ae69..7466a80f851 100644 --- a/chromium/v8/src/compiler/js-heap-broker.cc +++ b/chromium/v8/src/compiler/js-heap-broker.cc @@ -16,7 +16,6 @@ #include "src/compiler/bytecode-analysis.h" #include "src/compiler/graph-reducer.h" #include "src/compiler/per-isolate-compiler-cache.h" -#include "src/compiler/vector-slot-pair.h" #include "src/init/bootstrapper.h" #include "src/objects/allocation-site-inl.h" #include "src/objects/api-callbacks.h" @@ -26,6 +25,7 @@ #include "src/objects/js-array-buffer-inl.h" #include "src/objects/js-array-inl.h" #include "src/objects/js-regexp-inl.h" +#include "src/objects/literal-objects-inl.h" #include "src/objects/module-inl.h" #include "src/objects/objects-inl.h" #include "src/objects/template-objects-inl.h" @@ -256,13 +256,14 @@ class JSObjectField { uint64_t number_bits_ = 0; }; -struct FieldIndexHasher { - size_t operator()(FieldIndex field_index) const { - return field_index.index(); - } +class JSReceiverData : public HeapObjectData { + public: + JSReceiverData(JSHeapBroker* broker, ObjectData** storage, + Handle<JSReceiver> object) + : HeapObjectData(broker, storage, object) {} }; -class JSObjectData : public HeapObjectData { +class JSObjectData : public JSReceiverData { public: JSObjectData(JSHeapBroker* broker, ObjectData** storage, Handle<JSObject> object); @@ -277,16 +278,22 @@ class JSObjectData : public HeapObjectData { FixedArrayBaseData* elements() const; void SerializeObjectCreateMap(JSHeapBroker* broker); - MapData* object_create_map() const { // Can be nullptr. - CHECK(serialized_object_create_map_); + + MapData* object_create_map(JSHeapBroker* broker) const { // Can be nullptr. + if (!serialized_object_create_map_) { + DCHECK_NULL(object_create_map_); + TRACE_MISSING(broker, "object_create_map on " << this); + } return object_create_map_; } - ObjectData* GetOwnConstantElement(JSHeapBroker* broker, uint32_t index, - bool serialize); - ObjectData* GetOwnProperty(JSHeapBroker* broker, - Representation representation, - FieldIndex field_index, bool serialize); + ObjectData* GetOwnConstantElement( + JSHeapBroker* broker, uint32_t index, + SerializationPolicy policy = SerializationPolicy::kAssumeSerialized); + ObjectData* GetOwnDataProperty( + JSHeapBroker* broker, Representation representation, + FieldIndex field_index, + SerializationPolicy policy = SerializationPolicy::kAssumeSerialized); // This method is only used to assert our invariants. bool cow_or_empty_elements_tenured() const; @@ -316,7 +323,9 @@ class JSObjectData : public HeapObjectData { // (2) are known not to (possibly they don't exist at all). // In case (2), the second pair component is nullptr. // For simplicity, this may in theory overlap with inobject_fields_. - ZoneUnorderedMap<FieldIndex, ObjectData*, FieldIndexHasher> own_properties_; + // The keys of the map are the property_index() values of the + // respective property FieldIndex'es. + ZoneUnorderedMap<int, ObjectData*> own_properties_; }; void JSObjectData::SerializeObjectCreateMap(JSHeapBroker* broker) { @@ -353,24 +362,25 @@ base::Optional<ObjectRef> GetOwnElementFromHeap(JSHeapBroker* broker, return base::nullopt; } -ObjectRef GetOwnPropertyFromHeap(JSHeapBroker* broker, - Handle<JSObject> receiver, - Representation representation, - FieldIndex field_index) { +ObjectRef GetOwnDataPropertyFromHeap(JSHeapBroker* broker, + Handle<JSObject> receiver, + Representation representation, + FieldIndex field_index) { Handle<Object> constant = JSObject::FastPropertyAt(receiver, representation, field_index); return ObjectRef(broker, constant); } + } // namespace ObjectData* JSObjectData::GetOwnConstantElement(JSHeapBroker* broker, uint32_t index, - bool serialize) { + SerializationPolicy policy) { for (auto const& p : own_constant_elements_) { if (p.first == index) return p.second; } - if (!serialize) { + if (policy == SerializationPolicy::kAssumeSerialized) { TRACE_MISSING(broker, "knowledge about index " << index << " on " << this); return nullptr; } @@ -382,24 +392,24 @@ ObjectData* JSObjectData::GetOwnConstantElement(JSHeapBroker* broker, return result; } -ObjectData* JSObjectData::GetOwnProperty(JSHeapBroker* broker, - Representation representation, - FieldIndex field_index, - bool serialize) { - auto p = own_properties_.find(field_index); +ObjectData* JSObjectData::GetOwnDataProperty(JSHeapBroker* broker, + Representation representation, + FieldIndex field_index, + SerializationPolicy policy) { + auto p = own_properties_.find(field_index.property_index()); if (p != own_properties_.end()) return p->second; - if (!serialize) { + if (policy == SerializationPolicy::kAssumeSerialized) { TRACE_MISSING(broker, "knowledge about property with index " << field_index.property_index() << " on " << this); return nullptr; } - ObjectRef property = GetOwnPropertyFromHeap( + ObjectRef property = GetOwnDataPropertyFromHeap( broker, Handle<JSObject>::cast(object()), representation, field_index); ObjectData* result(property.data()); - own_properties_.insert(std::make_pair(field_index, result)); + own_properties_.insert(std::make_pair(field_index.property_index(), result)); return result; } @@ -446,6 +456,31 @@ void JSTypedArrayData::Serialize(JSHeapBroker* broker) { } } +class ArrayBoilerplateDescriptionData : public HeapObjectData { + public: + ArrayBoilerplateDescriptionData(JSHeapBroker* broker, ObjectData** storage, + Handle<ArrayBoilerplateDescription> object) + : HeapObjectData(broker, storage, object), + constants_elements_length_(object->constant_elements().length()) {} + + int constants_elements_length() const { return constants_elements_length_; } + + private: + int const constants_elements_length_; +}; + +class ObjectBoilerplateDescriptionData : public HeapObjectData { + public: + ObjectBoilerplateDescriptionData(JSHeapBroker* broker, ObjectData** storage, + Handle<ObjectBoilerplateDescription> object) + : HeapObjectData(broker, storage, object), size_(object->size()) {} + + int size() const { return size_; } + + private: + int const size_; +}; + class JSDataViewData : public JSObjectData { public: JSDataViewData(JSHeapBroker* broker, ObjectData** storage, @@ -465,6 +500,7 @@ class JSBoundFunctionData : public JSObjectData { Handle<JSBoundFunction> object); void Serialize(JSHeapBroker* broker); + bool serialized() const { return serialized_; } ObjectData* bound_target_function() const { return bound_target_function_; } ObjectData* bound_this() const { return bound_this_; } @@ -557,18 +593,6 @@ class HeapNumberData : public HeapObjectData { double const value_; }; -class MutableHeapNumberData : public HeapObjectData { - public: - MutableHeapNumberData(JSHeapBroker* broker, ObjectData** storage, - Handle<MutableHeapNumber> object) - : HeapObjectData(broker, storage, object), value_(object->value()) {} - - double value() const { return value_; } - - private: - double const value_; -}; - class ContextData : public HeapObjectData { public: ContextData(JSHeapBroker* broker, ObjectData** storage, @@ -576,12 +600,15 @@ class ContextData : public HeapObjectData { // {previous} will return the closest valid context possible to desired // {depth}, decrementing {depth} for each previous link successfully followed. - // If {serialize} is true, it will serialize contexts along the way. - ContextData* previous(JSHeapBroker* broker, size_t* depth, bool serialize); + ContextData* previous( + JSHeapBroker* broker, size_t* depth, + SerializationPolicy policy = SerializationPolicy::kAssumeSerialized); - // Returns nullptr if the slot index isn't valid or wasn't serialized - // (unless {serialize} is true). - ObjectData* GetSlot(JSHeapBroker* broker, int index, bool serialize); + // Returns nullptr if the slot index isn't valid or wasn't serialized, + // unless {policy} is {kSerializeIfNeeded}. + ObjectData* GetSlot( + JSHeapBroker* broker, int index, + SerializationPolicy policy = SerializationPolicy::kAssumeSerialized); private: ZoneMap<int, ObjectData*> slots_; @@ -593,10 +620,11 @@ ContextData::ContextData(JSHeapBroker* broker, ObjectData** storage, : HeapObjectData(broker, storage, object), slots_(broker->zone()) {} ContextData* ContextData::previous(JSHeapBroker* broker, size_t* depth, - bool serialize) { + SerializationPolicy policy) { if (*depth == 0) return this; - if (serialize && previous_ == nullptr) { + if (policy == SerializationPolicy::kSerializeIfNeeded && + previous_ == nullptr) { TraceScope tracer(broker, this, "ContextData::previous"); Handle<Context> context = Handle<Context>::cast(object()); Object prev = context->unchecked_previous(); @@ -607,20 +635,20 @@ ContextData* ContextData::previous(JSHeapBroker* broker, size_t* depth, if (previous_ != nullptr) { *depth = *depth - 1; - return previous_->previous(broker, depth, serialize); + return previous_->previous(broker, depth, policy); } return this; } ObjectData* ContextData::GetSlot(JSHeapBroker* broker, int index, - bool serialize) { + SerializationPolicy policy) { CHECK_GE(index, 0); auto search = slots_.find(index); if (search != slots_.end()) { return search->second; } - if (serialize) { + if (policy == SerializationPolicy::kSerializeIfNeeded) { Handle<Context> context = Handle<Context>::cast(object()); if (index < context->length()) { TraceScope tracer(broker, this, "ContextData::GetSlot"); @@ -680,8 +708,9 @@ class StringData : public NameData { bool is_external_string() const { return is_external_string_; } bool is_seq_string() const { return is_seq_string_; } - StringData* GetCharAsString(JSHeapBroker* broker, uint32_t index, - bool serialize); + StringData* GetCharAsString( + JSHeapBroker* broker, uint32_t index, + SerializationPolicy policy = SerializationPolicy::kAssumeSerialized); private: int const length_; @@ -730,14 +759,14 @@ class InternalizedStringData : public StringData { }; StringData* StringData::GetCharAsString(JSHeapBroker* broker, uint32_t index, - bool serialize) { + SerializationPolicy policy) { if (index >= static_cast<uint32_t>(length())) return nullptr; for (auto const& p : chars_as_strings_) { if (p.first == index) return p.second; } - if (!serialize) { + if (policy == SerializationPolicy::kAssumeSerialized) { TRACE_MISSING(broker, "knowledge about index " << index << " on " << this); return nullptr; } @@ -842,6 +871,12 @@ bool IsInlinableFastLiteral(Handle<JSObject> boilerplate) { } // namespace +class AccessorInfoData : public HeapObjectData { + public: + AccessorInfoData(JSHeapBroker* broker, ObjectData** storage, + Handle<AccessorInfo> object); +}; + class AllocationSiteData : public HeapObjectData { public: AllocationSiteData(JSHeapBroker* broker, ObjectData** storage, @@ -891,6 +926,7 @@ class ScriptContextTableData : public HeapObjectData { struct PropertyDescriptor { NameData* key = nullptr; + ObjectData* value = nullptr; PropertyDetails details = PropertyDetails::Empty(); FieldIndex field_index; MapData* field_owner = nullptr; @@ -926,8 +962,11 @@ class MapData : public HeapObjectData { bool supports_fast_array_resize() const { return supports_fast_array_resize_; } - bool IsMapOfCurrentGlobalProxy() const { - return is_map_of_current_global_proxy_; + bool IsMapOfTargetGlobalProxy() const { + return is_map_of_target_global_proxy_; + } + bool is_abandoned_prototype_map() const { + return is_abandoned_prototype_map_; } // Extra information. @@ -942,10 +981,14 @@ class MapData : public HeapObjectData { // on field owner(s). void SerializeOwnDescriptor(JSHeapBroker* broker, int descriptor_index); void SerializeOwnDescriptors(JSHeapBroker* broker); + ObjectData* GetStrongValue(int descriptor_index) const; DescriptorArrayData* instance_descriptors() const { return instance_descriptors_; } + void SerializeRootMap(JSHeapBroker* broker); + MapData* FindRootMap() const; + void SerializeConstructor(JSHeapBroker* broker); ObjectData* GetConstructor() const { CHECK(serialized_constructor_); @@ -984,7 +1027,8 @@ class MapData : public HeapObjectData { int const unused_property_fields_; bool const supports_fast_array_iteration_; bool const supports_fast_array_resize_; - bool const is_map_of_current_global_proxy_; + bool const is_map_of_target_global_proxy_; + bool const is_abandoned_prototype_map_; bool serialized_elements_kind_generalizations_ = false; ZoneVector<MapData*> elements_kind_generalizations_; @@ -1001,11 +1045,18 @@ class MapData : public HeapObjectData { bool serialized_prototype_ = false; ObjectData* prototype_ = nullptr; + bool serialized_root_map_ = false; + MapData* root_map_ = nullptr; + bool serialized_for_element_load_ = false; bool serialized_for_element_store_ = false; }; +AccessorInfoData::AccessorInfoData(JSHeapBroker* broker, ObjectData** storage, + Handle<AccessorInfo> object) + : HeapObjectData(broker, storage, object) {} + AllocationSiteData::AllocationSiteData(JSHeapBroker* broker, ObjectData** storage, Handle<AllocationSite> object) @@ -1103,8 +1154,9 @@ MapData::MapData(JSHeapBroker* broker, ObjectData** storage, Handle<Map> object) SupportsFastArrayIteration(broker->isolate(), object)), supports_fast_array_resize_( SupportsFastArrayResize(broker->isolate(), object)), - is_map_of_current_global_proxy_( - object->IsMapOfGlobalProxy(broker->isolate()->native_context())), + is_map_of_target_global_proxy_( + object->IsMapOfGlobalProxy(broker->target_native_context().object())), + is_abandoned_prototype_map_(object->is_abandoned_prototype_map()), elements_kind_generalizations_(broker->zone()) {} JSFunctionData::JSFunctionData(JSHeapBroker* broker, ObjectData** storage, @@ -1210,28 +1262,52 @@ FeedbackCellData::FeedbackCellData(JSHeapBroker* broker, ObjectData** storage, class FeedbackVectorData : public HeapObjectData { public: - const ZoneVector<ObjectData*>& feedback() { return feedback_; } - FeedbackVectorData(JSHeapBroker* broker, ObjectData** storage, Handle<FeedbackVector> object); - void SerializeSlots(JSHeapBroker* broker); + double invocation_count() const { return invocation_count_; } + + void Serialize(JSHeapBroker* broker); + const ZoneVector<ObjectData*>& feedback() { return feedback_; } + FeedbackCellData* GetClosureFeedbackCell(JSHeapBroker* broker, + int index) const; private: + double const invocation_count_; + bool serialized_ = false; ZoneVector<ObjectData*> feedback_; + ZoneVector<ObjectData*> closure_feedback_cell_array_; }; FeedbackVectorData::FeedbackVectorData(JSHeapBroker* broker, ObjectData** storage, Handle<FeedbackVector> object) - : HeapObjectData(broker, storage, object), feedback_(broker->zone()) {} + : HeapObjectData(broker, storage, object), + invocation_count_(object->invocation_count()), + feedback_(broker->zone()), + closure_feedback_cell_array_(broker->zone()) {} + +FeedbackCellData* FeedbackVectorData::GetClosureFeedbackCell( + JSHeapBroker* broker, int index) const { + CHECK_GE(index, 0); + + size_t cell_array_size = closure_feedback_cell_array_.size(); + if (!serialized_) { + DCHECK_EQ(cell_array_size, 0); + TRACE_BROKER_MISSING(broker, + " closure feedback cell array for vector " << this); + return nullptr; + } + CHECK_LT(index, cell_array_size); + return closure_feedback_cell_array_[index]->AsFeedbackCell(); +} -void FeedbackVectorData::SerializeSlots(JSHeapBroker* broker) { +void FeedbackVectorData::Serialize(JSHeapBroker* broker) { if (serialized_) return; serialized_ = true; - TraceScope tracer(broker, this, "FeedbackVectorData::SerializeSlots"); + TraceScope tracer(broker, this, "FeedbackVectorData::Serialize"); Handle<FeedbackVector> vector = Handle<FeedbackVector>::cast(object()); DCHECK(feedback_.empty()); feedback_.reserve(vector->length()); @@ -1252,6 +1328,16 @@ void FeedbackVectorData::SerializeSlots(JSHeapBroker* broker) { } DCHECK_EQ(vector->length(), feedback_.size()); TRACE(broker, "Copied " << feedback_.size() << " slots"); + + DCHECK(closure_feedback_cell_array_.empty()); + int length = vector->closure_feedback_cell_array().length(); + closure_feedback_cell_array_.reserve(length); + for (int i = 0; i < length; ++i) { + Handle<FeedbackCell> cell = vector->GetClosureFeedbackCell(i); + ObjectData* cell_data = broker->GetOrCreateData(cell); + closure_feedback_cell_array_.push_back(cell_data); + } + TRACE(broker, "Copied " << length << " feedback cells"); } class FixedArrayBaseData : public HeapObjectData { @@ -1300,21 +1386,26 @@ void JSBoundFunctionData::Serialize(JSHeapBroker* broker) { Handle<JSBoundFunction> function = Handle<JSBoundFunction>::cast(object()); DCHECK_NULL(bound_target_function_); - DCHECK_NULL(bound_this_); - DCHECK_NULL(bound_arguments_); - bound_target_function_ = broker->GetOrCreateData(function->bound_target_function()); - bound_this_ = broker->GetOrCreateData(function->bound_this()); + if (bound_target_function_->IsJSBoundFunction()) { + bound_target_function_->AsJSBoundFunction()->Serialize(broker); + } else if (bound_target_function_->IsJSFunction()) { + bound_target_function_->AsJSFunction()->Serialize(broker); + } + + DCHECK_NULL(bound_arguments_); bound_arguments_ = broker->GetOrCreateData(function->bound_arguments())->AsFixedArray(); - bound_arguments_->SerializeContents(broker); + + DCHECK_NULL(bound_this_); + bound_this_ = broker->GetOrCreateData(function->bound_this()); } JSObjectData::JSObjectData(JSHeapBroker* broker, ObjectData** storage, Handle<JSObject> object) - : HeapObjectData(broker, storage, object), + : JSReceiverData(broker, storage, object), inobject_fields_(broker->zone()), own_constant_elements_(broker->zone()), own_properties_(broker->zone()) {} @@ -1494,8 +1585,9 @@ class JSArrayData : public JSObjectData { void Serialize(JSHeapBroker* broker); ObjectData* length() const { return length_; } - ObjectData* GetOwnElement(JSHeapBroker* broker, uint32_t index, - bool serialize); + ObjectData* GetOwnElement( + JSHeapBroker* broker, uint32_t index, + SerializationPolicy policy = SerializationPolicy::kAssumeSerialized); private: bool serialized_ = false; @@ -1524,12 +1616,12 @@ void JSArrayData::Serialize(JSHeapBroker* broker) { } ObjectData* JSArrayData::GetOwnElement(JSHeapBroker* broker, uint32_t index, - bool serialize) { + SerializationPolicy policy) { for (auto const& p : own_elements_) { if (p.first == index) return p.second; } - if (!serialize) { + if (policy == SerializationPolicy::kAssumeSerialized) { TRACE_MISSING(broker, "knowledge about index " << index << " on " << this); return nullptr; } @@ -1654,7 +1746,7 @@ class SourceTextModuleData : public HeapObjectData { Handle<SourceTextModule> object); void Serialize(JSHeapBroker* broker); - CellData* GetCell(int cell_index) const; + CellData* GetCell(JSHeapBroker* broker, int cell_index) const; private: bool serialized_ = false; @@ -1669,8 +1761,14 @@ SourceTextModuleData::SourceTextModuleData(JSHeapBroker* broker, imports_(broker->zone()), exports_(broker->zone()) {} -CellData* SourceTextModuleData::GetCell(int cell_index) const { - CHECK(serialized_); +CellData* SourceTextModuleData::GetCell(JSHeapBroker* broker, + int cell_index) const { + if (!serialized_) { + DCHECK(imports_.empty()); + TRACE_BROKER_MISSING(broker, + "module cell " << cell_index << " on " << this); + return nullptr; + } CellData* cell; switch (SourceTextModuleDescriptor::GetCellIndexKind(cell_index)) { case SourceTextModuleDescriptor::kImport: @@ -1741,13 +1839,25 @@ void CellData::Serialize(JSHeapBroker* broker) { value_ = broker->GetOrCreateData(cell->value()); } +class JSGlobalObjectData : public JSObjectData { + public: + JSGlobalObjectData(JSHeapBroker* broker, ObjectData** storage, + Handle<JSGlobalObject> object); +}; + +JSGlobalObjectData::JSGlobalObjectData(JSHeapBroker* broker, + ObjectData** storage, + Handle<JSGlobalObject> object) + : JSObjectData(broker, storage, object) {} + class JSGlobalProxyData : public JSObjectData { public: JSGlobalProxyData(JSHeapBroker* broker, ObjectData** storage, Handle<JSGlobalProxy> object); - PropertyCellData* GetPropertyCell(JSHeapBroker* broker, NameData* name, - bool serialize); + PropertyCellData* GetPropertyCell( + JSHeapBroker* broker, NameData* name, + SerializationPolicy policy = SerializationPolicy::kAssumeSerialized); private: // Properties that either @@ -1764,10 +1874,11 @@ JSGlobalProxyData::JSGlobalProxyData(JSHeapBroker* broker, ObjectData** storage, namespace { base::Optional<PropertyCellRef> GetPropertyCellFromHeap(JSHeapBroker* broker, Handle<Name> name) { - LookupIterator it(broker->isolate(), - handle(broker->native_context().object()->global_object(), - broker->isolate()), - name, LookupIterator::OWN); + LookupIterator it( + broker->isolate(), + handle(broker->target_native_context().object()->global_object(), + broker->isolate()), + name, LookupIterator::OWN); it.TryLookupCachedProperty(); if (it.state() == LookupIterator::DATA && it.GetHolder<JSObject>()->IsJSGlobalObject()) { @@ -1777,15 +1888,14 @@ base::Optional<PropertyCellRef> GetPropertyCellFromHeap(JSHeapBroker* broker, } } // namespace -PropertyCellData* JSGlobalProxyData::GetPropertyCell(JSHeapBroker* broker, - NameData* name, - bool serialize) { +PropertyCellData* JSGlobalProxyData::GetPropertyCell( + JSHeapBroker* broker, NameData* name, SerializationPolicy policy) { CHECK_NOT_NULL(name); for (auto const& p : properties_) { if (p.first == name) return p.second; } - if (!serialize) { + if (policy == SerializationPolicy::kAssumeSerialized) { TRACE_MISSING(broker, "knowledge about global property " << name); return nullptr; } @@ -1896,6 +2006,13 @@ void MapData::SerializeOwnDescriptors(JSHeapBroker* broker) { } } +ObjectData* MapData::GetStrongValue(int descriptor_index) const { + auto data = instance_descriptors_->contents().find(descriptor_index); + if (data == instance_descriptors_->contents().end()) return nullptr; + + return data->second.value; +} + void MapData::SerializeOwnDescriptor(JSHeapBroker* broker, int descriptor_index) { TraceScope tracer(broker, this, "MapData::SerializeOwnDescriptor"); @@ -1907,7 +2024,7 @@ void MapData::SerializeOwnDescriptor(JSHeapBroker* broker, } ZoneMap<int, PropertyDescriptor>& contents = - instance_descriptors_->contents(); + instance_descriptors()->contents(); CHECK_LT(descriptor_index, map->NumberOfOwnDescriptors()); if (contents.find(descriptor_index) != contents.end()) return; @@ -1919,6 +2036,11 @@ void MapData::SerializeOwnDescriptor(JSHeapBroker* broker, PropertyDescriptor d; d.key = broker->GetOrCreateData(descriptors->GetKey(descriptor_index))->AsName(); + MaybeObject value = descriptors->GetValue(descriptor_index); + HeapObject obj; + if (value.GetHeapObjectIfStrong(&obj)) { + d.value = broker->GetOrCreateData(handle(obj, broker->isolate())); + } d.details = descriptors->GetDetails(descriptor_index); if (d.details.location() == kField) { d.field_index = FieldIndex::ForDescriptor(*map, descriptor_index); @@ -1941,6 +2063,19 @@ void MapData::SerializeOwnDescriptor(JSHeapBroker* broker, << contents.size() << " total)"); } +void MapData::SerializeRootMap(JSHeapBroker* broker) { + if (serialized_root_map_) return; + serialized_root_map_ = true; + + TraceScope tracer(broker, this, "MapData::SerializeRootMap"); + Handle<Map> map = Handle<Map>::cast(object()); + DCHECK_NULL(root_map_); + root_map_ = + broker->GetOrCreateData(map->FindRootMap(broker->isolate()))->AsMap(); +} + +MapData* MapData::FindRootMap() const { return root_map_; } + void JSObjectData::SerializeRecursiveAsBoilerplate(JSHeapBroker* broker, int depth) { if (serialized_as_boilerplate_) return; @@ -2029,15 +2164,16 @@ void JSObjectData::SerializeRecursiveAsBoilerplate(JSHeapBroker* broker, } else { Handle<Object> value(boilerplate->RawFastPropertyAt(field_index), isolate); - // In case of unboxed double fields we use a sentinel NaN value to mark + // In case of double fields we use a sentinel NaN value to mark // uninitialized fields. A boilerplate value with such a field may migrate - // from its unboxed double to a tagged representation. In the process the - // raw double is converted to a heap number. The sentinel value carries no - // special meaning when it occurs in a heap number, so we would like to - // recover the uninitialized value. - // We check for the sentinel here, specifically, since migrations might - // have been triggered as part of boilerplate serialization. - if (value->IsHeapNumber() && + // from its double to a tagged representation. If the double is unboxed, + // the raw double is converted to a heap number, otherwise the (boxed) + // double ceases to be mutable, and becomes a normal heap number. The + // sentinel value carries no special meaning when it occurs in a heap + // number, so we would like to recover the uninitialized value. We check + // for the sentinel here, specifically, since migrations might have been + // triggered as part of boilerplate serialization. + if (!details.representation().IsDouble() && value->IsHeapNumber() && HeapNumber::cast(*value).value_as_bits() == kHoleNanInt64) { value = isolate->factory()->uninitialized_value(); } @@ -2079,7 +2215,8 @@ bool ObjectRef::equals(const ObjectRef& other) const { Isolate* ObjectRef::isolate() const { return broker()->isolate(); } -ContextRef ContextRef::previous(size_t* depth, bool serialize) const { +ContextRef ContextRef::previous(size_t* depth, + SerializationPolicy policy) const { DCHECK_NOT_NULL(depth); if (broker()->mode() == JSHeapBroker::kDisabled) { AllowHandleAllocation handle_allocation; @@ -2092,10 +2229,11 @@ ContextRef ContextRef::previous(size_t* depth, bool serialize) const { return ContextRef(broker(), handle(current, broker()->isolate())); } ContextData* current = this->data()->AsContext(); - return ContextRef(broker(), current->previous(broker(), depth, serialize)); + return ContextRef(broker(), current->previous(broker(), depth, policy)); } -base::Optional<ObjectRef> ContextRef::get(int index, bool serialize) const { +base::Optional<ObjectRef> ContextRef::get(int index, + SerializationPolicy policy) const { if (broker()->mode() == JSHeapBroker::kDisabled) { AllowHandleAllocation handle_allocation; AllowHandleDereference handle_dereference; @@ -2103,7 +2241,7 @@ base::Optional<ObjectRef> ContextRef::get(int index, bool serialize) const { return ObjectRef(broker(), value); } ObjectData* optional_slot = - data()->AsContext()->GetSlot(broker(), index, serialize); + data()->AsContext()->GetSlot(broker(), index, policy); if (optional_slot != nullptr) { return ObjectRef(broker(), optional_slot); } @@ -2121,13 +2259,13 @@ JSHeapBroker::JSHeapBroker(Isolate* isolate, Zone* broker_zone, tracing_enabled_(tracing_enabled), feedback_(zone()), bytecode_analyses_(zone()), - ais_for_loading_then_(zone()), - ais_for_loading_exec_(zone()) { - // Note that this initialization of the refs_ pointer with the minimal - // initial capacity is redundant in the normal use case (concurrent - // compilation enabled, standard objects to be serialized), as the map - // is going to be replaced immediatelly with a larger capacity one. - // It doesn't seem to affect the performance in a noticeable way though. + property_access_infos_(zone()), + typed_array_string_tags_(zone()) { + // Note that this initialization of {refs_} with the minimal initial capacity + // is redundant in the normal use case (concurrent compilation enabled, + // standard objects to be serialized), as the map is going to be replaced + // immediately with a larger-capacity one. It doesn't seem to affect the + // performance in a noticeable way though. TRACE(this, "Constructing heap broker"); } @@ -2136,13 +2274,6 @@ std::ostream& JSHeapBroker::Trace() { << std::string(trace_indentation_ * 2, ' '); } -void JSHeapBroker::StartSerializing() { - CHECK_EQ(mode_, kDisabled); - TRACE(this, "Starting serialization"); - mode_ = kSerializing; - refs_->Clear(); -} - void JSHeapBroker::StopSerializing() { CHECK_EQ(mode_, kSerializing); TRACE(this, "Stopping serialization"); @@ -2157,39 +2288,54 @@ void JSHeapBroker::Retire() { bool JSHeapBroker::SerializingAllowed() const { return mode() == kSerializing; } -void JSHeapBroker::SetNativeContextRef() { - native_context_ = NativeContextRef(this, isolate()->native_context()); +void JSHeapBroker::SetTargetNativeContextRef( + Handle<NativeContext> native_context) { + // The MapData constructor uses {target_native_context_}. This creates a + // benign cycle that we break by setting {target_native_context_} right before + // starting to serialize (thus creating dummy data), and then again properly + // right after. + DCHECK((mode() == kDisabled && !target_native_context_.has_value()) || + (mode() == kSerializing && + target_native_context_->object().equals(native_context) && + target_native_context_->data_->kind() == kUnserializedHeapObject)); + target_native_context_ = NativeContextRef(this, native_context); } bool IsShareable(Handle<Object> object, Isolate* isolate) { - Builtins* const b = isolate->builtins(); - int index; RootIndex root_index; - return (object->IsHeapObject() && - b->IsBuiltinHandle(Handle<HeapObject>::cast(object), &index)) || + bool is_builtin_handle = + object->IsHeapObject() && isolate->builtins()->IsBuiltinHandle( + Handle<HeapObject>::cast(object), &index); + return is_builtin_handle || isolate->roots_table().IsRootHandle(object, &root_index); } -void JSHeapBroker::SerializeShareableObjects() { +void JSHeapBroker::InitializeRefsMap() { + TraceScope tracer(this, "JSHeapBroker::InitializeRefsMap"); + + DCHECK_NULL(compiler_cache_); PerIsolateCompilerCache::Setup(isolate()); compiler_cache_ = isolate()->compiler_cache(); if (compiler_cache_->HasSnapshot()) { - RefsMap* snapshot = compiler_cache_->GetSnapshot(); - - refs_ = new (zone()) RefsMap(snapshot, zone()); + TRACE(this, "Importing existing RefsMap snapshot"); + DCHECK_NULL(refs_); + refs_ = new (zone()) RefsMap(compiler_cache_->GetSnapshot(), zone()); return; } - TraceScope tracer( - this, "JSHeapBroker::SerializeShareableObjects (building snapshot)"); - + TRACE(this, "Building RefsMap snapshot"); + DCHECK_NULL(refs_); refs_ = new (zone()) RefsMap(kInitialRefsBucketCount, AddressMatcher(), zone()); + // Temporarily use the "compiler zone" for serialization, such that the + // serialized data survives this compilation. + DCHECK_EQ(current_zone_, broker_zone_); current_zone_ = compiler_cache_->zone(); + // Serialize various builtins. Builtins* const b = isolate()->builtins(); { Builtins::Name builtins[] = { @@ -2199,17 +2345,28 @@ void JSHeapBroker::SerializeShareableObjects() { Builtins::kAllocateRegularInOldGeneration, Builtins::kArgumentsAdaptorTrampoline, Builtins::kArrayConstructorImpl, + Builtins::kArrayIncludesHoleyDoubles, + Builtins::kArrayIncludesPackedDoubles, + Builtins::kArrayIncludesSmiOrObject, + Builtins::kArrayIndexOfHoleyDoubles, + Builtins::kArrayIndexOfPackedDoubles, + Builtins::kArrayIndexOfSmiOrObject, + Builtins::kCallApiCallback, Builtins::kCallFunctionForwardVarargs, Builtins::kCallFunction_ReceiverIsAny, Builtins::kCallFunction_ReceiverIsNotNullOrUndefined, Builtins::kCallFunction_ReceiverIsNullOrUndefined, + Builtins::kCloneFastJSArray, + Builtins::kCompileLazy, Builtins::kConstructFunctionForwardVarargs, Builtins::kForInFilter, + Builtins::kGetProperty, + Builtins::kIncBlockCounter, Builtins::kJSBuiltinsConstructStub, Builtins::kJSConstructStubGeneric, Builtins::kStringAdd_CheckNone, - Builtins::kStringAdd_ConvertLeft, - Builtins::kStringAdd_ConvertRight, + Builtins::kStringAddConvertLeft, + Builtins::kStringAddConvertRight, Builtins::kToNumber, Builtins::kToObject, }; @@ -2223,12 +2380,13 @@ void JSHeapBroker::SerializeShareableObjects() { } } + // TODO(mslekova): Serialize root objects (from factory). + + // Verify. for (RefsMap::Entry* p = refs_->Start(); p != nullptr; p = refs_->Next(p)) { CHECK(IsShareable(p->value->object(), isolate())); } - // TODO(mslekova): - // Serialize root objects (from factory). compiler_cache()->SetSnapshot(refs_); current_zone_ = broker_zone_; } @@ -2252,6 +2410,25 @@ void JSHeapBroker::CollectArrayAndObjectPrototypes() { CHECK(!array_and_object_prototypes_.empty()); } +void JSHeapBroker::SerializeTypedArrayStringTags() { +#define TYPED_ARRAY_STRING_TAG(Type, type, TYPE, ctype) \ + do { \ + ObjectData* data = GetOrCreateData( \ + isolate()->factory()->InternalizeUtf8String(#Type "Array")); \ + typed_array_string_tags_.push_back(data); \ + } while (false); + + TYPED_ARRAYS(TYPED_ARRAY_STRING_TAG) +#undef TYPED_ARRAY_STRING_TAG +} + +StringRef JSHeapBroker::GetTypedArrayStringTag(ElementsKind kind) { + DCHECK(IsTypedArrayElementsKind(kind)); + size_t idx = kind - FIRST_FIXED_TYPED_ARRAY_ELEMENTS_KIND; + CHECK_LT(idx, typed_array_string_tags_.size()); + return StringRef(this, typed_array_string_tags_[idx]); +} + bool JSHeapBroker::IsArrayOrObjectPrototype(const JSObjectRef& object) const { if (mode() == kDisabled) { return isolate()->IsInAnyContext(*object.object(), @@ -2264,22 +2441,29 @@ bool JSHeapBroker::IsArrayOrObjectPrototype(const JSObjectRef& object) const { array_and_object_prototypes_.end(); } -void JSHeapBroker::SerializeStandardObjects() { - if (mode() == kDisabled) return; - CHECK_EQ(mode(), kSerializing); +void JSHeapBroker::InitializeAndStartSerializing( + Handle<NativeContext> native_context) { + TraceScope tracer(this, "JSHeapBroker::InitializeAndStartSerializing"); - SerializeShareableObjects(); + CHECK_EQ(mode_, kDisabled); + mode_ = kSerializing; - TraceScope tracer(this, "JSHeapBroker::SerializeStandardObjects"); + // Throw away the dummy data that we created while disabled. + refs_->Clear(); + refs_ = nullptr; - CollectArrayAndObjectPrototypes(); + InitializeRefsMap(); - SetNativeContextRef(); - native_context().Serialize(); + SetTargetNativeContextRef(native_context); + target_native_context().Serialize(); - Factory* const f = isolate()->factory(); + CollectArrayAndObjectPrototypes(); + SerializeTypedArrayStringTags(); - // Maps, strings, oddballs + // Serialize standard objects. + // + // - Maps, strings, oddballs + Factory* const f = isolate()->factory(); GetOrCreateData(f->arguments_marker_map()); GetOrCreateData(f->bigint_string()); GetOrCreateData(f->block_context_map()); @@ -2300,7 +2484,6 @@ void JSHeapBroker::SerializeStandardObjects() { GetOrCreateData(f->length_string()); GetOrCreateData(f->many_closures_cell_map()); GetOrCreateData(f->minus_zero_value()); - GetOrCreateData(f->mutable_heap_number_map()); GetOrCreateData(f->name_dictionary_map()); GetOrCreateData(f->NaN_string()); GetOrCreateData(f->null_map()); @@ -2312,6 +2495,8 @@ void JSHeapBroker::SerializeStandardObjects() { GetOrCreateData(f->optimized_out()); GetOrCreateData(f->optimized_out_map()); GetOrCreateData(f->property_array_map()); + GetOrCreateData(f->ReflectHas_string()); + GetOrCreateData(f->ReflectGet_string()); GetOrCreateData(f->sloppy_arguments_elements_map()); GetOrCreateData(f->stale_register()); GetOrCreateData(f->stale_register_map()); @@ -2328,8 +2513,7 @@ void JSHeapBroker::SerializeStandardObjects() { GetOrCreateData(f->uninitialized_map()); GetOrCreateData(f->with_context_map()); GetOrCreateData(f->zero_string()); - - // Protector cells + // - Cells GetOrCreateData(f->array_buffer_detaching_protector()) ->AsPropertyCell() ->Serialize(this); @@ -2340,6 +2524,7 @@ void JSHeapBroker::SerializeStandardObjects() { GetOrCreateData(f->array_species_protector()) ->AsPropertyCell() ->Serialize(this); + GetOrCreateData(f->many_closures_cell())->AsFeedbackCell(); GetOrCreateData(f->no_elements_protector()) ->AsPropertyCell() ->Serialize(this); @@ -2353,8 +2538,7 @@ void JSHeapBroker::SerializeStandardObjects() { ->AsPropertyCell() ->Serialize(this); GetOrCreateData(f->string_length_protector())->AsCell()->Serialize(this); - - // CEntry stub + // - CEntry stub GetOrCreateData( CodeFactory::CEntry(isolate(), 1, kDontSaveFPRegs, kArgvOnStack, true)); @@ -2425,7 +2609,7 @@ base::Optional<MapRef> JSObjectRef::GetObjectCreateMap() const { return base::Optional<MapRef>(); } } - MapData* map_data = data()->AsJSObject()->object_create_map(); + MapData* map_data = data()->AsJSObject()->object_create_map(broker()); return map_data != nullptr ? MapRef(broker(), map_data) : base::Optional<MapRef>(); } @@ -2535,13 +2719,14 @@ bool MapRef::supports_fast_array_resize() const { return data()->AsMap()->supports_fast_array_resize(); } -bool MapRef::IsMapOfCurrentGlobalProxy() const { +bool MapRef::IsMapOfTargetGlobalProxy() const { if (broker()->mode() == JSHeapBroker::kDisabled) { AllowHandleDereference allow_handle_dereference; AllowHandleAllocation handle_allocation; - return object()->IsMapOfGlobalProxy(broker()->isolate()->native_context()); + return object()->IsMapOfGlobalProxy( + broker()->target_native_context().object()); } - return data()->AsMap()->IsMapOfCurrentGlobalProxy(); + return data()->AsMap()->IsMapOfTargetGlobalProxy(); } int JSFunctionRef::InitialMapInstanceSizeWithMinSlack() const { @@ -2612,6 +2797,18 @@ ObjectRef FeedbackVectorRef::get(FeedbackSlot slot) const { return ObjectRef(broker(), data()->AsFeedbackVector()->feedback().at(i)); } +FeedbackCellRef FeedbackVectorRef::GetClosureFeedbackCell(int index) const { + if (broker()->mode() == JSHeapBroker::kDisabled) { + AllowHandleAllocation handle_allocation; + AllowHandleDereference handle_dereference; + return FeedbackCellRef(broker(), object()->GetClosureFeedbackCell(index)); + } + + return FeedbackCellRef( + broker(), + data()->AsFeedbackVector()->GetClosureFeedbackCell(broker(), index)); +} + double JSObjectRef::RawFastDoublePropertyAt(FieldIndex index) const { if (broker()->mode() == JSHeapBroker::kDisabled) { AllowHandleDereference handle_dereference; @@ -2789,6 +2986,22 @@ base::Optional<double> StringRef::ToNumber() { return data()->AsString()->to_number(); } +int ArrayBoilerplateDescriptionRef::constants_elements_length() const { + if (broker()->mode() == JSHeapBroker::kDisabled) { + AllowHandleDereference allow_handle_dereference; + return object()->constant_elements().length(); + } + return data()->AsArrayBoilerplateDescription()->constants_elements_length(); +} + +int ObjectBoilerplateDescriptionRef::size() const { + if (broker()->mode() == JSHeapBroker::kDisabled) { + AllowHandleDereference allow_handle_dereference; + return object()->size(); + } + return data()->AsObjectBoilerplateDescription()->size(); +} + ObjectRef FixedArrayRef::get(int i) const { if (broker()->mode() == JSHeapBroker::kDisabled) { AllowHandleAllocation handle_allocation; @@ -2954,11 +3167,13 @@ BIMODAL_ACCESSOR_C(BytecodeArray, interpreter::Register, BIMODAL_ACCESSOR(Cell, Object, value) +BIMODAL_ACCESSOR_C(FeedbackVector, double, invocation_count) + BIMODAL_ACCESSOR(HeapObject, Map, map) BIMODAL_ACCESSOR(JSArray, Object, length) -BIMODAL_ACCESSOR(JSBoundFunction, Object, bound_target_function) +BIMODAL_ACCESSOR(JSBoundFunction, JSReceiver, bound_target_function) BIMODAL_ACCESSOR(JSBoundFunction, Object, bound_this) BIMODAL_ACCESSOR(JSBoundFunction, FixedArray, bound_arguments) @@ -3003,6 +3218,7 @@ BIMODAL_ACCESSOR(Map, HeapObject, prototype) BIMODAL_ACCESSOR_C(Map, InstanceType, instance_type) BIMODAL_ACCESSOR(Map, Object, GetConstructor) BIMODAL_ACCESSOR(Map, HeapObject, GetBackPointer) +BIMODAL_ACCESSOR_C(Map, bool, is_abandoned_prototype_map) #define DEF_NATIVE_CONTEXT_ACCESSOR(type, name) \ BIMODAL_ACCESSOR(NativeContext, type, name) @@ -3047,7 +3263,7 @@ bool FunctionTemplateInfoRef::has_call_code() const { BIMODAL_ACCESSOR_C(FunctionTemplateInfo, bool, accept_any_receiver) HolderLookupResult FunctionTemplateInfoRef::LookupHolderOfExpectedType( - MapRef receiver_map, bool serialize) { + MapRef receiver_map, SerializationPolicy policy) { const HolderLookupResult not_found; if (broker()->mode() == JSHeapBroker::kDisabled) { @@ -3083,7 +3299,7 @@ HolderLookupResult FunctionTemplateInfoRef::LookupHolderOfExpectedType( if (lookup_it != fti_data->known_receivers().cend()) { return lookup_it->second; } - if (!serialize) { + if (policy == SerializationPolicy::kAssumeSerialized) { TRACE_BROKER_MISSING(broker(), "holder for receiver with map " << receiver_map); return not_found; @@ -3129,6 +3345,37 @@ BIMODAL_ACCESSOR_C(String, int, length) BIMODAL_ACCESSOR(FeedbackCell, HeapObject, value) +ObjectRef MapRef::GetStrongValue(int descriptor_index) const { + if (broker()->mode() == JSHeapBroker::kDisabled) { + AllowHandleDereference allow_handle_dereference; + return ObjectRef(broker(), + handle(object()->instance_descriptors().GetStrongValue( + descriptor_index), + broker()->isolate())); + } + return ObjectRef(broker(), data()->AsMap()->GetStrongValue(descriptor_index)); +} + +void MapRef::SerializeRootMap() { + if (broker()->mode() == JSHeapBroker::kDisabled) return; + CHECK_EQ(broker()->mode(), JSHeapBroker::kSerializing); + data()->AsMap()->SerializeRootMap(broker()); +} + +base::Optional<MapRef> MapRef::FindRootMap() const { + if (broker()->mode() == JSHeapBroker::kDisabled) { + AllowHandleDereference allow_handle_dereference; + return MapRef(broker(), handle(object()->FindRootMap(broker()->isolate()), + broker()->isolate())); + } + MapData* map_data = data()->AsMap()->FindRootMap(); + if (map_data) { + return MapRef(broker(), map_data); + } + TRACE_BROKER_MISSING(broker(), "root map for object " << *this); + return base::nullopt; +} + void* JSTypedArrayRef::external_pointer() const { if (broker()->mode() == JSHeapBroker::kDisabled) { AllowHandleDereference allow_handle_dereference; @@ -3297,7 +3544,7 @@ Maybe<double> ObjectRef::OddballToNumber() const { } base::Optional<ObjectRef> ObjectRef::GetOwnConstantElement( - uint32_t index, bool serialize) const { + uint32_t index, SerializationPolicy policy) const { if (broker()->mode() == JSHeapBroker::kDisabled) { return (IsJSObject() || IsString()) ? GetOwnElementFromHeap(broker(), object(), index, true) @@ -3306,35 +3553,36 @@ base::Optional<ObjectRef> ObjectRef::GetOwnConstantElement( ObjectData* element = nullptr; if (IsJSObject()) { element = - data()->AsJSObject()->GetOwnConstantElement(broker(), index, serialize); + data()->AsJSObject()->GetOwnConstantElement(broker(), index, policy); } else if (IsString()) { - element = data()->AsString()->GetCharAsString(broker(), index, serialize); + element = data()->AsString()->GetCharAsString(broker(), index, policy); } if (element == nullptr) return base::nullopt; return ObjectRef(broker(), element); } -base::Optional<ObjectRef> JSObjectRef::GetOwnProperty( +base::Optional<ObjectRef> JSObjectRef::GetOwnDataProperty( Representation field_representation, FieldIndex index, - bool serialize) const { + SerializationPolicy policy) const { if (broker()->mode() == JSHeapBroker::kDisabled) { - return GetOwnPropertyFromHeap(broker(), Handle<JSObject>::cast(object()), - field_representation, index); + return GetOwnDataPropertyFromHeap(broker(), + Handle<JSObject>::cast(object()), + field_representation, index); } - ObjectData* property = data()->AsJSObject()->GetOwnProperty( - broker(), field_representation, index, serialize); + ObjectData* property = data()->AsJSObject()->GetOwnDataProperty( + broker(), field_representation, index, policy); if (property == nullptr) return base::nullopt; return ObjectRef(broker(), property); } -base::Optional<ObjectRef> JSArrayRef::GetOwnCowElement(uint32_t index, - bool serialize) const { +base::Optional<ObjectRef> JSArrayRef::GetOwnCowElement( + uint32_t index, SerializationPolicy policy) const { if (broker()->mode() == JSHeapBroker::kDisabled) { if (!object()->elements().IsCowArray()) return base::nullopt; return GetOwnElementFromHeap(broker(), object(), index, false); } - if (serialize) { + if (policy == SerializationPolicy::kSerializeIfNeeded) { data()->AsJSObject()->SerializeElements(broker()); } else if (!data()->AsJSObject()->serialized_elements()) { TRACE(broker(), "'elements' on " << this); @@ -3343,7 +3591,7 @@ base::Optional<ObjectRef> JSArrayRef::GetOwnCowElement(uint32_t index, if (!elements().map().IsFixedCowArrayMap()) return base::nullopt; ObjectData* element = - data()->AsJSArray()->GetOwnElement(broker(), index, serialize); + data()->AsJSArray()->GetOwnElement(broker(), index, policy); if (element == nullptr) return base::nullopt; return ObjectRef(broker(), element); } @@ -3353,27 +3601,25 @@ double HeapNumberRef::value() const { return data()->AsHeapNumber()->value(); } -double MutableHeapNumberRef::value() const { - IF_BROKER_DISABLED_ACCESS_HANDLE_C(MutableHeapNumber, value); - return data()->AsMutableHeapNumber()->value(); -} - uint64_t BigIntRef::AsUint64() const { IF_BROKER_DISABLED_ACCESS_HANDLE_C(BigInt, AsUint64); return data()->AsBigInt()->AsUint64(); } -CellRef SourceTextModuleRef::GetCell(int cell_index) const { +base::Optional<CellRef> SourceTextModuleRef::GetCell(int cell_index) const { if (broker()->mode() == JSHeapBroker::kDisabled) { AllowHandleAllocation handle_allocation; AllowHandleDereference allow_handle_dereference; return CellRef(broker(), handle(object()->GetCell(cell_index), broker()->isolate())); } - return CellRef(broker(), data()->AsSourceTextModule()->GetCell(cell_index)); + CellData* cell = data()->AsSourceTextModule()->GetCell(broker(), cell_index); + if (cell == nullptr) return base::nullopt; + return CellRef(broker(), cell); } -ObjectRef::ObjectRef(JSHeapBroker* broker, Handle<Object> object) +ObjectRef::ObjectRef(JSHeapBroker* broker, Handle<Object> object, + bool check_type) : broker_(broker) { switch (broker->mode()) { case JSHeapBroker::kSerialized: @@ -3398,6 +3644,10 @@ ObjectRef::ObjectRef(JSHeapBroker* broker, Handle<Object> object) case JSHeapBroker::kRetired: UNREACHABLE(); } + if (!data_) { // TODO(mslekova): Remove once we're on the background thread. + AllowHandleDereference handle_dereference; + object->Print(); + } CHECK_WITH_MSG(data_ != nullptr, "Object is not known to the heap broker"); } @@ -3489,8 +3739,8 @@ Float64 FixedDoubleArrayData::Get(int i) const { return contents_[i]; } -void FeedbackVectorRef::SerializeSlots() { - data()->AsFeedbackVector()->SerializeSlots(broker()); +void FeedbackVectorRef::Serialize() { + data()->AsFeedbackVector()->Serialize(broker()); } bool NameRef::IsUniqueName() const { @@ -3597,8 +3847,13 @@ void JSFunctionRef::Serialize() { data()->AsJSFunction()->Serialize(broker()); } +bool JSBoundFunctionRef::serialized() const { + if (broker()->mode() == JSHeapBroker::kDisabled) return true; + return data()->AsJSBoundFunction()->serialized(); +} + bool JSFunctionRef::serialized() const { - CHECK_NE(broker()->mode(), JSHeapBroker::kDisabled); + if (broker()->mode() == JSHeapBroker::kDisabled) return true; return data()->AsJSFunction()->serialized(); } @@ -3614,10 +3869,9 @@ bool JSFunctionRef::IsSerializedForCompilation() const { shared().IsSerializedForCompilation(feedback_vector()); } -JSArrayRef SharedFunctionInfoRef::GetTemplateObject(ObjectRef description, - FeedbackVectorRef vector, - FeedbackSlot slot, - bool serialize) { +JSArrayRef SharedFunctionInfoRef::GetTemplateObject( + ObjectRef description, FeedbackVectorRef vector, FeedbackSlot slot, + SerializationPolicy policy) { // Look in the feedback vector for the array. A Smi indicates that it's // not yet cached here. ObjectRef candidate = vector.get(slot); @@ -3632,22 +3886,22 @@ JSArrayRef SharedFunctionInfoRef::GetTemplateObject(ObjectRef description, Handle<TemplateObjectDescription>::cast(description.object()); Handle<JSArray> template_object = TemplateObjectDescription::GetTemplateObject( - broker()->isolate(), broker()->native_context().object(), tod, - object(), slot.ToInt()); + broker()->isolate(), broker()->target_native_context().object(), + tod, object(), slot.ToInt()); return JSArrayRef(broker(), template_object); } JSArrayData* array = data()->AsSharedFunctionInfo()->GetTemplateObject(slot); if (array != nullptr) return JSArrayRef(broker(), array); - CHECK(serialize); + CHECK_EQ(policy, SerializationPolicy::kSerializeIfNeeded); CHECK(broker()->SerializingAllowed()); Handle<TemplateObjectDescription> tod = Handle<TemplateObjectDescription>::cast(description.object()); Handle<JSArray> template_object = TemplateObjectDescription::GetTemplateObject( - broker()->isolate(), broker()->native_context().object(), tod, + broker()->isolate(), broker()->target_native_context().object(), tod, object(), slot.ToInt()); array = broker()->GetOrCreateData(template_object)->AsJSArray(); data()->AsSharedFunctionInfo()->SetTemplateObject(slot, array); @@ -3663,15 +3917,17 @@ void SharedFunctionInfoRef::SetSerializedForCompilation( void SharedFunctionInfoRef::SerializeFunctionTemplateInfo() { CHECK_EQ(broker()->mode(), JSHeapBroker::kSerializing); - data()->AsSharedFunctionInfo()->SerializeFunctionTemplateInfo(broker()); } base::Optional<FunctionTemplateInfoRef> SharedFunctionInfoRef::function_template_info() const { if (broker()->mode() == JSHeapBroker::kDisabled) { - return FunctionTemplateInfoRef( - broker(), handle(object()->function_data(), broker()->isolate())); + if (object()->IsApiFunction()) { + return FunctionTemplateInfoRef( + broker(), handle(object()->function_data(), broker()->isolate())); + } + return base::nullopt; } FunctionTemplateInfoData* function_template_info = data()->AsSharedFunctionInfo()->function_template_info(); @@ -3703,6 +3959,16 @@ void MapRef::SerializeOwnDescriptor(int descriptor_index) { data()->AsMap()->SerializeOwnDescriptor(broker(), descriptor_index); } +bool MapRef::serialized_own_descriptor(int descriptor_index) const { + CHECK_LT(descriptor_index, NumberOfOwnDescriptors()); + if (broker()->mode() == JSHeapBroker::kDisabled) return true; + DescriptorArrayData* desc_array_data = + data()->AsMap()->instance_descriptors(); + if (!desc_array_data) return false; + return desc_array_data->contents().find(descriptor_index) != + desc_array_data->contents().end(); +} + void MapRef::SerializeBackPointer() { if (broker()->mode() == JSHeapBroker::kDisabled) return; CHECK_EQ(broker()->mode(), JSHeapBroker::kSerializing); @@ -3762,13 +4028,13 @@ void FunctionTemplateInfoRef::SerializeCallCode() { } base::Optional<PropertyCellRef> JSGlobalProxyRef::GetPropertyCell( - NameRef const& name, bool serialize) const { + NameRef const& name, SerializationPolicy policy) const { if (broker()->mode() == JSHeapBroker::kDisabled) { return GetPropertyCellFromHeap(broker(), name.object()); } PropertyCellData* property_cell_data = - data()->AsJSGlobalProxy()->GetPropertyCell( - broker(), name.data()->AsName(), serialize); + data()->AsJSGlobalProxy()->GetPropertyCell(broker(), + name.data()->AsName(), policy); if (property_cell_data == nullptr) return base::nullopt; return PropertyCellRef(broker(), property_cell_data); } @@ -3787,64 +4053,133 @@ bool CanInlineElementAccess(MapRef const& map) { return false; } -InsufficientFeedback::InsufficientFeedback() - : ProcessedFeedback(kInsufficient) {} +ProcessedFeedback::ProcessedFeedback(Kind kind, FeedbackSlotKind slot_kind) + : kind_(kind), slot_kind_(slot_kind) {} + +KeyedAccessMode ElementAccessFeedback::keyed_mode() const { + return keyed_mode_; +} + +ZoneVector<ElementAccessFeedback::TransitionGroup> const& +ElementAccessFeedback::transition_groups() const { + return transition_groups_; +} + +ElementAccessFeedback const& ElementAccessFeedback::Refine( + ZoneVector<Handle<Map>> const& inferred_maps, Zone* zone) const { + ElementAccessFeedback& refined_feedback = + *new (zone) ElementAccessFeedback(zone, keyed_mode(), slot_kind()); + if (inferred_maps.empty()) return refined_feedback; + + ZoneUnorderedSet<Handle<Map>, Handle<Map>::hash, Handle<Map>::equal_to> + inferred(zone); + inferred.insert(inferred_maps.begin(), inferred_maps.end()); + + for (auto const& group : transition_groups()) { + DCHECK(!group.empty()); + TransitionGroup new_group(zone); + for (size_t i = 1; i < group.size(); ++i) { + Handle<Map> source = group[i]; + if (inferred.find(source) != inferred.end()) { + new_group.push_back(source); + } + } + + Handle<Map> target = group.front(); + bool const keep_target = + inferred.find(target) != inferred.end() || new_group.size() > 1; + if (keep_target) { + new_group.push_back(target); + // The target must be at the front, the order of sources doesn't matter. + std::swap(new_group[0], new_group[new_group.size() - 1]); + } + + if (!new_group.empty()) { + DCHECK(new_group.size() == 1 || new_group.front().equals(target)); + refined_feedback.transition_groups_.push_back(std::move(new_group)); + } + } + return refined_feedback; +} + +InsufficientFeedback::InsufficientFeedback(FeedbackSlotKind slot_kind) + : ProcessedFeedback(kInsufficient, slot_kind) {} -GlobalAccessFeedback::GlobalAccessFeedback(PropertyCellRef cell) - : ProcessedFeedback(kGlobalAccess), +GlobalAccessFeedback::GlobalAccessFeedback(PropertyCellRef cell, + FeedbackSlotKind slot_kind) + : ProcessedFeedback(kGlobalAccess, slot_kind), cell_or_context_(cell), - index_and_immutable_(0 /* doesn't matter */) {} + index_and_immutable_(0 /* doesn't matter */) { + DCHECK(IsGlobalICKind(slot_kind)); +} + +GlobalAccessFeedback::GlobalAccessFeedback(FeedbackSlotKind slot_kind) + : ProcessedFeedback(kGlobalAccess, slot_kind), + index_and_immutable_(0 /* doesn't matter */) { + DCHECK(IsGlobalICKind(slot_kind)); +} GlobalAccessFeedback::GlobalAccessFeedback(ContextRef script_context, - int slot_index, bool immutable) - : ProcessedFeedback(kGlobalAccess), + int slot_index, bool immutable, + FeedbackSlotKind slot_kind) + : ProcessedFeedback(kGlobalAccess, slot_kind), cell_or_context_(script_context), index_and_immutable_(FeedbackNexus::SlotIndexBits::encode(slot_index) | FeedbackNexus::ImmutabilityBit::encode(immutable)) { DCHECK_EQ(this->slot_index(), slot_index); DCHECK_EQ(this->immutable(), immutable); + DCHECK(IsGlobalICKind(slot_kind)); } +bool GlobalAccessFeedback::IsMegamorphic() const { + return !cell_or_context_.has_value(); +} bool GlobalAccessFeedback::IsPropertyCell() const { - return cell_or_context_.IsPropertyCell(); + return cell_or_context_.has_value() && cell_or_context_->IsPropertyCell(); +} +bool GlobalAccessFeedback::IsScriptContextSlot() const { + return cell_or_context_.has_value() && cell_or_context_->IsContext(); } PropertyCellRef GlobalAccessFeedback::property_cell() const { - DCHECK(IsPropertyCell()); - return cell_or_context_.AsPropertyCell(); + CHECK(IsPropertyCell()); + return cell_or_context_->AsPropertyCell(); } ContextRef GlobalAccessFeedback::script_context() const { - DCHECK(IsScriptContextSlot()); - return cell_or_context_.AsContext(); + CHECK(IsScriptContextSlot()); + return cell_or_context_->AsContext(); } int GlobalAccessFeedback::slot_index() const { - CHECK(IsScriptContextSlot()); + DCHECK(IsScriptContextSlot()); return FeedbackNexus::SlotIndexBits::decode(index_and_immutable_); } bool GlobalAccessFeedback::immutable() const { - CHECK(IsScriptContextSlot()); + DCHECK(IsScriptContextSlot()); return FeedbackNexus::ImmutabilityBit::decode(index_and_immutable_); } base::Optional<ObjectRef> GlobalAccessFeedback::GetConstantHint() const { - if (IsScriptContextSlot()) { - if (immutable()) return script_context().get(slot_index()); - } else { + if (IsPropertyCell()) { return property_cell().value(); + } else if (IsScriptContextSlot() && immutable()) { + return script_context().get(slot_index()); + } else { + return base::nullopt; } - return {}; } KeyedAccessMode KeyedAccessMode::FromNexus(FeedbackNexus const& nexus) { - if (IsKeyedLoadICKind(nexus.kind())) { + FeedbackSlotKind kind = nexus.kind(); + if (IsKeyedLoadICKind(kind)) { return KeyedAccessMode(AccessMode::kLoad, nexus.GetKeyedAccessLoadMode()); } - if (IsKeyedHasICKind(nexus.kind())) { + if (IsKeyedHasICKind(kind)) { return KeyedAccessMode(AccessMode::kHas, nexus.GetKeyedAccessLoadMode()); } - if (IsKeyedStoreICKind(nexus.kind())) { + if (IsKeyedStoreICKind(kind)) { return KeyedAccessMode(AccessMode::kStore, nexus.GetKeyedAccessStoreMode()); } - if (IsStoreInArrayLiteralICKind(nexus.kind())) { + if (IsStoreInArrayLiteralICKind(kind) || + IsStoreDataPropertyInLiteralKind(kind)) { return KeyedAccessMode(AccessMode::kStoreInLiteral, nexus.GetKeyedAccessStoreMode()); } @@ -3890,59 +4225,40 @@ KeyedAccessMode::KeyedAccessMode(AccessMode access_mode, } ElementAccessFeedback::ElementAccessFeedback(Zone* zone, - KeyedAccessMode const& keyed_mode) - : ProcessedFeedback(kElementAccess), - receiver_maps(zone), - transitions(zone), - keyed_mode(keyed_mode) {} - -ElementAccessFeedback::MapIterator::MapIterator( - ElementAccessFeedback const& processed, JSHeapBroker* broker) - : processed_(processed), broker_(broker) { - CHECK_LT(processed.receiver_maps.size(), - std::numeric_limits<size_t>::max() - processed.transitions.size()); -} - -bool ElementAccessFeedback::MapIterator::done() const { - return index_ >= - processed_.receiver_maps.size() + processed_.transitions.size(); -} - -void ElementAccessFeedback::MapIterator::advance() { index_++; } - -MapRef ElementAccessFeedback::MapIterator::current() const { - CHECK(!done()); - size_t receiver_maps_size = processed_.receiver_maps.size(); - Handle<Map> map; - if (index_ < receiver_maps_size) { - map = processed_.receiver_maps[index_]; - } else { - map = processed_.transitions[index_ - receiver_maps_size].first; + KeyedAccessMode const& keyed_mode, + FeedbackSlotKind slot_kind) + : ProcessedFeedback(kElementAccess, slot_kind), + keyed_mode_(keyed_mode), + transition_groups_(zone) { + DCHECK(IsKeyedLoadICKind(slot_kind) || IsKeyedHasICKind(slot_kind) || + IsStoreDataPropertyInLiteralKind(slot_kind) || + IsKeyedStoreICKind(slot_kind) || + IsStoreInArrayLiteralICKind(slot_kind)); +} + +bool ElementAccessFeedback::HasOnlyStringMaps(JSHeapBroker* broker) const { + for (auto const& group : transition_groups()) { + for (Handle<Map> map : group) { + if (!MapRef(broker, map).IsStringMap()) return false; + } } - return MapRef(broker_, map); -} - -ElementAccessFeedback::MapIterator ElementAccessFeedback::all_maps( - JSHeapBroker* broker) const { - return MapIterator(*this, broker); + return true; } -NamedAccessFeedback::NamedAccessFeedback( - NameRef const& name, ZoneVector<PropertyAccessInfo> const& access_infos) - : ProcessedFeedback(kNamedAccess), - name_(name), - access_infos_(access_infos) { - CHECK(!access_infos.empty()); +NamedAccessFeedback::NamedAccessFeedback(NameRef const& name, + ZoneVector<Handle<Map>> const& maps, + FeedbackSlotKind slot_kind) + : ProcessedFeedback(kNamedAccess, slot_kind), name_(name), maps_(maps) { + DCHECK(IsLoadICKind(slot_kind) || IsStoreICKind(slot_kind) || + IsStoreOwnICKind(slot_kind) || IsKeyedLoadICKind(slot_kind) || + IsKeyedHasICKind(slot_kind) || IsKeyedStoreICKind(slot_kind) || + IsStoreInArrayLiteralICKind(slot_kind) || + IsStoreDataPropertyInLiteralKind(slot_kind)); } -FeedbackSource::FeedbackSource(FeedbackNexus const& nexus) - : vector(nexus.vector_handle()), slot(nexus.slot()) {} - -FeedbackSource::FeedbackSource(VectorSlotPair const& pair) - : vector(pair.vector()), slot(pair.slot()) {} - void JSHeapBroker::SetFeedback(FeedbackSource const& source, ProcessedFeedback const* feedback) { + CHECK(source.IsValid()); auto insertion = feedback_.insert({source, feedback}); CHECK(insertion.second); } @@ -3951,80 +4267,90 @@ bool JSHeapBroker::HasFeedback(FeedbackSource const& source) const { return feedback_.find(source) != feedback_.end(); } -ProcessedFeedback const* JSHeapBroker::GetFeedback( +ProcessedFeedback const& JSHeapBroker::GetFeedback( FeedbackSource const& source) const { + DCHECK(source.IsValid()); auto it = feedback_.find(source); CHECK_NE(it, feedback_.end()); - return it->second; + return *it->second; } -GlobalAccessFeedback const* JSHeapBroker::GetGlobalAccessFeedback( +FeedbackSlotKind JSHeapBroker::GetFeedbackSlotKind( FeedbackSource const& source) const { - ProcessedFeedback const* feedback = GetFeedback(source); - if (feedback == nullptr) return nullptr; - CHECK_EQ(feedback->kind(), ProcessedFeedback::kGlobalAccess); - return static_cast<GlobalAccessFeedback const*>(feedback); -} - -ElementAccessFeedback const* JSHeapBroker::ProcessFeedbackMapsForElementAccess( - MapHandles const& maps, KeyedAccessMode const& keyed_mode) { - DCHECK(!maps.empty()); - - // Collect possible transition targets. - MapHandles possible_transition_targets; - possible_transition_targets.reserve(maps.size()); - for (Handle<Map> map : maps) { - if (CanInlineElementAccess(MapRef(this, map)) && - IsFastElementsKind(map->elements_kind()) && - GetInitialFastElementsKind() != map->elements_kind()) { - possible_transition_targets.push_back(map); - } + if (FLAG_concurrent_inlining) { + ProcessedFeedback const& processed = GetFeedback(source); + return processed.slot_kind(); } + FeedbackNexus nexus(source.vector, source.slot); + return nexus.kind(); +} - ElementAccessFeedback* result = - new (zone()) ElementAccessFeedback(zone(), keyed_mode); +bool JSHeapBroker::FeedbackIsInsufficient(FeedbackSource const& source) const { + return FLAG_concurrent_inlining + ? GetFeedback(source).IsInsufficient() + : FeedbackNexus(source.vector, source.slot).IsUninitialized(); +} - // Separate the actual receiver maps and the possible transition sources. +namespace { +MapHandles GetRelevantReceiverMaps(Isolate* isolate, MapHandles const& maps) { + MapHandles result; for (Handle<Map> map : maps) { - // Don't generate elements kind transitions from stable maps. - Map transition_target = map->is_stable() - ? Map() - : map->FindElementsKindTransitionedMap( - isolate(), possible_transition_targets); - if (transition_target.is_null()) { - result->receiver_maps.push_back(map); - } else { - result->transitions.emplace_back(map, - handle(transition_target, isolate())); + if (Map::TryUpdate(isolate, map).ToHandle(&map) && + !map->is_abandoned_prototype_map()) { + DCHECK(!map->is_deprecated()); + result.push_back(map); } } + return result; +} // namespace +} // namespace -#ifdef ENABLE_SLOW_DCHECKS - // No transition sources appear in {receiver_maps}. - // All transition targets appear in {receiver_maps}. - for (auto& transition : result->transitions) { - CHECK(std::none_of( - result->receiver_maps.cbegin(), result->receiver_maps.cend(), - [&](Handle<Map> map) { return map.equals(transition.first); })); - CHECK(std::any_of( - result->receiver_maps.cbegin(), result->receiver_maps.cend(), - [&](Handle<Map> map) { return map.equals(transition.second); })); +ProcessedFeedback const& JSHeapBroker::ReadFeedbackForPropertyAccess( + FeedbackSource const& source, AccessMode mode, + base::Optional<NameRef> static_name) { + FeedbackNexus nexus(source.vector, source.slot); + FeedbackSlotKind kind = nexus.kind(); + if (nexus.IsUninitialized()) return *new (zone()) InsufficientFeedback(kind); + + MapHandles maps; + nexus.ExtractMaps(&maps); + DCHECK_NE(nexus.ic_state(), PREMONOMORPHIC); + if (!maps.empty()) { + maps = GetRelevantReceiverMaps(isolate(), maps); + if (maps.empty()) return *new (zone()) InsufficientFeedback(kind); + } + + base::Optional<NameRef> name = + static_name.has_value() ? static_name : GetNameFeedback(nexus); + if (name.has_value()) { + return *new (zone()) NamedAccessFeedback( + *name, ZoneVector<Handle<Map>>(maps.begin(), maps.end(), zone()), kind); + } else if (nexus.GetKeyType() == ELEMENT && !maps.empty()) { + return ProcessFeedbackMapsForElementAccess( + maps, KeyedAccessMode::FromNexus(nexus), kind); + } else { + // No actionable feedback. + DCHECK(maps.empty()); + // TODO(neis): Investigate if we really want to treat cleared the same as + // megamorphic (also for global accesses). + // TODO(neis): Using ElementAccessFeedback here is kind of an abuse. + return *new (zone()) + ElementAccessFeedback(zone(), KeyedAccessMode::FromNexus(nexus), kind); } -#endif - CHECK(!result->receiver_maps.empty()); - - return result; } -GlobalAccessFeedback const* JSHeapBroker::ProcessFeedbackForGlobalAccess( +ProcessedFeedback const& JSHeapBroker::ReadFeedbackForGlobalAccess( FeedbackSource const& source) { FeedbackNexus nexus(source.vector, source.slot); DCHECK(nexus.kind() == FeedbackSlotKind::kLoadGlobalInsideTypeof || nexus.kind() == FeedbackSlotKind::kLoadGlobalNotInsideTypeof || nexus.kind() == FeedbackSlotKind::kStoreGlobalSloppy || nexus.kind() == FeedbackSlotKind::kStoreGlobalStrict); + if (nexus.IsUninitialized()) { + return *new (zone()) InsufficientFeedback(nexus.kind()); + } if (nexus.ic_state() != MONOMORPHIC || nexus.GetFeedback()->IsCleared()) { - return nullptr; + return *new (zone()) GlobalAccessFeedback(nexus.kind()); } Handle<Object> feedback_value(nexus.GetFeedback()->GetHeapObjectOrSmi(), @@ -4039,7 +4365,7 @@ GlobalAccessFeedback const* JSHeapBroker::ProcessFeedbackForGlobalAccess( int const context_slot_index = FeedbackNexus::SlotIndexBits::decode(number); bool const immutable = FeedbackNexus::ImmutabilityBit::decode(number); Handle<Context> context = ScriptContextTable::GetContext( - isolate(), native_context().script_context_table().object(), + isolate(), target_native_context().script_context_table().object(), script_context_index); { ObjectRef contents(this, @@ -4049,10 +4375,11 @@ GlobalAccessFeedback const* JSHeapBroker::ProcessFeedbackForGlobalAccess( } ContextRef context_ref(this, context); if (immutable) { - context_ref.get(context_slot_index, true); + context_ref.get(context_slot_index, + SerializationPolicy::kSerializeIfNeeded); } - return new (zone()) - GlobalAccessFeedback(context_ref, context_slot_index, immutable); + return *new (zone()) GlobalAccessFeedback(context_ref, context_slot_index, + immutable, nexus.kind()); } CHECK(feedback_value->IsPropertyCell()); @@ -4060,11 +4387,275 @@ GlobalAccessFeedback const* JSHeapBroker::ProcessFeedbackForGlobalAccess( // object and the feedback is the cell holding its value. PropertyCellRef cell(this, Handle<PropertyCell>::cast(feedback_value)); cell.Serialize(); - return new (zone()) GlobalAccessFeedback(cell); + return *new (zone()) GlobalAccessFeedback(cell, nexus.kind()); +} + +BinaryOperationHint JSHeapBroker::ReadFeedbackForBinaryOperation( + FeedbackSource const& source) const { + return FeedbackNexus(source.vector, source.slot).GetBinaryOperationFeedback(); +} + +CompareOperationHint JSHeapBroker::ReadFeedbackForCompareOperation( + FeedbackSource const& source) const { + return FeedbackNexus(source.vector, source.slot) + .GetCompareOperationFeedback(); +} + +ForInHint JSHeapBroker::ReadFeedbackForForIn( + FeedbackSource const& source) const { + return FeedbackNexus(source.vector, source.slot).GetForInFeedback(); +} + +ProcessedFeedback const& JSHeapBroker::ReadFeedbackForInstanceOf( + FeedbackSource const& source) { + FeedbackNexus nexus(source.vector, source.slot); + if (nexus.IsUninitialized()) + return *new (zone()) InsufficientFeedback(nexus.kind()); + + base::Optional<JSObjectRef> optional_constructor; + { + MaybeHandle<JSObject> maybe_constructor = nexus.GetConstructorFeedback(); + Handle<JSObject> constructor; + if (maybe_constructor.ToHandle(&constructor)) { + optional_constructor = JSObjectRef(this, constructor); + } + } + return *new (zone()) InstanceOfFeedback(optional_constructor, nexus.kind()); +} + +ProcessedFeedback const& JSHeapBroker::ReadFeedbackForCall( + FeedbackSource const& source) { + FeedbackNexus nexus(source.vector, source.slot); + if (nexus.IsUninitialized()) + return *new (zone()) InsufficientFeedback(nexus.kind()); + + base::Optional<HeapObjectRef> target_ref; + { + MaybeObject maybe_target = nexus.GetFeedback(); + HeapObject target_object; + if (maybe_target->GetHeapObject(&target_object)) { + target_ref = HeapObjectRef(this, handle(target_object, isolate())); + } + } + float frequency = nexus.ComputeCallFrequency(); + SpeculationMode mode = nexus.GetSpeculationMode(); + return *new (zone()) CallFeedback(target_ref, frequency, mode, nexus.kind()); +} + +BinaryOperationHint JSHeapBroker::GetFeedbackForBinaryOperation( + FeedbackSource const& source) { + ProcessedFeedback const& feedback = + FLAG_concurrent_inlining ? GetFeedback(source) + : ProcessFeedbackForBinaryOperation(source); + return feedback.IsInsufficient() ? BinaryOperationHint::kNone + : feedback.AsBinaryOperation().value(); +} + +CompareOperationHint JSHeapBroker::GetFeedbackForCompareOperation( + FeedbackSource const& source) { + ProcessedFeedback const& feedback = + FLAG_concurrent_inlining ? GetFeedback(source) + : ProcessFeedbackForCompareOperation(source); + return feedback.IsInsufficient() ? CompareOperationHint::kNone + : feedback.AsCompareOperation().value(); +} + +ForInHint JSHeapBroker::GetFeedbackForForIn(FeedbackSource const& source) { + ProcessedFeedback const& feedback = FLAG_concurrent_inlining + ? GetFeedback(source) + : ProcessFeedbackForForIn(source); + return feedback.IsInsufficient() ? ForInHint::kNone + : feedback.AsForIn().value(); +} + +ProcessedFeedback const& JSHeapBroker::GetFeedbackForPropertyAccess( + FeedbackSource const& source, AccessMode mode, + base::Optional<NameRef> static_name) { + return FLAG_concurrent_inlining + ? GetFeedback(source) + : ProcessFeedbackForPropertyAccess(source, mode, static_name); +} + +ProcessedFeedback const& JSHeapBroker::GetFeedbackForInstanceOf( + FeedbackSource const& source) { + return FLAG_concurrent_inlining ? GetFeedback(source) + : ProcessFeedbackForInstanceOf(source); +} + +ProcessedFeedback const& JSHeapBroker::GetFeedbackForCall( + FeedbackSource const& source) { + return FLAG_concurrent_inlining ? GetFeedback(source) + : ProcessFeedbackForCall(source); +} + +ProcessedFeedback const& JSHeapBroker::GetFeedbackForGlobalAccess( + FeedbackSource const& source) { + return FLAG_concurrent_inlining ? GetFeedback(source) + : ProcessFeedbackForGlobalAccess(source); +} + +ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForBinaryOperation( + FeedbackSource const& source) { + if (HasFeedback(source)) return GetFeedback(source); + BinaryOperationHint hint = ReadFeedbackForBinaryOperation(source); + ProcessedFeedback const* feedback; + if (hint == BinaryOperationHint::kNone) { + feedback = + new (zone()) InsufficientFeedback(source.vector->GetKind(source.slot)); + } else { + feedback = new (zone()) + BinaryOperationFeedback(hint, source.vector->GetKind(source.slot)); + } + SetFeedback(source, feedback); + return *feedback; +} + +ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForCompareOperation( + FeedbackSource const& source) { + if (HasFeedback(source)) return GetFeedback(source); + CompareOperationHint hint = ReadFeedbackForCompareOperation(source); + ProcessedFeedback const* feedback; + if (hint == CompareOperationHint::kNone) { + feedback = + new (zone()) InsufficientFeedback(source.vector->GetKind(source.slot)); + } else { + feedback = new (zone()) + CompareOperationFeedback(hint, source.vector->GetKind(source.slot)); + } + SetFeedback(source, feedback); + return *feedback; +} + +ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForForIn( + FeedbackSource const& source) { + if (HasFeedback(source)) return GetFeedback(source); + ForInHint hint = ReadFeedbackForForIn(source); + ProcessedFeedback const* feedback; + if (hint == ForInHint::kNone) { + feedback = + new (zone()) InsufficientFeedback(source.vector->GetKind(source.slot)); + } else { + feedback = + new (zone()) ForInFeedback(hint, source.vector->GetKind(source.slot)); + } + SetFeedback(source, feedback); + return *feedback; +} + +ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForPropertyAccess( + FeedbackSource const& source, AccessMode mode, + base::Optional<NameRef> static_name) { + if (HasFeedback(source)) return GetFeedback(source); + ProcessedFeedback const& feedback = + ReadFeedbackForPropertyAccess(source, mode, static_name); + SetFeedback(source, &feedback); + return feedback; +} + +ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForInstanceOf( + FeedbackSource const& source) { + if (HasFeedback(source)) return GetFeedback(source); + ProcessedFeedback const& feedback = ReadFeedbackForInstanceOf(source); + SetFeedback(source, &feedback); + return feedback; +} + +ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForCall( + FeedbackSource const& source) { + if (HasFeedback(source)) return GetFeedback(source); + ProcessedFeedback const& feedback = ReadFeedbackForCall(source); + SetFeedback(source, &feedback); + return feedback; +} + +ProcessedFeedback const& JSHeapBroker::ProcessFeedbackForGlobalAccess( + FeedbackSource const& source) { + if (HasFeedback(source)) return GetFeedback(source); + ProcessedFeedback const& feedback = ReadFeedbackForGlobalAccess(source); + SetFeedback(source, &feedback); + return feedback; +} + +ElementAccessFeedback const& JSHeapBroker::ProcessFeedbackMapsForElementAccess( + MapHandles const& maps, KeyedAccessMode const& keyed_mode, + FeedbackSlotKind slot_kind) { + DCHECK(!maps.empty()); + + // Collect possible transition targets. + MapHandles possible_transition_targets; + possible_transition_targets.reserve(maps.size()); + for (Handle<Map> map : maps) { + MapRef map_ref(this, map); + map_ref.SerializeRootMap(); + + if (CanInlineElementAccess(map_ref) && + IsFastElementsKind(map->elements_kind()) && + GetInitialFastElementsKind() != map->elements_kind()) { + possible_transition_targets.push_back(map); + } + } + + using TransitionGroup = ElementAccessFeedback::TransitionGroup; + ZoneUnorderedMap<Handle<Map>, TransitionGroup, Handle<Map>::hash, + Handle<Map>::equal_to> + transition_groups(zone()); + + // Separate the actual receiver maps and the possible transition sources. + for (Handle<Map> map : maps) { + // Don't generate elements kind transitions from stable maps. + Map transition_target = map->is_stable() + ? Map() + : map->FindElementsKindTransitionedMap( + isolate(), possible_transition_targets); + if (transition_target.is_null()) { + TransitionGroup group(1, map, zone()); + transition_groups.insert({map, group}); + } else { + Handle<Map> target(transition_target, isolate()); + TransitionGroup new_group(1, target, zone()); + TransitionGroup& actual_group = + transition_groups.insert({target, new_group}).first->second; + actual_group.push_back(map); + } + } + + ElementAccessFeedback* result = + new (zone()) ElementAccessFeedback(zone(), keyed_mode, slot_kind); + for (auto entry : transition_groups) { + result->AddGroup(std::move(entry.second)); + } + + CHECK(!result->transition_groups().empty()); + return *result; +} + +void ElementAccessFeedback::AddGroup(TransitionGroup&& group) { + CHECK(!group.empty()); + transition_groups_.push_back(std::move(group)); + +#ifdef ENABLE_SLOW_DCHECKS + // Check that each of the group's maps occurs exactly once in the whole + // feedback. This implies that "a source is not a target". + for (Handle<Map> map : group) { + int count = 0; + for (TransitionGroup const& some_group : transition_groups()) { + count += std::count_if( + some_group.begin(), some_group.end(), + [&](Handle<Map> some_map) { return some_map.equals(map); }); + } + CHECK_EQ(count, 1); + } +#endif } std::ostream& operator<<(std::ostream& os, const ObjectRef& ref) { - return os << ref.data(); + if (ref.broker()->mode() == JSHeapBroker::kDisabled) { + // If the broker is disabled we cannot be in a background thread so it's + // safe to read the heap. + return os << ref.data() << " {" << ref.object() << "}"; + } else { + return os << ref.data(); + } } base::Optional<NameRef> JSHeapBroker::GetNameFeedback( @@ -4074,67 +4665,77 @@ base::Optional<NameRef> JSHeapBroker::GetNameFeedback( return NameRef(this, handle(raw_name, isolate())); } -PropertyAccessInfo JSHeapBroker::GetAccessInfoForLoadingThen(MapRef map) { - auto access_info = ais_for_loading_then_.find(map); - if (access_info == ais_for_loading_then_.end()) { - TRACE_BROKER_MISSING( - this, "access info for reducing JSResolvePromise with map " << map); +PropertyAccessInfo JSHeapBroker::GetPropertyAccessInfo( + MapRef map, NameRef name, AccessMode access_mode, + CompilationDependencies* dependencies, SerializationPolicy policy) { + PropertyAccessTarget target({map, name, access_mode}); + auto it = property_access_infos_.find(target); + if (it != property_access_infos_.end()) return it->second; + + if (policy == SerializationPolicy::kAssumeSerialized) { + TRACE_BROKER_MISSING(this, "PropertyAccessInfo for " + << access_mode << " of property " << name + << " on map " << map); return PropertyAccessInfo::Invalid(zone()); } - return access_info->second; -} -void JSHeapBroker::CreateAccessInfoForLoadingThen( - MapRef map, CompilationDependencies* dependencies) { - auto access_info = ais_for_loading_then_.find(map); - if (access_info == ais_for_loading_then_.end()) { - AccessInfoFactory access_info_factory(this, dependencies, zone()); - Handle<Name> then_string = isolate()->factory()->then_string(); - ais_for_loading_then_.insert( - std::make_pair(map, access_info_factory.ComputePropertyAccessInfo( - map.object(), then_string, AccessMode::kLoad))); + CHECK_NOT_NULL(dependencies); + AccessInfoFactory factory(this, dependencies, zone()); + PropertyAccessInfo access_info = factory.ComputePropertyAccessInfo( + map.object(), name.object(), access_mode); + if (FLAG_concurrent_inlining) { + CHECK(SerializingAllowed()); + TRACE(this, "Storing PropertyAccessInfo for " + << access_mode << " of property " << name << " on map " + << map); + property_access_infos_.insert({target, access_info}); } + return access_info; } -PropertyAccessInfo JSHeapBroker::GetAccessInfoForLoadingExec(MapRef map) { - auto access_info = ais_for_loading_exec_.find(map); - if (access_info == ais_for_loading_exec_.end()) { - TRACE_BROKER_MISSING(this, - "access info for property 'exec' on map " << map); - return PropertyAccessInfo::Invalid(zone()); - } - return access_info->second; +BinaryOperationFeedback const& ProcessedFeedback::AsBinaryOperation() const { + CHECK_EQ(kBinaryOperation, kind()); + return *static_cast<BinaryOperationFeedback const*>(this); } -PropertyAccessInfo const& JSHeapBroker::CreateAccessInfoForLoadingExec( - MapRef map, CompilationDependencies* dependencies) { - auto access_info = ais_for_loading_exec_.find(map); - if (access_info != ais_for_loading_exec_.end()) { - return access_info->second; - } - - ZoneVector<PropertyAccessInfo> access_infos(zone()); - AccessInfoFactory access_info_factory(this, dependencies, zone()); - PropertyAccessInfo ai_exec = access_info_factory.ComputePropertyAccessInfo( - map.object(), isolate()->factory()->exec_string(), AccessMode::kLoad); +CallFeedback const& ProcessedFeedback::AsCall() const { + CHECK_EQ(kCall, kind()); + return *static_cast<CallFeedback const*>(this); +} - auto inserted_ai = ais_for_loading_exec_.insert(std::make_pair(map, ai_exec)); - return inserted_ai.first->second; +CompareOperationFeedback const& ProcessedFeedback::AsCompareOperation() const { + CHECK_EQ(kCompareOperation, kind()); + return *static_cast<CompareOperationFeedback const*>(this); } -ElementAccessFeedback const* ProcessedFeedback::AsElementAccess() const { +ElementAccessFeedback const& ProcessedFeedback::AsElementAccess() const { CHECK_EQ(kElementAccess, kind()); - return static_cast<ElementAccessFeedback const*>(this); + return *static_cast<ElementAccessFeedback const*>(this); +} + +ForInFeedback const& ProcessedFeedback::AsForIn() const { + CHECK_EQ(kForIn, kind()); + return *static_cast<ForInFeedback const*>(this); +} + +GlobalAccessFeedback const& ProcessedFeedback::AsGlobalAccess() const { + CHECK_EQ(kGlobalAccess, kind()); + return *static_cast<GlobalAccessFeedback const*>(this); +} + +InstanceOfFeedback const& ProcessedFeedback::AsInstanceOf() const { + CHECK_EQ(kInstanceOf, kind()); + return *static_cast<InstanceOfFeedback const*>(this); } -NamedAccessFeedback const* ProcessedFeedback::AsNamedAccess() const { +NamedAccessFeedback const& ProcessedFeedback::AsNamedAccess() const { CHECK_EQ(kNamedAccess, kind()); - return static_cast<NamedAccessFeedback const*>(this); + return *static_cast<NamedAccessFeedback const*>(this); } BytecodeAnalysis const& JSHeapBroker::GetBytecodeAnalysis( Handle<BytecodeArray> bytecode_array, BailoutId osr_bailout_id, - bool analyze_liveness, bool serialize) { + bool analyze_liveness, SerializationPolicy policy) { ObjectData* bytecode_array_data = GetData(bytecode_array); CHECK_NOT_NULL(bytecode_array_data); @@ -4154,7 +4755,7 @@ BytecodeAnalysis const& JSHeapBroker::GetBytecodeAnalysis( return *it->second; } - CHECK(serialize); + CHECK_EQ(policy, SerializationPolicy::kSerializeIfNeeded); BytecodeAnalysis* analysis = new (zone()) BytecodeAnalysis( bytecode_array, zone(), osr_bailout_id, analyze_liveness); DCHECK_EQ(analysis->osr_bailout_id(), osr_bailout_id); |