diff options
Diffstat (limited to 'chromium/v8/src/compiler/wasm-compiler.cc')
| -rw-r--r-- | chromium/v8/src/compiler/wasm-compiler.cc | 387 |
1 files changed, 198 insertions, 189 deletions
diff --git a/chromium/v8/src/compiler/wasm-compiler.cc b/chromium/v8/src/compiler/wasm-compiler.cc index f91c21fd1d4..8446640bfc8 100644 --- a/chromium/v8/src/compiler/wasm-compiler.cc +++ b/chromium/v8/src/compiler/wasm-compiler.cc @@ -44,6 +44,7 @@ #include "src/roots/roots.h" #include "src/tracing/trace-event.h" #include "src/trap-handler/trap-handler.h" +#include "src/wasm/code-space-access.h" #include "src/wasm/function-body-decoder-impl.h" #include "src/wasm/function-compiler.h" #include "src/wasm/graph-builder-interface.h" @@ -196,14 +197,7 @@ class WasmGraphAssembler : public GraphAssembler { return Call(call_descriptor, call_target, args...); } - void EnsureEnd() { - if (graph()->end() == nullptr) { - graph()->SetEnd(graph()->NewNode(mcgraph()->common()->End(0))); - } - } - void MergeControlToEnd(Node* node) { - EnsureEnd(); NodeProperties::MergeControlToEnd(graph(), mcgraph()->common(), node); } @@ -212,7 +206,6 @@ class WasmGraphAssembler : public GraphAssembler { if (FLAG_debug_code) { auto ok = MakeLabel(); GotoIfNot(condition, &ok); - EnsureEnd(); Unreachable(); Bind(&ok); } @@ -472,7 +465,6 @@ WasmGraphBuilder::WasmGraphBuilder( mcgraph_(mcgraph), env_(env), has_simd_(ContainsSimd(sig)), - untrusted_code_mitigations_(FLAG_untrusted_code_mitigations), sig_(sig), source_position_table_(source_position_table), isolate_(isolate) { @@ -501,6 +493,8 @@ void WasmGraphBuilder::Start(unsigned params) { gasm_->LoadFunctionDataFromJSFunction( Param(Linkage::kJSCallClosureParamIndex, "%closure"))) : Param(wasm::kWasmInstanceParameterIndex); + + graph()->SetEnd(graph()->NewNode(mcgraph()->common()->End(0))); } Node* WasmGraphBuilder::Param(int index, const char* debug_name) { @@ -660,8 +654,9 @@ void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position) { mcgraph()->machine()->StackPointerGreaterThan(StackCheckKind::kWasm), limit, effect())); - Diamond stack_check(graph(), mcgraph()->common(), check, BranchHint::kTrue); - stack_check.Chain(control()); + Node* if_true; + Node* if_false; + gasm_->Branch(check, &if_true, &if_false, BranchHint::kTrue); if (stack_check_call_operator_ == nullptr) { // Build and cache the stack check call operator and the constant @@ -682,15 +677,18 @@ void WasmGraphBuilder::StackCheck(wasm::WasmCodePosition position) { stack_check_call_operator_ = mcgraph()->common()->Call(call_descriptor); } - Node* call = graph()->NewNode(stack_check_call_operator_.get(), - stack_check_code_node_.get(), effect(), - stack_check.if_false); - + Node* call = + graph()->NewNode(stack_check_call_operator_.get(), + stack_check_code_node_.get(), effect(), if_false); SetSourcePosition(call, position); - Node* ephi = stack_check.EffectPhi(effect(), call); + DCHECK_GT(call->op()->ControlOutputCount(), 0); + Node* merge = graph()->NewNode(mcgraph()->common()->Merge(2), if_true, call); + DCHECK_GT(call->op()->EffectOutputCount(), 0); + Node* ephi = graph()->NewNode(mcgraph()->common()->EffectPhi(2), effect(), + call, merge); - SetEffectControl(ephi, stack_check.merge); + SetEffectControl(ephi, merge); } void WasmGraphBuilder::PatchInStackCheckIfNeeded() { @@ -2901,18 +2899,18 @@ Node* WasmGraphBuilder::BuildCallNode(const wasm::FunctionSig* sig, return call; } -Node* WasmGraphBuilder::BuildWasmCall( - const wasm::FunctionSig* sig, base::Vector<Node*> args, - base::Vector<Node*> rets, wasm::WasmCodePosition position, - Node* instance_node, UseRetpoline use_retpoline, Node* frame_state) { - CallDescriptor* call_descriptor = - GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline, - kWasmFunction, frame_state != nullptr); +Node* WasmGraphBuilder::BuildWasmCall(const wasm::FunctionSig* sig, + base::Vector<Node*> args, + base::Vector<Node*> rets, + wasm::WasmCodePosition position, + Node* instance_node, Node* frame_state) { + CallDescriptor* call_descriptor = GetWasmCallDescriptor( + mcgraph()->zone(), sig, kWasmFunction, frame_state != nullptr); const Operator* op = mcgraph()->common()->Call(call_descriptor); Node* call = BuildCallNode(sig, args, position, instance_node, op, frame_state); - // TODO(manoskouk): Don't always set control if we ever add properties to wasm - // calls. + // TODO(manoskouk): If we have kNoThrow calls, do not set them as control. + DCHECK_GT(call->op()->ControlOutputCount(), 0); SetControl(call); size_t ret_count = sig->return_count(); @@ -2935,15 +2933,14 @@ Node* WasmGraphBuilder::BuildWasmCall( Node* WasmGraphBuilder::BuildWasmReturnCall(const wasm::FunctionSig* sig, base::Vector<Node*> args, wasm::WasmCodePosition position, - Node* instance_node, - UseRetpoline use_retpoline) { + Node* instance_node) { CallDescriptor* call_descriptor = - GetWasmCallDescriptor(mcgraph()->zone(), sig, use_retpoline); + GetWasmCallDescriptor(mcgraph()->zone(), sig); const Operator* op = mcgraph()->common()->TailCall(call_descriptor); Node* call = BuildCallNode(sig, args, position, instance_node, op); - // TODO(manoskouk): {call} will not always be a control node if we ever add - // properties to wasm calls. + // TODO(manoskouk): If we have kNoThrow calls, do not merge them to end. + DCHECK_GT(call->op()->ControlOutputCount(), 0); gasm_->MergeControlToEnd(call); return call; @@ -2982,15 +2979,13 @@ Node* WasmGraphBuilder::BuildImportCall(const wasm::FunctionSig* sig, Node* target_node = gasm_->LoadFromObject( MachineType::Pointer(), imported_targets, func_index_times_pointersize); args[0] = target_node; - const UseRetpoline use_retpoline = - untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline; switch (continuation) { case kCallContinues: - return BuildWasmCall(sig, args, rets, position, ref_node, use_retpoline); + return BuildWasmCall(sig, args, rets, position, ref_node); case kReturnCall: DCHECK(rets.empty()); - return BuildWasmReturnCall(sig, args, position, ref_node, use_retpoline); + return BuildWasmReturnCall(sig, args, position, ref_node); } } @@ -3010,7 +3005,7 @@ Node* WasmGraphBuilder::CallDirect(uint32_t index, base::Vector<Node*> args, Address code = static_cast<Address>(index); args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL); - return BuildWasmCall(sig, args, rets, position, nullptr, kNoRetpoline); + return BuildWasmCall(sig, args, rets, position, nullptr); } Node* WasmGraphBuilder::CallIndirect(uint32_t table_index, uint32_t sig_index, @@ -3095,16 +3090,6 @@ Node* WasmGraphBuilder::BuildIndirectCall(uint32_t table_index, Node* in_bounds = gasm_->Uint32LessThan(key, ift_size); TrapIfFalse(wasm::kTrapTableOutOfBounds, in_bounds, position); - // Mask the key to prevent SSCA. - if (untrusted_code_mitigations_) { - // mask = ((key - size) & ~key) >> 31 - Node* neg_key = gasm_->Word32Xor(key, Int32Constant(-1)); - Node* masked_diff = - gasm_->Word32And(gasm_->Int32Sub(key, ift_size), neg_key); - Node* mask = gasm_->Word32Sar(masked_diff, Int32Constant(31)); - key = gasm_->Word32And(key, mask); - } - const wasm::ValueType table_type = env_->module->tables[table_index].type; // Check that the table entry is not null and that the type of the function is // **identical with** the function type declared at the call site (no @@ -3140,16 +3125,12 @@ Node* WasmGraphBuilder::BuildIndirectCall(uint32_t table_index, intptr_scaled_key); args[0] = target; - const UseRetpoline use_retpoline = - untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline; switch (continuation) { case kCallContinues: - return BuildWasmCall(sig, args, rets, position, target_instance, - use_retpoline); + return BuildWasmCall(sig, args, rets, position, target_instance); case kReturnCall: - return BuildWasmReturnCall(sig, args, position, target_instance, - use_retpoline); + return BuildWasmReturnCall(sig, args, position, target_instance); } } @@ -3178,7 +3159,7 @@ Node* WasmGraphBuilder::BuildLoadCallTargetFromExportedFunctionData( } // TODO(9495): Support CAPI function refs. -Node* WasmGraphBuilder::BuildCallRef(uint32_t sig_index, +Node* WasmGraphBuilder::BuildCallRef(const wasm::FunctionSig* sig, base::Vector<Node*> args, base::Vector<Node*> rets, CheckForNull null_check, @@ -3189,8 +3170,6 @@ Node* WasmGraphBuilder::BuildCallRef(uint32_t sig_index, position); } - const wasm::FunctionSig* sig = env_->module->signature(sig_index); - Node* function_data = gasm_->LoadFunctionDataFromJSFunction(args[0]); auto load_target = gasm_->MakeLabel(); @@ -3244,31 +3223,43 @@ Node* WasmGraphBuilder::BuildCallRef(uint32_t sig_index, args[0] = end_label.PhiAt(0); - const UseRetpoline use_retpoline = - untrusted_code_mitigations_ ? kRetpoline : kNoRetpoline; - Node* call = continuation == kCallContinues - ? BuildWasmCall(sig, args, rets, position, instance_node, - use_retpoline) - : BuildWasmReturnCall(sig, args, position, instance_node, - use_retpoline); + ? BuildWasmCall(sig, args, rets, position, instance_node) + : BuildWasmReturnCall(sig, args, position, instance_node); return call; } -Node* WasmGraphBuilder::CallRef(uint32_t sig_index, base::Vector<Node*> args, +void WasmGraphBuilder::CompareToExternalFunctionAtIndex( + Node* func_ref, uint32_t function_index, Node** success_control, + Node** failure_control) { + // Since we are comparing to a function reference, it is guaranteed that + // instance->wasm_external_functions() has been initialized. + Node* external_functions = gasm_->LoadFromObject( + MachineType::TaggedPointer(), GetInstance(), + wasm::ObjectAccess::ToTagged( + WasmInstanceObject::kWasmExternalFunctionsOffset)); + Node* function_ref = gasm_->LoadFixedArrayElement( + external_functions, gasm_->IntPtrConstant(function_index), + MachineType::AnyTagged()); + gasm_->Branch(gasm_->WordEqual(function_ref, func_ref), success_control, + failure_control, BranchHint::kTrue); +} + +Node* WasmGraphBuilder::CallRef(const wasm::FunctionSig* sig, + base::Vector<Node*> args, base::Vector<Node*> rets, WasmGraphBuilder::CheckForNull null_check, wasm::WasmCodePosition position) { - return BuildCallRef(sig_index, args, rets, null_check, - IsReturnCall::kCallContinues, position); + return BuildCallRef(sig, args, rets, null_check, IsReturnCall::kCallContinues, + position); } -Node* WasmGraphBuilder::ReturnCallRef(uint32_t sig_index, +Node* WasmGraphBuilder::ReturnCallRef(const wasm::FunctionSig* sig, base::Vector<Node*> args, WasmGraphBuilder::CheckForNull null_check, wasm::WasmCodePosition position) { - return BuildCallRef(sig_index, args, {}, null_check, - IsReturnCall::kReturnCall, position); + return BuildCallRef(sig, args, {}, null_check, IsReturnCall::kReturnCall, + position); } Node* WasmGraphBuilder::ReturnCall(uint32_t index, base::Vector<Node*> args, @@ -3287,7 +3278,7 @@ Node* WasmGraphBuilder::ReturnCall(uint32_t index, base::Vector<Node*> args, Address code = static_cast<Address>(index); args[0] = mcgraph()->RelocatableIntPtrConstant(code, RelocInfo::WASM_CALL); - return BuildWasmReturnCall(sig, args, position, nullptr, kNoRetpoline); + return BuildWasmReturnCall(sig, args, position, nullptr); } Node* WasmGraphBuilder::ReturnCallIndirect(uint32_t table_index, @@ -3416,15 +3407,6 @@ void WasmGraphBuilder::InitInstanceCache( // Load the memory size. instance_cache->mem_size = LOAD_MUTABLE_INSTANCE_FIELD(MemorySize, MachineType::UintPtr()); - - if (untrusted_code_mitigations_) { - // Load the memory mask. - instance_cache->mem_mask = - LOAD_INSTANCE_FIELD(MemoryMask, MachineType::UintPtr()); - } else { - // Explicitly set to nullptr to ensure a SEGV when we try to use it. - instance_cache->mem_mask = nullptr; - } } void WasmGraphBuilder::PrepareInstanceCacheForLoop( @@ -3435,10 +3417,6 @@ void WasmGraphBuilder::PrepareInstanceCacheForLoop( INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); INTRODUCE_PHI(mem_size, MachineType::PointerRepresentation()); - if (untrusted_code_mitigations_) { - INTRODUCE_PHI(mem_mask, MachineType::PointerRepresentation()); - } - #undef INTRODUCE_PHI } @@ -3453,10 +3431,6 @@ void WasmGraphBuilder::NewInstanceCacheMerge(WasmInstanceCacheNodes* to, INTRODUCE_PHI(mem_start, MachineType::PointerRepresentation()); INTRODUCE_PHI(mem_size, MachineRepresentation::kWord32); - if (untrusted_code_mitigations_) { - INTRODUCE_PHI(mem_mask, MachineRepresentation::kWord32); - } - #undef INTRODUCE_PHI } @@ -3467,10 +3441,6 @@ void WasmGraphBuilder::MergeInstanceCacheInto(WasmInstanceCacheNodes* to, merge, to->mem_size, from->mem_size); to->mem_start = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(), merge, to->mem_start, from->mem_start); - if (untrusted_code_mitigations_) { - to->mem_mask = CreateOrMergeIntoPhi(MachineType::PointerRepresentation(), - merge, to->mem_mask, from->mem_mask); - } } Node* WasmGraphBuilder::CreateOrMergeIntoPhi(MachineRepresentation rep, @@ -3623,7 +3593,7 @@ Node* WasmGraphBuilder::BuildCallToRuntimeWithContext(Runtime::FunctionId f, Builtin::kCEntry_Return1_DontSaveFPRegs_ArgvOnStack_NoBuiltinExit; Node* centry_stub = gasm_->LoadFromObject(MachineType::Pointer(), isolate_root, - IsolateData::builtin_slot_offset(centry_id)); + IsolateData::BuiltinSlotOffset(centry_id)); // TODO(titzer): allow arbitrary number of runtime arguments // At the moment we only allow 5 parameters. If more parameters are needed, // increase this constant accordingly. @@ -3839,13 +3809,6 @@ WasmGraphBuilder::BoundsCheckMem(uint8_t access_size, Node* index, // Introduce the actual bounds check. Node* cond = gasm_->UintLessThan(index, effective_size); TrapIfFalse(wasm::kTrapMemOutOfBounds, cond, position); - - if (untrusted_code_mitigations_) { - // In the fallthrough case, condition the index with the memory mask. - Node* mem_mask = instance_cache_->mem_mask; - DCHECK_NOT_NULL(mem_mask); - index = gasm_->WordAnd(index, mem_mask); - } return {index, kDynamicallyChecked}; } @@ -4345,13 +4308,6 @@ Node* WasmGraphBuilder::BuildAsmjsLoadMem(MachineType type, Node* index) { gasm_->UintLessThan(index, mem_size), BranchHint::kTrue); bounds_check.Chain(control()); - if (untrusted_code_mitigations_) { - // Condition the index with the memory mask. - Node* mem_mask = instance_cache_->mem_mask; - DCHECK_NOT_NULL(mem_mask); - index = gasm_->WordAnd(index, mem_mask); - } - Node* load = graph()->NewNode(mcgraph()->machine()->Load(type), mem_start, index, effect(), bounds_check.if_true); SetEffectControl(bounds_check.EffectPhi(load, effect()), bounds_check.merge); @@ -4396,13 +4352,6 @@ Node* WasmGraphBuilder::BuildAsmjsStoreMem(MachineType type, Node* index, BranchHint::kTrue); bounds_check.Chain(control()); - if (untrusted_code_mitigations_) { - // Condition the index with the memory mask. - Node* mem_mask = instance_cache_->mem_mask; - DCHECK_NOT_NULL(mem_mask); - index = gasm_->Word32And(index, mem_mask); - } - index = BuildChangeUint32ToUintPtr(index); const Operator* store_op = mcgraph()->machine()->Store(StoreRepresentation( type.representation(), WriteBarrierKind::kNoWriteBarrier)); @@ -5240,16 +5189,26 @@ Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs, const Operator* (MachineOperatorBuilder::*)(MachineType); using OperatorByRep = const Operator* (MachineOperatorBuilder::*)(MachineRepresentation); + using OperatorByAtomicLoadRep = + const Operator* (MachineOperatorBuilder::*)(AtomicLoadParameters); + using OperatorByAtomicStoreRep = + const Operator* (MachineOperatorBuilder::*)(AtomicStoreParameters); const Type type; const MachineType machine_type; const OperatorByType operator_by_type = nullptr; const OperatorByRep operator_by_rep = nullptr; + const OperatorByAtomicLoadRep operator_by_atomic_load_params = nullptr; + const OperatorByAtomicStoreRep operator_by_atomic_store_rep = nullptr; constexpr AtomicOpInfo(Type t, MachineType m, OperatorByType o) : type(t), machine_type(m), operator_by_type(o) {} constexpr AtomicOpInfo(Type t, MachineType m, OperatorByRep o) : type(t), machine_type(m), operator_by_rep(o) {} + constexpr AtomicOpInfo(Type t, MachineType m, OperatorByAtomicLoadRep o) + : type(t), machine_type(m), operator_by_atomic_load_params(o) {} + constexpr AtomicOpInfo(Type t, MachineType m, OperatorByAtomicStoreRep o) + : type(t), machine_type(m), operator_by_atomic_store_rep(o) {} // Constexpr, hence just a table lookup in most compilers. static constexpr AtomicOpInfo Get(wasm::WasmOpcode opcode) { @@ -5358,11 +5317,21 @@ Node* WasmGraphBuilder::AtomicOp(wasm::WasmOpcode opcode, Node* const* inputs, // {offset} is validated to be within uintptr_t range in {BoundsCheckMem}. uintptr_t capped_offset = static_cast<uintptr_t>(offset); if (info.type != AtomicOpInfo::kSpecial) { - const Operator* op = - info.operator_by_type - ? (mcgraph()->machine()->*info.operator_by_type)(info.machine_type) - : (mcgraph()->machine()->*info.operator_by_rep)( - info.machine_type.representation()); + const Operator* op; + if (info.operator_by_type) { + op = (mcgraph()->machine()->*info.operator_by_type)(info.machine_type); + } else if (info.operator_by_rep) { + op = (mcgraph()->machine()->*info.operator_by_rep)( + info.machine_type.representation()); + } else if (info.operator_by_atomic_load_params) { + op = (mcgraph()->machine()->*info.operator_by_atomic_load_params)( + AtomicLoadParameters(info.machine_type, AtomicMemoryOrder::kSeqCst)); + } else { + op = (mcgraph()->machine()->*info.operator_by_atomic_store_rep)( + AtomicStoreParameters(info.machine_type.representation(), + WriteBarrierKind::kNoWriteBarrier, + AtomicMemoryOrder::kSeqCst)); + } Node* input_nodes[6] = {MemBuffer(capped_offset), index}; int num_actual_inputs = info.type; @@ -5610,13 +5579,17 @@ Node* WasmGraphBuilder::ArrayNewWithRtt(uint32_t array_index, wasm::WasmCodePosition position) { TrapIfFalse(wasm::kTrapArrayTooLarge, gasm_->Uint32LessThanOrEqual( - length, gasm_->Uint32Constant(wasm::kV8MaxWasmArrayLength)), + length, gasm_->Uint32Constant(WasmArray::MaxLength(type))), position); wasm::ValueType element_type = type->element_type(); + // TODO(7748): Consider using gasm_->Allocate(). Builtin stub = ChooseArrayAllocationBuiltin(element_type, initial_value); - Node* a = - gasm_->CallBuiltin(stub, Operator::kEliminatable, rtt, length, - Int32Constant(element_type.element_size_bytes())); + // Do NOT mark this as Operator::kEliminatable, because that would cause the + // Call node to have no control inputs, which means it could get scheduled + // before the check/trap above. + Node* a = gasm_->CallBuiltin( + stub, Operator::kNoDeopt | Operator::kNoThrow, rtt, length, + Int32Constant(element_type.element_size_bytes())); if (initial_value != nullptr) { // TODO(manoskouk): If the loop is ever removed here, we have to update // ArrayNewWithRtt() in graph-builder-interface.cc to not mark the current @@ -5628,8 +5601,6 @@ Node* WasmGraphBuilder::ArrayNewWithRtt(uint32_t array_index, Node* element_size = Int32Constant(element_type.element_size_bytes()); Node* end_offset = gasm_->Int32Add(start_offset, gasm_->Int32Mul(element_size, length)); - // Loops need the graph's end to have been set up. - gasm_->EnsureEnd(); gasm_->Goto(&loop, start_offset); gasm_->Bind(&loop); { @@ -5646,6 +5617,25 @@ Node* WasmGraphBuilder::ArrayNewWithRtt(uint32_t array_index, return a; } +Node* WasmGraphBuilder::ArrayInit(uint32_t array_index, + const wasm::ArrayType* type, Node* rtt, + base::Vector<Node*> elements) { + wasm::ValueType element_type = type->element_type(); + // TODO(7748): Consider using gasm_->Allocate(). + Node* array = + gasm_->CallBuiltin(Builtin::kWasmAllocateArray_Uninitialized, + Operator::kNoDeopt | Operator::kNoThrow, rtt, + Int32Constant(static_cast<int32_t>(elements.size())), + Int32Constant(element_type.element_size_bytes())); + for (int i = 0; i < static_cast<int>(elements.size()); i++) { + Node* offset = + gasm_->WasmArrayElementOffset(Int32Constant(i), element_type); + gasm_->StoreToObject(ObjectAccessForGCStores(element_type), array, offset, + elements[i]); + } + return array; +} + Node* WasmGraphBuilder::RttCanon(uint32_t type_index) { Node* maps_list = LOAD_INSTANCE_FIELD(ManagedObjectMaps, MachineType::TaggedPointer()); @@ -6005,24 +5995,40 @@ Node* WasmGraphBuilder::ArrayLen(Node* array_object, CheckForNull null_check, return gasm_->LoadWasmArrayLength(array_object); } -// TODO(7748): Change {CallBuiltin} to {BuildCCall}. Add an option to copy in a -// loop for small array sizes. To find the length limit, run -// test/mjsunit/wasm/array-copy-benchmark.js. +// TODO(7748): Add an option to copy in a loop for small array sizes. To find +// the length limit, run test/mjsunit/wasm/array-copy-benchmark.js. void WasmGraphBuilder::ArrayCopy(Node* dst_array, Node* dst_index, - Node* src_array, Node* src_index, Node* length, + CheckForNull dst_null_check, Node* src_array, + Node* src_index, CheckForNull src_null_check, + Node* length, wasm::WasmCodePosition position) { - // TODO(7748): Skip null checks when possible. - TrapIfTrue(wasm::kTrapNullDereference, gasm_->WordEqual(dst_array, RefNull()), - position); - TrapIfTrue(wasm::kTrapNullDereference, gasm_->WordEqual(src_array, RefNull()), - position); + if (dst_null_check == kWithNullCheck) { + TrapIfTrue(wasm::kTrapNullDereference, + gasm_->WordEqual(dst_array, RefNull()), position); + } + if (src_null_check == kWithNullCheck) { + TrapIfTrue(wasm::kTrapNullDereference, + gasm_->WordEqual(src_array, RefNull()), position); + } BoundsCheckArrayCopy(dst_array, dst_index, length, position); BoundsCheckArrayCopy(src_array, src_index, length, position); - Operator::Properties copy_properties = - Operator::kIdempotent | Operator::kNoThrow | Operator::kNoDeopt; - // The builtin needs the int parameters first. - gasm_->CallBuiltin(Builtin::kWasmArrayCopy, copy_properties, dst_index, - src_index, length, dst_array, src_array); + + auto skip = gasm_->MakeLabel(); + + gasm_->GotoIf(gasm_->WordEqual(length, Int32Constant(0)), &skip, + BranchHint::kFalse); + + Node* function = + gasm_->ExternalConstant(ExternalReference::wasm_array_copy()); + MachineType arg_types[]{ + MachineType::TaggedPointer(), MachineType::TaggedPointer(), + MachineType::Uint32(), MachineType::TaggedPointer(), + MachineType::Uint32(), MachineType::Uint32()}; + MachineSignature sig(0, 6, arg_types); + BuildCCall(&sig, function, GetInstance(), dst_array, dst_index, src_array, + src_index, length); + gasm_->Goto(&skip); + gasm_->Bind(&skip); } // 1 bit V8 Smi tag, 31 bits V8 Smi shift, 1 bit i31ref high-bit truncation. @@ -6659,8 +6665,7 @@ class WasmWrapperGraphBuilder : public WasmGraphBuilder { // The (cached) call target is the jump table slot for that function. args[0] = BuildLoadCallTargetFromExportedFunctionData(function_data); BuildWasmCall(sig_, base::VectorOf(args), base::VectorOf(rets), - wasm::kNoCodePosition, nullptr, kNoRetpoline, - frame_state); + wasm::kNoCodePosition, nullptr, frame_state); } } @@ -6929,8 +6934,9 @@ class WasmWrapperGraphBuilder : public WasmGraphBuilder { // Convert wasm numbers to JS values. pos = AddArgumentNodes(base::VectorOf(args), pos, wasm_count, sig_); - args[pos++] = undefined_node; // new target - args[pos++] = Int32Constant(wasm_count); // argument count + args[pos++] = undefined_node; // new target + args[pos++] = + Int32Constant(JSParameterCount(wasm_count)); // argument count args[pos++] = function_context; args[pos++] = effect(); args[pos++] = control(); @@ -6957,8 +6963,9 @@ class WasmWrapperGraphBuilder : public WasmGraphBuilder { for (int i = wasm_count; i < expected_arity; ++i) { args[pos++] = undefined_node; } - args[pos++] = undefined_node; // new target - args[pos++] = Int32Constant(wasm_count); // argument count + args[pos++] = undefined_node; // new target + args[pos++] = + Int32Constant(JSParameterCount(wasm_count)); // argument count Node* function_context = gasm_->LoadContextFromJSFunction(callable_node); @@ -6981,7 +6988,8 @@ class WasmWrapperGraphBuilder : public WasmGraphBuilder { args[pos++] = gasm_->GetBuiltinPointerTarget(Builtin::kCall_ReceiverIsAny); args[pos++] = callable_node; - args[pos++] = Int32Constant(wasm_count); // argument count + args[pos++] = + Int32Constant(JSParameterCount(wasm_count)); // argument count args[pos++] = undefined_node; // receiver auto call_descriptor = Linkage::GetStubCallDescriptor( @@ -7162,8 +7170,9 @@ class WasmWrapperGraphBuilder : public WasmGraphBuilder { int pos = 0; args[pos++] = gasm_->GetBuiltinPointerTarget(Builtin::kCall_ReceiverIsAny); args[pos++] = callable; - args[pos++] = Int32Constant(wasm_count); // argument count - args[pos++] = UndefinedValue(); // receiver + args[pos++] = + Int32Constant(JSParameterCount(wasm_count)); // argument count + args[pos++] = UndefinedValue(); // receiver auto call_descriptor = Linkage::GetStubCallDescriptor( graph()->zone(), CallTrampolineDescriptor{}, wasm_count + 1, @@ -7457,7 +7466,7 @@ std::pair<WasmImportCallKind, Handle<JSReceiver>> ResolveWasmImportCall( return std::make_pair(WasmImportCallKind::kUseCallBuiltin, callable); } - if (shared->internal_formal_parameter_count() == + if (shared->internal_formal_parameter_count_without_receiver() == expected_sig->parameter_count()) { return std::make_pair(WasmImportCallKind::kJSFunctionArityMatch, callable); @@ -7538,7 +7547,7 @@ wasm::WasmCompilationResult CompileWasmMathIntrinsic( wasm::CompilationEnv env( nullptr, wasm::kNoBoundsChecks, wasm::RuntimeExceptionSupport::kNoRuntimeExceptionSupport, - wasm::WasmFeatures::All()); + wasm::WasmFeatures::All(), wasm::DynamicTiering::kDisabled); WasmGraphBuilder builder(&env, mcgraph->zone(), mcgraph, sig, source_positions); @@ -7569,11 +7578,12 @@ wasm::WasmCompilationResult CompileWasmMathIntrinsic( call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor); } - wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub( - call_descriptor, mcgraph, CodeKind::WASM_FUNCTION, - wasm::WasmCode::kFunction, debug_name, WasmStubAssemblerOptions(), - source_positions); - return result; + // The code does not call to JS, but conceptually it is an import wrapper, + // hence use {WASM_TO_JS_FUNCTION} here. + // TODO(wasm): Rename this to {WASM_IMPORT_CALL}? + return Pipeline::GenerateCodeForWasmNativeStub( + call_descriptor, mcgraph, CodeKind::WASM_TO_JS_FUNCTION, debug_name, + WasmStubAssemblerOptions(), source_positions); } } // namespace @@ -7623,17 +7633,13 @@ wasm::WasmCompilationResult CompileWasmImportCallWrapper( // Schedule and compile to machine code. CallDescriptor* incoming = - GetWasmCallDescriptor(&zone, sig, WasmGraphBuilder::kNoRetpoline, - WasmCallKind::kWasmImportWrapper); + GetWasmCallDescriptor(&zone, sig, WasmCallKind::kWasmImportWrapper); if (machine->Is32()) { incoming = GetI32WasmCallDescriptor(&zone, incoming); } - wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub( - incoming, mcgraph, CodeKind::WASM_TO_JS_FUNCTION, - wasm::WasmCode::kWasmToJsWrapper, func_name, WasmStubAssemblerOptions(), - source_position_table); - result.kind = wasm::WasmCompilationResult::kWasmToJsWrapper; - return result; + return Pipeline::GenerateCodeForWasmNativeStub( + incoming, mcgraph, CodeKind::WASM_TO_JS_FUNCTION, func_name, + WasmStubAssemblerOptions(), source_position_table); } wasm::WasmCode* CompileWasmCapiCallWrapper(wasm::NativeModule* native_module, @@ -7665,24 +7671,27 @@ wasm::WasmCode* CompileWasmCapiCallWrapper(wasm::NativeModule* native_module, // Run the compiler pipeline to generate machine code. CallDescriptor* call_descriptor = - GetWasmCallDescriptor(&zone, sig, WasmGraphBuilder::kNoRetpoline, - WasmCallKind::kWasmCapiFunction); + GetWasmCallDescriptor(&zone, sig, WasmCallKind::kWasmCapiFunction); if (mcgraph->machine()->Is32()) { call_descriptor = GetI32WasmCallDescriptor(&zone, call_descriptor); } const char* debug_name = "WasmCapiCall"; wasm::WasmCompilationResult result = Pipeline::GenerateCodeForWasmNativeStub( - call_descriptor, mcgraph, CodeKind::WASM_TO_CAPI_FUNCTION, - wasm::WasmCode::kWasmToCapiWrapper, debug_name, + call_descriptor, mcgraph, CodeKind::WASM_TO_CAPI_FUNCTION, debug_name, WasmStubAssemblerOptions(), source_positions); - std::unique_ptr<wasm::WasmCode> wasm_code = native_module->AddCode( - wasm::kAnonymousFuncIndex, result.code_desc, result.frame_slot_count, - result.tagged_parameter_slots, - result.protected_instructions_data.as_vector(), - result.source_positions.as_vector(), wasm::WasmCode::kWasmToCapiWrapper, - wasm::ExecutionTier::kNone, wasm::kNoDebugging); - return native_module->PublishCode(std::move(wasm_code)); + wasm::WasmCode* published_code; + { + wasm::CodeSpaceWriteScope code_space_write_scope(native_module); + std::unique_ptr<wasm::WasmCode> wasm_code = native_module->AddCode( + wasm::kAnonymousFuncIndex, result.code_desc, result.frame_slot_count, + result.tagged_parameter_slots, + result.protected_instructions_data.as_vector(), + result.source_positions.as_vector(), wasm::WasmCode::kWasmToCapiWrapper, + wasm::ExecutionTier::kNone, wasm::kNoDebugging); + published_code = native_module->PublishCode(std::move(wasm_code)); + } + return published_code; } MaybeHandle<Code> CompileWasmToJSWrapper(Isolate* isolate, @@ -7716,8 +7725,7 @@ MaybeHandle<Code> CompileWasmToJSWrapper(Isolate* isolate, // Generate the call descriptor. CallDescriptor* incoming = - GetWasmCallDescriptor(zone.get(), sig, WasmGraphBuilder::kNoRetpoline, - WasmCallKind::kWasmImportWrapper); + GetWasmCallDescriptor(zone.get(), sig, WasmCallKind::kWasmImportWrapper); // Run the compilation job synchronously. std::unique_ptr<OptimizedCompilationJob> job( @@ -7853,7 +7861,7 @@ bool BuildGraphForWasmFunction(wasm::CompilationEnv* env, auto* allocator = wasm::GetWasmEngine()->allocator(); wasm::VoidResult graph_construction_result = wasm::BuildTFGraph( allocator, env->enabled_features, env->module, &builder, detected, - func_body, loop_infos, node_origins, func_index); + func_body, loop_infos, node_origins, func_index, wasm::kRegularFunction); if (graph_construction_result.failed()) { if (FLAG_trace_wasm_compiler) { StdoutStream{} << "Compilation failed: " @@ -7886,8 +7894,9 @@ base::Vector<const char> GetDebugName(Zone* zone, int index) { } // namespace wasm::WasmCompilationResult ExecuteTurbofanWasmCompilation( - wasm::CompilationEnv* env, const wasm::FunctionBody& func_body, - int func_index, Counters* counters, wasm::WasmFeatures* detected) { + wasm::CompilationEnv* env, const wasm::WireBytesStorage* wire_bytes_storage, + const wasm::FunctionBody& func_body, int func_index, Counters* counters, + wasm::WasmFeatures* detected) { TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"), "wasm.CompileTopTier", "func_index", func_index, "body_size", func_body.end - func_body.start); @@ -7936,12 +7945,14 @@ wasm::WasmCompilationResult ExecuteTurbofanWasmCompilation( } if (ContainsSimd(func_body.sig) && !CpuFeatures::SupportsWasmSimd128()) { - call_descriptor = GetI32WasmCallDescriptorForSimd(&zone, call_descriptor); + // Fail compilation if hardware does not support SIMD. + return wasm::WasmCompilationResult{}; } - Pipeline::GenerateCodeForWasmFunction( - &info, mcgraph, call_descriptor, source_positions, node_origins, - func_body, env->module, func_index, &loop_infos); + Pipeline::GenerateCodeForWasmFunction(&info, env, wire_bytes_storage, mcgraph, + call_descriptor, source_positions, + node_origins, func_body, env->module, + func_index, &loop_infos); if (counters) { int zone_bytes = @@ -7997,10 +8008,9 @@ class LinkageLocationAllocator { } // namespace // General code uses the above configuration data. -CallDescriptor* GetWasmCallDescriptor( - Zone* zone, const wasm::FunctionSig* fsig, - WasmGraphBuilder::UseRetpoline use_retpoline, WasmCallKind call_kind, - bool need_frame_state) { +CallDescriptor* GetWasmCallDescriptor(Zone* zone, const wasm::FunctionSig* fsig, + WasmCallKind call_kind, + bool need_frame_state) { // The extra here is to accomodate the instance object as first parameter // and, when specified, the additional callable. bool extra_callable_param = @@ -8078,10 +8088,9 @@ CallDescriptor* GetWasmCallDescriptor( descriptor_kind = CallDescriptor::kCallWasmCapiFunction; } - CallDescriptor::Flags flags = - use_retpoline ? CallDescriptor::kRetpoline - : need_frame_state ? CallDescriptor::kNeedsFrameState - : CallDescriptor::kNoFlags; + CallDescriptor::Flags flags = need_frame_state + ? CallDescriptor::kNeedsFrameState + : CallDescriptor::kNoFlags; return zone->New<CallDescriptor>( // -- descriptor_kind, // kind target_type, // target MachineType |