diff options
Diffstat (limited to 'chromium/v8/src/wasm/baseline')
13 files changed, 582 insertions, 468 deletions
diff --git a/chromium/v8/src/wasm/baseline/arm/liftoff-assembler-arm.h b/chromium/v8/src/wasm/baseline/arm/liftoff-assembler-arm.h index 9379a3b78a2..af969f387e5 100644 --- a/chromium/v8/src/wasm/baseline/arm/liftoff-assembler-arm.h +++ b/chromium/v8/src/wasm/baseline/arm/liftoff-assembler-arm.h @@ -534,16 +534,14 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value, } } -void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, - int size) { - DCHECK_LE(offset, kMaxInt); +void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) { + DCHECK_LE(0, offset); DCHECK_EQ(4, size); ldr(dst, liftoff::GetInstanceOperand()); ldr(dst, MemOperand(dst, offset)); } -void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, - uint32_t offset) { +void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) { LoadFromInstance(dst, offset, kTaggedSize); } @@ -2261,6 +2259,18 @@ void LiftoffAssembler::LoadTransform(LiftoffRegister dst, Register src_addr, NeonMemOperand(actual_src_addr)); vmovl(NeonU32, liftoff::GetSimd128Register(dst), dst.low_fp()); } + } else if (transform == LoadTransformationKind::kZeroExtend) { + Simd128Register dest = liftoff::GetSimd128Register(dst); + if (memtype == MachineType::Int32()) { + vmov(dest, 0); + vld1s(Neon32, NeonListOperand(dst.low_fp()), 0, + NeonMemOperand(actual_src_addr)); + } else { + DCHECK_EQ(MachineType::Int64(), memtype); + vmov(dest.high(), 0); + vld1(Neon64, NeonListOperand(dest.low()), + NeonMemOperand(actual_src_addr)); + } } else { DCHECK_EQ(LoadTransformationKind::kSplat, transform); if (memtype == MachineType::Int8()) { @@ -2923,6 +2933,23 @@ void LiftoffAssembler::emit_i32x4_max_u(LiftoffRegister dst, liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } +void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + QwNeonRegister dest = liftoff::GetSimd128Register(dst); + QwNeonRegister left = liftoff::GetSimd128Register(lhs); + QwNeonRegister right = liftoff::GetSimd128Register(rhs); + + UseScratchRegisterScope temps(this); + Simd128Register scratch = temps.AcquireQ(); + + vmull(NeonS16, scratch, left.low(), right.low()); + vpadd(Neon32, dest.low(), scratch.low(), scratch.high()); + + vmull(NeonS16, scratch, left.high(), right.high()); + vpadd(Neon32, dest.high(), scratch.low(), scratch.high()); +} + void LiftoffAssembler::emit_i16x8_splat(LiftoffRegister dst, LiftoffRegister src) { vdup(Neon16, liftoff::GetSimd128Register(dst), src.gp()); @@ -3017,9 +3044,9 @@ void LiftoffAssembler::emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } -void LiftoffAssembler::emit_i16x8_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { vqadd(NeonS16, liftoff::GetSimd128Register(dst), liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } @@ -3030,16 +3057,16 @@ void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } -void LiftoffAssembler::emit_i16x8_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { vqsub(NeonS16, liftoff::GetSimd128Register(dst), liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } -void LiftoffAssembler::emit_i16x8_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { vqsub(NeonU16, liftoff::GetSimd128Register(dst), liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } @@ -3050,9 +3077,9 @@ void LiftoffAssembler::emit_i16x8_mul(LiftoffRegister dst, LiftoffRegister lhs, liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } -void LiftoffAssembler::emit_i16x8_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { vqadd(NeonU16, liftoff::GetSimd128Register(dst), liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } @@ -3135,7 +3162,6 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, } int table_size = src1 == src2 ? 2 : 4; - uint32_t mask = table_size == 2 ? 0x0F0F0F0F : 0x1F1F1F1F; int scratch_s_base = scratch.code() * 4; for (int j = 0; j < 4; j++) { @@ -3143,11 +3169,9 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, for (int i = 3; i >= 0; i--) { imm = (imm << 8) | shuffle[j * 4 + i]; } - uint32_t four_lanes = imm; + DCHECK_EQ(0, imm & (table_size == 2 ? 0xF0F0F0F0 : 0xE0E0E0E0)); // Ensure indices are in [0,15] if table_size is 2, or [0,31] if 4. - four_lanes &= mask; - vmov(SwVfpRegister::from_code(scratch_s_base + j), - Float32::FromBits(four_lanes)); + vmov(SwVfpRegister::from_code(scratch_s_base + j), Float32::FromBits(imm)); } DwVfpRegister table_base = src1.low(); @@ -3279,9 +3303,9 @@ void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } -void LiftoffAssembler::emit_i8x16_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { vqadd(NeonS8, liftoff::GetSimd128Register(dst), liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } @@ -3292,16 +3316,16 @@ void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } -void LiftoffAssembler::emit_i8x16_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { vqsub(NeonS8, liftoff::GetSimd128Register(dst), liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } -void LiftoffAssembler::emit_i8x16_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { vqsub(NeonU8, liftoff::GetSimd128Register(dst), liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } @@ -3312,9 +3336,9 @@ void LiftoffAssembler::emit_i8x16_mul(LiftoffRegister dst, LiftoffRegister lhs, liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } -void LiftoffAssembler::emit_i8x16_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { vqadd(NeonU8, liftoff::GetSimd128Register(dst), liftoff::GetSimd128Register(lhs), liftoff::GetSimd128Register(rhs)); } diff --git a/chromium/v8/src/wasm/baseline/arm64/liftoff-assembler-arm64.h b/chromium/v8/src/wasm/baseline/arm64/liftoff-assembler-arm64.h index 161ada8526a..402f0d2e84f 100644 --- a/chromium/v8/src/wasm/baseline/arm64/liftoff-assembler-arm64.h +++ b/chromium/v8/src/wasm/baseline/arm64/liftoff-assembler-arm64.h @@ -186,25 +186,36 @@ int LiftoffAssembler::PrepareStackFrame() { void LiftoffAssembler::PrepareTailCall(int num_callee_stack_params, int stack_param_delta) { UseScratchRegisterScope temps(this); - Register scratch = temps.AcquireX(); + temps.Exclude(x16, x17); + + // This is the previous stack pointer value (before we push the lr and the + // fp). We need to keep it to autenticate the lr and adjust the new stack + // pointer afterwards. + Add(x16, fp, 16); + + // Load the fp and lr of the old frame, they will be pushed in the new frame + // during the actual call. +#ifdef V8_ENABLE_CONTROL_FLOW_INTEGRITY + Ldp(fp, x17, MemOperand(fp)); + Autib1716(); + Mov(lr, x17); +#else + Ldp(fp, lr, MemOperand(fp)); +#endif - // Push the return address and frame pointer to complete the stack frame. - sub(sp, sp, 16); - ldr(scratch, MemOperand(fp, 8)); - Poke(scratch, 8); - ldr(scratch, MemOperand(fp, 0)); - Poke(scratch, 0); + temps.Include(x17); + + Register scratch = temps.AcquireX(); - // Shift the whole frame upwards. - int slot_count = num_callee_stack_params + 2; + // Shift the whole frame upwards, except for fp and lr. + int slot_count = num_callee_stack_params; for (int i = slot_count - 1; i >= 0; --i) { ldr(scratch, MemOperand(sp, i * 8)); - str(scratch, MemOperand(fp, (i - stack_param_delta) * 8)); + str(scratch, MemOperand(x16, (i - stack_param_delta) * 8)); } - // Set the new stack and frame pointer. - Sub(sp, fp, stack_param_delta * 8); - Pop<kAuthLR>(fp, lr); + // Set the new stack pointer. + Sub(sp, x16, stack_param_delta * 8); } void LiftoffAssembler::PatchPrepareStackFrame(int offset, int frame_size) { @@ -302,9 +313,8 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value, } } -void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, - int size) { - DCHECK_LE(offset, kMaxInt); +void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) { + DCHECK_LE(0, offset); Ldr(dst, liftoff::GetInstanceOperand()); DCHECK(size == 4 || size == 8); if (size == 4) { @@ -314,9 +324,8 @@ void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, } } -void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, - uint32_t offset) { - DCHECK_LE(offset, kMaxInt); +void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) { + DCHECK_LE(0, offset); Ldr(dst, liftoff::GetInstanceOperand()); LoadTaggedPointerField(dst, MemOperand(dst, offset)); } @@ -1496,6 +1505,13 @@ void LiftoffAssembler::LoadTransform(LiftoffRegister dst, Register src_addr, Ldr(dst.fp().D(), src_op); Uxtl(dst.fp().V2D(), dst.fp().V2S()); } + } else if (transform == LoadTransformationKind::kZeroExtend) { + if (memtype == MachineType::Int32()) { + Ldr(dst.fp().S(), src_op); + } else { + DCHECK_EQ(MachineType::Int64(), memtype); + Ldr(dst.fp().D(), src_op); + } } else { // ld1r only allows no offset or post-index, so emit an add. DCHECK_EQ(LoadTransformationKind::kSplat, transform); @@ -2004,6 +2020,17 @@ void LiftoffAssembler::emit_i32x4_max_u(LiftoffRegister dst, Umax(dst.fp().V4S(), lhs.fp().V4S(), rhs.fp().V4S()); } +void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + UseScratchRegisterScope scope(this); + VRegister tmp1 = scope.AcquireV(kFormat4S); + VRegister tmp2 = scope.AcquireV(kFormat4S); + Smull(tmp1, lhs.fp().V4H(), rhs.fp().V4H()); + Smull2(tmp2, lhs.fp().V8H(), rhs.fp().V8H()); + Addp(dst.fp().V4S(), tmp1, tmp2); +} + void LiftoffAssembler::emit_i16x8_splat(LiftoffRegister dst, LiftoffRegister src) { Dup(dst.fp().V8H(), src.gp().W()); @@ -2106,9 +2133,9 @@ void LiftoffAssembler::emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, Add(dst.fp().V8H(), lhs.fp().V8H(), rhs.fp().V8H()); } -void LiftoffAssembler::emit_i16x8_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { Sqadd(dst.fp().V8H(), lhs.fp().V8H(), rhs.fp().V8H()); } @@ -2117,15 +2144,15 @@ void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, Sub(dst.fp().V8H(), lhs.fp().V8H(), rhs.fp().V8H()); } -void LiftoffAssembler::emit_i16x8_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { Sqsub(dst.fp().V8H(), lhs.fp().V8H(), rhs.fp().V8H()); } -void LiftoffAssembler::emit_i16x8_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { Uqsub(dst.fp().V8H(), lhs.fp().V8H(), rhs.fp().V8H()); } @@ -2134,9 +2161,9 @@ void LiftoffAssembler::emit_i16x8_mul(LiftoffRegister dst, LiftoffRegister lhs, Mul(dst.fp().V8H(), lhs.fp().V8H(), rhs.fp().V8H()); } -void LiftoffAssembler::emit_i16x8_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { Uqadd(dst.fp().V8H(), lhs.fp().V8H(), rhs.fp().V8H()); } @@ -2188,12 +2215,13 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, Mov(src2.Q(), rhs.fp().Q()); } - uint8_t mask = lhs == rhs ? 0x0F : 0x1F; int64_t imms[2] = {0, 0}; for (int i = 7; i >= 0; i--) { - imms[0] = (imms[0] << 8) | (shuffle[i] & mask); - imms[1] = (imms[1] << 8) | (shuffle[i + 8] & mask); + imms[0] = (imms[0] << 8) | (shuffle[i]); + imms[1] = (imms[1] << 8) | (shuffle[i + 8]); } + DCHECK_EQ(0, (imms[0] | imms[1]) & + (lhs == rhs ? 0xF0F0F0F0F0F0F0F0 : 0xE0E0E0E0E0E0E0E0)); Movi(temp.V16B(), imms[1], imms[0]); @@ -2308,9 +2336,9 @@ void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, Add(dst.fp().V16B(), lhs.fp().V16B(), rhs.fp().V16B()); } -void LiftoffAssembler::emit_i8x16_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { Sqadd(dst.fp().V16B(), lhs.fp().V16B(), rhs.fp().V16B()); } @@ -2319,15 +2347,15 @@ void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, Sub(dst.fp().V16B(), lhs.fp().V16B(), rhs.fp().V16B()); } -void LiftoffAssembler::emit_i8x16_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { Sqsub(dst.fp().V16B(), lhs.fp().V16B(), rhs.fp().V16B()); } -void LiftoffAssembler::emit_i8x16_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { Uqsub(dst.fp().V16B(), lhs.fp().V16B(), rhs.fp().V16B()); } @@ -2336,9 +2364,9 @@ void LiftoffAssembler::emit_i8x16_mul(LiftoffRegister dst, LiftoffRegister lhs, Mul(dst.fp().V16B(), lhs.fp().V16B(), rhs.fp().V16B()); } -void LiftoffAssembler::emit_i8x16_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { Uqadd(dst.fp().V16B(), lhs.fp().V16B(), rhs.fp().V16B()); } diff --git a/chromium/v8/src/wasm/baseline/ia32/liftoff-assembler-ia32.h b/chromium/v8/src/wasm/baseline/ia32/liftoff-assembler-ia32.h index 9bfad9313b9..5e640093c43 100644 --- a/chromium/v8/src/wasm/baseline/ia32/liftoff-assembler-ia32.h +++ b/chromium/v8/src/wasm/baseline/ia32/liftoff-assembler-ia32.h @@ -261,16 +261,14 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value, } } -void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, - int size) { - DCHECK_LE(offset, kMaxInt); +void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) { + DCHECK_LE(0, offset); mov(dst, liftoff::GetInstanceOperand()); DCHECK_EQ(4, size); mov(dst, Operand(dst, offset)); } -void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, - uint32_t offset) { +void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) { LoadFromInstance(dst, offset, kTaggedSize); } @@ -2653,6 +2651,13 @@ void LiftoffAssembler::LoadTransform(LiftoffRegister dst, Register src_addr, } else if (memtype == MachineType::Uint32()) { Pmovzxdq(dst.fp(), src_op); } + } else if (transform == LoadTransformationKind::kZeroExtend) { + if (memtype == MachineType::Int32()) { + movss(dst.fp(), src_op); + } else { + DCHECK_EQ(MachineType::Int64(), memtype); + movsd(dst.fp(), src_op); + } } else { DCHECK_EQ(LoadTransformationKind::kSplat, transform); if (memtype == MachineType::Int8()) { @@ -2688,15 +2693,7 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, for (int i = 3; i >= 0; i--) { push_imm32(imms[i]); } - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpshufb(dst.fp(), lhs.fp(), Operand(esp, 0)); - } else { - if (dst != lhs) { - movups(dst.fp(), lhs.fp()); - } - pshufb(dst.fp(), Operand(esp, 0)); - } + Pshufb(dst.fp(), lhs.fp(), Operand(esp, 0)); mov(esp, tmp.gp()); return; } @@ -2711,7 +2708,7 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, } push(Immediate(mask)); } - Pshufb(liftoff::kScratchDoubleReg, Operand(esp, 0)); + Pshufb(liftoff::kScratchDoubleReg, lhs.fp(), Operand(esp, 0)); for (int i = 3; i >= 0; i--) { uint32_t mask = 0; @@ -2722,10 +2719,7 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, } push(Immediate(mask)); } - if (dst.fp() != rhs.fp()) { - movups(dst.fp(), rhs.fp()); - } - Pshufb(dst.fp(), Operand(esp, 0)); + Pshufb(dst.fp(), rhs.fp(), Operand(esp, 0)); Por(dst.fp(), liftoff::kScratchDoubleReg); mov(esp, tmp.gp()); } @@ -2739,10 +2733,7 @@ void LiftoffAssembler::emit_i8x16_swizzle(LiftoffRegister dst, TurboAssembler::Move(mask, uint32_t{0x70707070}); Pshufd(mask, mask, uint8_t{0x0}); Paddusb(mask, rhs.fp()); - if (lhs != dst) { - Movaps(dst.fp(), lhs.fp()); - } - Pshufb(dst.fp(), mask); + Pshufb(dst.fp(), lhs.fp(), mask); } void LiftoffAssembler::emit_i8x16_splat(LiftoffRegister dst, @@ -3199,16 +3190,16 @@ void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i8x16_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdCommutativeBinOp<&Assembler::vpaddsb, &Assembler::paddsb>( this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i8x16_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdCommutativeBinOp<&Assembler::vpaddusb, &Assembler::paddusb>( this, dst, lhs, rhs); } @@ -3219,16 +3210,16 @@ void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i8x16_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdNonCommutativeBinOp<&Assembler::vpsubsb, &Assembler::psubsb>( this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i8x16_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdNonCommutativeBinOp<&Assembler::vpsubusb, &Assembler::psubusb>(this, dst, lhs, rhs); @@ -3397,16 +3388,16 @@ void LiftoffAssembler::emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i16x8_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdCommutativeBinOp<&Assembler::vpaddsw, &Assembler::paddsw>( this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i16x8_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdCommutativeBinOp<&Assembler::vpaddusw, &Assembler::paddusw>( this, dst, lhs, rhs); } @@ -3417,16 +3408,16 @@ void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i16x8_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdNonCommutativeBinOp<&Assembler::vpsubsw, &Assembler::psubsw>( this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i16x8_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdNonCommutativeBinOp<&Assembler::vpsubusw, &Assembler::psubusw>(this, dst, lhs, rhs); @@ -3576,6 +3567,13 @@ void LiftoffAssembler::emit_i32x4_max_u(LiftoffRegister dst, this, dst, lhs, rhs, base::Optional<CpuFeature>(SSE4_1)); } +void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + liftoff::EmitSimdCommutativeBinOp<&Assembler::vpmaddwd, &Assembler::pmaddwd>( + this, dst, lhs, rhs); +} + void LiftoffAssembler::emit_i64x2_neg(LiftoffRegister dst, LiftoffRegister src) { DoubleRegister reg = diff --git a/chromium/v8/src/wasm/baseline/liftoff-assembler.cc b/chromium/v8/src/wasm/baseline/liftoff-assembler.cc index e219025e539..dea5221ac63 100644 --- a/chromium/v8/src/wasm/baseline/liftoff-assembler.cc +++ b/chromium/v8/src/wasm/baseline/liftoff-assembler.cc @@ -37,6 +37,7 @@ class StackTransferRecipe { struct RegisterLoad { enum LoadKind : uint8_t { + kNop, // no-op, used for high fp of a fp pair. kConstant, // load a constant value into a register. kStack, // fill a register from a stack slot. kLowHalfStack, // fill a register from the low half of a stack slot. @@ -63,6 +64,10 @@ class StackTransferRecipe { return {half == kLowWord ? kLowHalfStack : kHighHalfStack, kWasmI32, offset}; } + static RegisterLoad Nop() { + // ValueType does not matter. + return {kNop, kWasmI32, 0}; + } private: RegisterLoad(LoadKind kind, ValueType type, int32_t value) @@ -71,6 +76,8 @@ class StackTransferRecipe { public: explicit StackTransferRecipe(LiftoffAssembler* wasm_asm) : asm_(wasm_asm) {} + StackTransferRecipe(const StackTransferRecipe&) = delete; + StackTransferRecipe& operator=(const StackTransferRecipe&) = delete; ~StackTransferRecipe() { Execute(); } void Execute() { @@ -217,11 +224,11 @@ class StackTransferRecipe { RegisterLoad::HalfStack(stack_offset, kHighWord); } else if (dst.is_fp_pair()) { DCHECK_EQ(kWasmS128, type); - // load_dst_regs_.set above will set both low and high fp regs. - // But unlike gp_pair, we load a kWasm128 in one go in ExecuteLoads. - // So unset the top fp register to skip loading it. - load_dst_regs_.clear(dst.high()); + // Only need register_load for low_gp since we load 128 bits at one go. + // Both low and high need to be set in load_dst_regs_ but when iterating + // over it, both low and high will be cleared, so we won't load twice. *register_load(dst.low()) = RegisterLoad::Stack(stack_offset, type); + *register_load(dst.high()) = RegisterLoad::Nop(); } else { *register_load(dst) = RegisterLoad::Stack(stack_offset, type); } @@ -318,6 +325,8 @@ class StackTransferRecipe { for (LiftoffRegister dst : load_dst_regs_) { RegisterLoad* load = register_load(dst); switch (load->kind) { + case RegisterLoad::kNop: + break; case RegisterLoad::kConstant: asm_->LoadConstant(dst, load->type == kWasmI64 ? WasmValue(int64_t{load->value}) @@ -343,8 +352,6 @@ class StackTransferRecipe { } load_dst_regs_ = {}; } - - DISALLOW_COPY_AND_ASSIGN(StackTransferRecipe); }; class RegisterReuseMap { @@ -519,9 +526,7 @@ int LiftoffAssembler::GetTotalFrameSlotCountForGC() const { namespace { -constexpr AssemblerOptions DefaultLiftoffOptions() { - return AssemblerOptions{}; -} +AssemblerOptions DefaultLiftoffOptions() { return AssemblerOptions{}; } } // namespace diff --git a/chromium/v8/src/wasm/baseline/liftoff-assembler.h b/chromium/v8/src/wasm/baseline/liftoff-assembler.h index e2bd99841fa..895abbbbb45 100644 --- a/chromium/v8/src/wasm/baseline/liftoff-assembler.h +++ b/chromium/v8/src/wasm/baseline/liftoff-assembler.h @@ -478,8 +478,8 @@ class LiftoffAssembler : public TurboAssembler { inline void LoadConstant(LiftoffRegister, WasmValue, RelocInfo::Mode rmode = RelocInfo::NONE); - inline void LoadFromInstance(Register dst, uint32_t offset, int size); - inline void LoadTaggedPointerFromInstance(Register dst, uint32_t offset); + inline void LoadFromInstance(Register dst, int offset, int size); + inline void LoadTaggedPointerFromInstance(Register dst, int offset); inline void SpillInstance(Register instance); inline void FillInstanceInto(Register dst); inline void LoadTaggedPointer(Register dst, Register src_addr, @@ -675,6 +675,15 @@ class LiftoffAssembler : public TurboAssembler { } } + inline void emit_ptrsize_zeroextend_i32(Register dst, Register src) { + if (kSystemPointerSize == 8) { + emit_type_conversion(kExprI64UConvertI32, LiftoffRegister(dst), + LiftoffRegister(src)); + } else if (dst != src) { + Move(dst, src, kWasmI32); + } + } + // f32 binops. inline void emit_f32_add(DoubleRegister dst, DoubleRegister lhs, DoubleRegister rhs); @@ -852,20 +861,16 @@ class LiftoffAssembler : public TurboAssembler { int32_t rhs); inline void emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs); - inline void emit_i8x16_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs); - inline void emit_i8x16_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs); + inline void emit_i8x16_add_sat_s(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); + inline void emit_i8x16_add_sat_u(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); inline void emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs); - inline void emit_i8x16_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs); - inline void emit_i8x16_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs); + inline void emit_i8x16_sub_sat_s(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); + inline void emit_i8x16_sub_sat_u(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); inline void emit_i8x16_mul(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs); inline void emit_i8x16_min_s(LiftoffRegister dst, LiftoffRegister lhs, @@ -894,20 +899,16 @@ class LiftoffAssembler : public TurboAssembler { int32_t rhs); inline void emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs); - inline void emit_i16x8_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs); - inline void emit_i16x8_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs); + inline void emit_i16x8_add_sat_s(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); + inline void emit_i16x8_add_sat_u(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); inline void emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs); - inline void emit_i16x8_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs); - inline void emit_i16x8_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs); + inline void emit_i16x8_sub_sat_s(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); + inline void emit_i16x8_sub_sat_u(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); inline void emit_i16x8_mul(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs); inline void emit_i16x8_min_s(LiftoffRegister dst, LiftoffRegister lhs, @@ -948,6 +949,8 @@ class LiftoffAssembler : public TurboAssembler { LiftoffRegister rhs); inline void emit_i32x4_max_u(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs); + inline void emit_i32x4_dot_i16x8_s(LiftoffRegister dst, LiftoffRegister lhs, + LiftoffRegister rhs); inline void emit_i64x2_neg(LiftoffRegister dst, LiftoffRegister src); inline void emit_i64x2_shl(LiftoffRegister dst, LiftoffRegister lhs, LiftoffRegister rhs); @@ -1302,6 +1305,8 @@ void LiftoffAssembler::emit_i64_xori(LiftoffRegister dst, LiftoffRegister lhs, class LiftoffStackSlots { public: explicit LiftoffStackSlots(LiftoffAssembler* wasm_asm) : asm_(wasm_asm) {} + LiftoffStackSlots(const LiftoffStackSlots&) = delete; + LiftoffStackSlots& operator=(const LiftoffStackSlots&) = delete; void Add(const LiftoffAssembler::VarState& src, uint32_t src_offset, RegPairHalf half) { @@ -1328,8 +1333,6 @@ class LiftoffStackSlots { base::SmallVector<Slot, 8> slots_; LiftoffAssembler* const asm_; - - DISALLOW_COPY_AND_ASSIGN(LiftoffStackSlots); }; } // namespace wasm diff --git a/chromium/v8/src/wasm/baseline/liftoff-compiler.cc b/chromium/v8/src/wasm/baseline/liftoff-compiler.cc index c9625b06d8a..1ead202ea07 100644 --- a/chromium/v8/src/wasm/baseline/liftoff-compiler.cc +++ b/chromium/v8/src/wasm/baseline/liftoff-compiler.cc @@ -258,9 +258,9 @@ class DebugSideTableBuilder { class LiftoffCompiler { public: // TODO(clemensb): Make this a template parameter. - static constexpr Decoder::ValidateFlag validate = Decoder::kValidate; + static constexpr Decoder::ValidateFlag validate = Decoder::kBooleanValidation; - using Value = ValueBase; + using Value = ValueBase<validate>; static constexpr auto kI32 = ValueType::kI32; static constexpr auto kI64 = ValueType::kI64; @@ -273,7 +273,7 @@ class LiftoffCompiler { LiftoffAssembler::CacheState state; }; - struct Control : public ControlBase<Value> { + struct Control : public ControlBase<Value, validate> { std::unique_ptr<ElseState> else_state; LiftoffAssembler::CacheState label_state; MovableLabel label; @@ -557,7 +557,7 @@ class LiftoffCompiler { void StartFunctionBody(FullDecoder* decoder, Control* block) { for (uint32_t i = 0; i < __ num_locals(); ++i) { if (!CheckSupportedType(decoder, - FLAG_liftoff_extern_ref + FLAG_experimental_liftoff_extern_ref ? kSupportedTypes : kSupportedTypesWithoutRefs, __ local_type(i), "param")) @@ -621,7 +621,7 @@ class LiftoffCompiler { } } - if (FLAG_liftoff_extern_ref) { + if (FLAG_experimental_liftoff_extern_ref) { // Initialize all reference type locals with ref.null. for (uint32_t param_idx = num_params; param_idx < __ num_locals(); ++param_idx) { @@ -690,30 +690,6 @@ class LiftoffCompiler { } if (FLAG_trace_wasm) TraceFunctionEntry(decoder); - - // If we are generating debug code, do check the "hook on function call" - // flag. If set, trigger a break. - if (V8_UNLIKELY(for_debugging_)) { - // If there is a breakpoint set on the first instruction (== start of the - // function), then skip the check for "hook on function call", since we - // will unconditionally break there anyway. - bool has_breakpoint = next_breakpoint_ptr_ != nullptr && - (*next_breakpoint_ptr_ == 0 || - *next_breakpoint_ptr_ == decoder->position()); - if (!has_breakpoint) { - DEBUG_CODE_COMMENT("check hook on function call"); - Register flag = __ GetUnusedRegister(kGpReg, {}).gp(); - LOAD_INSTANCE_FIELD(flag, HookOnFunctionCallAddress, - kSystemPointerSize); - Label no_break; - __ Load(LiftoffRegister{flag}, flag, no_reg, 0, LoadType::kI32Load8U, - {}); - // Unary "equal" means "equals zero". - __ emit_cond_jump(kEqual, &no_break, kWasmI32, flag); - EmitBreakpoint(decoder); - __ bind(&no_break); - } - } } void GenerateOutOfLineCode(OutOfLineCode* ool) { @@ -799,14 +775,14 @@ class LiftoffCompiler { } V8_NOINLINE void EmitDebuggingInfo(FullDecoder* decoder, WasmOpcode opcode) { - DCHECK(V8_UNLIKELY(for_debugging_)); + DCHECK(for_debugging_); + if (!WasmOpcodes::IsBreakable(opcode)) return; + bool has_breakpoint = false; if (next_breakpoint_ptr_) { if (*next_breakpoint_ptr_ == 0) { // A single breakpoint at offset 0 indicates stepping. DCHECK_EQ(next_breakpoint_ptr_ + 1, next_breakpoint_end_); - if (WasmOpcodes::IsBreakable(opcode)) { - EmitBreakpoint(decoder); - } + has_breakpoint = true; } else { while (next_breakpoint_ptr_ != next_breakpoint_end_ && *next_breakpoint_ptr_ < decoder->position()) { @@ -816,18 +792,34 @@ class LiftoffCompiler { if (next_breakpoint_ptr_ == next_breakpoint_end_) { next_breakpoint_ptr_ = next_breakpoint_end_ = nullptr; } else if (*next_breakpoint_ptr_ == decoder->position()) { - DCHECK(WasmOpcodes::IsBreakable(opcode)); - EmitBreakpoint(decoder); + has_breakpoint = true; } } } - if (dead_breakpoint_ == decoder->position()) { + if (has_breakpoint) { + EmitBreakpoint(decoder); + // Once we emitted a breakpoint, we don't need to check the "hook on + // function call" any more. + checked_hook_on_function_call_ = true; + } else if (!checked_hook_on_function_call_) { + checked_hook_on_function_call_ = true; + // Check the "hook on function call" flag. If set, trigger a break. + DEBUG_CODE_COMMENT("check hook on function call"); + Register flag = __ GetUnusedRegister(kGpReg, {}).gp(); + LOAD_INSTANCE_FIELD(flag, HookOnFunctionCallAddress, kSystemPointerSize); + Label no_break; + __ Load(LiftoffRegister{flag}, flag, no_reg, 0, LoadType::kI32Load8U, {}); + // Unary "equal" means "equals zero". + __ emit_cond_jump(kEqual, &no_break, kWasmI32, flag); + EmitBreakpoint(decoder); + __ bind(&no_break); + } else if (dead_breakpoint_ == decoder->position()) { DCHECK(!next_breakpoint_ptr_ || *next_breakpoint_ptr_ != dead_breakpoint_); // The top frame is paused at this position, but the breakpoint was - // removed. Adding a dead breakpoint here ensures that the source position - // exists, and that the offset to the return address is the same as in the - // old code. + // removed. Adding a dead breakpoint here ensures that the source + // position exists, and that the offset to the return address is the + // same as in the old code. Label cont; __ emit_jump(&cont); EmitBreakpoint(decoder); @@ -843,7 +835,8 @@ class LiftoffCompiler { #ifdef DEBUG SLOW_DCHECK(__ ValidateCacheState()); if (WasmOpcodes::IsPrefixOpcode(opcode)) { - opcode = decoder->read_prefixed_opcode<Decoder::kValidate>(decoder->pc()); + opcode = decoder->read_prefixed_opcode<Decoder::kFullValidation>( + decoder->pc()); } DEBUG_CODE_COMMENT(WasmOpcodes::OpcodeName(opcode)); #endif @@ -1635,7 +1628,7 @@ class LiftoffCompiler { } void RefNull(FullDecoder* decoder, ValueType type, Value*) { - if (!FLAG_liftoff_extern_ref) { + if (!FLAG_experimental_liftoff_extern_ref) { unsupported(decoder, kRefTypes, "ref_null"); return; } @@ -1818,7 +1811,7 @@ class LiftoffCompiler { const GlobalIndexImmediate<validate>& imm) { const auto* global = &env_->module->globals[imm.index]; if (!CheckSupportedType(decoder, - FLAG_liftoff_extern_ref + FLAG_experimental_liftoff_extern_ref ? kSupportedTypes : kSupportedTypesWithoutRefs, global->type, "global")) { @@ -1857,7 +1850,7 @@ class LiftoffCompiler { const GlobalIndexImmediate<validate>& imm) { auto* global = &env_->module->globals[imm.index]; if (!CheckSupportedType(decoder, - FLAG_liftoff_extern_ref + FLAG_experimental_liftoff_extern_ref ? kSupportedTypes : kSupportedTypesWithoutRefs, global->type, "global")) { @@ -2187,25 +2180,36 @@ class LiftoffCompiler { __ SpillAllRegisters(); LiftoffRegList pinned = LiftoffRegList::ForRegs(index); - // Get one register for computing the address (offset + index). - LiftoffRegister address = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); - // Compute offset+index in address. - __ LoadConstant(address, WasmValue(offset)); - __ emit_i32_add(address.gp(), address.gp(), index); + // Get one register for computing the effective offset (offset + index). + LiftoffRegister effective_offset = + pinned.set(__ GetUnusedRegister(kGpReg, pinned)); + __ LoadConstant(effective_offset, WasmValue(offset)); + __ emit_i32_add(effective_offset.gp(), effective_offset.gp(), index); // Get a register to hold the stack slot for MemoryTracingInfo. LiftoffRegister info = pinned.set(__ GetUnusedRegister(kGpReg, pinned)); // Allocate stack slot for MemoryTracingInfo. __ AllocateStackSlot(info.gp(), sizeof(MemoryTracingInfo)); + // Reuse the {effective_offset} register for all information to be stored in + // the MemoryTracingInfo struct. + LiftoffRegister data = effective_offset; + // Now store all information into the MemoryTracingInfo struct. - __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, address), address, - StoreType::kI32Store, pinned); - __ LoadConstant(address, WasmValue(is_store ? 1 : 0)); - __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, is_store), address, + if (kSystemPointerSize == 8) { + // Zero-extend the effective offset to u64. + CHECK(__ emit_type_conversion(kExprI64UConvertI32, data, effective_offset, + nullptr)); + } + __ Store( + info.gp(), no_reg, offsetof(MemoryTracingInfo, offset), data, + kSystemPointerSize == 8 ? StoreType::kI64Store : StoreType::kI32Store, + pinned); + __ LoadConstant(data, WasmValue(is_store ? 1 : 0)); + __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, is_store), data, StoreType::kI32Store8, pinned); - __ LoadConstant(address, WasmValue(static_cast<int>(rep))); - __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, mem_rep), address, + __ LoadConstant(data, WasmValue(static_cast<int>(rep))); + __ Store(info.gp(), no_reg, offsetof(MemoryTracingInfo, mem_rep), data, StoreType::kI32Store8, pinned); WasmTraceMemoryDescriptor descriptor; @@ -2290,15 +2294,11 @@ class LiftoffCompiler { return; } - if (transform == LoadTransformationKind::kZeroExtend) { - unsupported(decoder, kSimd, "prototyping s128 load zero extend"); - return; - } - LiftoffRegList pinned; Register index = pinned.set(__ PopToRegister()).gp(); - // For load splats, LoadType is the size of the load, and for load - // extends, LoadType is the size of the lane, and it always loads 8 bytes. + // For load splats and load zero, LoadType is the size of the load, and for + // load extends, LoadType is the size of the lane, and it always loads 8 + // bytes. uint32_t access_size = transform == LoadTransformationKind::kExtend ? 8 : type.size(); if (BoundsCheckMem(decoder, access_size, imm.offset, index, pinned, @@ -2333,6 +2333,12 @@ class LiftoffCompiler { } } + void LoadLane(FullDecoder* decoder, LoadType type, const Value& value, + const Value& index, const MemoryAccessImmediate<validate>& imm, + const uint8_t laneidx, Value* result) { + unsupported(decoder, kSimd, "simd load lane"); + } + void StoreMem(FullDecoder* decoder, StoreType type, const MemoryAccessImmediate<validate>& imm, const Value& index_val, const Value& value_val) { @@ -2367,6 +2373,12 @@ class LiftoffCompiler { } } + void StoreLane(FullDecoder* decoder, StoreType type, + const MemoryAccessImmediate<validate>& imm, const Value& index, + const Value& value, const uint8_t laneidx) { + unsupported(decoder, kSimd, "simd load lane"); + } + void CurrentMemoryPages(FullDecoder* decoder, Value* result) { Register mem_size = __ GetUnusedRegister(kGpReg, {}).gp(); LOAD_INSTANCE_FIELD(mem_size, MemorySize, kSystemPointerSize); @@ -2661,20 +2673,16 @@ class LiftoffCompiler { &LiftoffAssembler::emit_i8x16_shri_u); case wasm::kExprI8x16Add: return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_add); - case wasm::kExprI8x16AddSaturateS: - return EmitBinOp<kS128, kS128>( - &LiftoffAssembler::emit_i8x16_add_saturate_s); - case wasm::kExprI8x16AddSaturateU: - return EmitBinOp<kS128, kS128>( - &LiftoffAssembler::emit_i8x16_add_saturate_u); + case wasm::kExprI8x16AddSatS: + return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_add_sat_s); + case wasm::kExprI8x16AddSatU: + return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_add_sat_u); case wasm::kExprI8x16Sub: return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_sub); - case wasm::kExprI8x16SubSaturateS: - return EmitBinOp<kS128, kS128>( - &LiftoffAssembler::emit_i8x16_sub_saturate_s); - case wasm::kExprI8x16SubSaturateU: - return EmitBinOp<kS128, kS128>( - &LiftoffAssembler::emit_i8x16_sub_saturate_u); + case wasm::kExprI8x16SubSatS: + return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_sub_sat_s); + case wasm::kExprI8x16SubSatU: + return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_sub_sat_u); case wasm::kExprI8x16Mul: return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i8x16_mul); case wasm::kExprI8x16MinS: @@ -2704,20 +2712,16 @@ class LiftoffCompiler { &LiftoffAssembler::emit_i16x8_shri_u); case wasm::kExprI16x8Add: return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_add); - case wasm::kExprI16x8AddSaturateS: - return EmitBinOp<kS128, kS128>( - &LiftoffAssembler::emit_i16x8_add_saturate_s); - case wasm::kExprI16x8AddSaturateU: - return EmitBinOp<kS128, kS128>( - &LiftoffAssembler::emit_i16x8_add_saturate_u); + case wasm::kExprI16x8AddSatS: + return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_add_sat_s); + case wasm::kExprI16x8AddSatU: + return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_add_sat_u); case wasm::kExprI16x8Sub: return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_sub); - case wasm::kExprI16x8SubSaturateS: - return EmitBinOp<kS128, kS128>( - &LiftoffAssembler::emit_i16x8_sub_saturate_s); - case wasm::kExprI16x8SubSaturateU: - return EmitBinOp<kS128, kS128>( - &LiftoffAssembler::emit_i16x8_sub_saturate_u); + case wasm::kExprI16x8SubSatS: + return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_sub_sat_s); + case wasm::kExprI16x8SubSatU: + return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_sub_sat_u); case wasm::kExprI16x8Mul: return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i16x8_mul); case wasm::kExprI16x8MinS: @@ -2759,6 +2763,9 @@ class LiftoffCompiler { return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_max_s); case wasm::kExprI32x4MaxU: return EmitBinOp<kS128, kS128>(&LiftoffAssembler::emit_i32x4_max_u); + case wasm::kExprI32x4DotI16x8S: + return EmitBinOp<kS128, kS128>( + &LiftoffAssembler::emit_i32x4_dot_i16x8_s); case wasm::kExprI64x2Neg: return EmitUnOp<kS128, kS128>(&LiftoffAssembler::emit_i64x2_neg); case wasm::kExprI64x2Shl: @@ -3241,13 +3248,15 @@ class LiftoffCompiler { uint32_t offset = imm.offset; index_reg = AddMemoryMasking(index_reg, &offset, &pinned); - Register index_plus_offset = index_reg; + Register index_plus_offset = + __ cache_state()->is_used(LiftoffRegister(index_reg)) + ? pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp() + : index_reg; if (offset) { - if (__ cache_state()->is_used(LiftoffRegister(index_reg))) { - index_plus_offset = - pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); - } __ emit_i32_addi(index_plus_offset, index_reg, offset); + __ emit_ptrsize_zeroextend_i32(index_plus_offset, index_plus_offset); + } else { + __ emit_ptrsize_zeroextend_i32(index_plus_offset, index_reg); } LiftoffAssembler::VarState timeout = @@ -3288,7 +3297,7 @@ class LiftoffCompiler { } } - ValueType sig_reps[] = {kWasmI32, type, kWasmI64}; + ValueType sig_reps[] = {kPointerValueType, type, kWasmI64}; FunctionSig sig(0, 3, sig_reps); __ PrepareBuiltinCall(&sig, call_descriptor, @@ -3316,16 +3325,18 @@ class LiftoffCompiler { uint32_t offset = imm.offset; index_reg = AddMemoryMasking(index_reg, &offset, &pinned); - Register index_plus_offset = index_reg; + Register index_plus_offset = + __ cache_state()->is_used(LiftoffRegister(index_reg)) + ? pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp() + : index_reg; if (offset) { - if (__ cache_state()->is_used(LiftoffRegister(index_reg))) { - index_plus_offset = - pinned.set(__ GetUnusedRegister(kGpReg, pinned)).gp(); - } __ emit_i32_addi(index_plus_offset, index_reg, offset); + __ emit_ptrsize_zeroextend_i32(index_plus_offset, index_plus_offset); + } else { + __ emit_ptrsize_zeroextend_i32(index_plus_offset, index_reg); } - ValueType sig_reps[] = {kWasmI32, kWasmI32, kWasmI32}; + ValueType sig_reps[] = {kWasmI32, kPointerValueType, kWasmI32}; FunctionSig sig(1, 2, sig_reps); auto call_descriptor = GetBuiltinCallDescriptor<WasmAtomicNotifyDescriptor>(compilation_zone_); @@ -3809,7 +3820,7 @@ class LiftoffCompiler { const Value args[], Value returns[], CallKind call_kind) { for (ValueType ret : imm.sig->returns()) { if (!CheckSupportedType(decoder, - FLAG_liftoff_extern_ref + FLAG_experimental_liftoff_extern_ref ? kSupportedTypes : kSupportedTypesWithoutRefs, ret, "return")) { @@ -3891,7 +3902,7 @@ class LiftoffCompiler { } for (ValueType ret : imm.sig->returns()) { if (!CheckSupportedType(decoder, - FLAG_liftoff_extern_ref + FLAG_experimental_liftoff_extern_ref ? kSupportedTypes : kSupportedTypesWithoutRefs, ret, "return")) { @@ -3918,9 +3929,10 @@ class LiftoffCompiler { // Bounds check against the table size. Label* invalid_func_label = AddOutOfLineTrap( - decoder->position(), WasmCode::kThrowWasmTrapFuncInvalid); + decoder->position(), WasmCode::kThrowWasmTrapTableOutOfBounds); - uint32_t canonical_sig_num = env_->module->signature_ids[imm.sig_index]; + uint32_t canonical_sig_num = + env_->module->canonicalized_type_ids[imm.sig_index]; DCHECK_GE(canonical_sig_num, 0); DCHECK_GE(kMaxInt, canonical_sig_num); @@ -4060,6 +4072,11 @@ class LiftoffCompiler { // address in OSR is correct. int dead_breakpoint_ = 0; + // Remember whether the "hook on function call" has already been checked. + // This happens at the first breakable opcode in the function (if compiling + // for debugging). + bool checked_hook_on_function_call_ = false; + bool has_outstanding_op() const { return outstanding_op_ != kNoOutstandingOp; } @@ -4097,15 +4114,11 @@ WasmCompilationResult ExecuteLiftoffCompilation( std::unique_ptr<DebugSideTable>* debug_sidetable, int dead_breakpoint) { int func_body_size = static_cast<int>(func_body.end - func_body.start); TRACE_EVENT2(TRACE_DISABLED_BY_DEFAULT("v8.wasm.detailed"), - "wasm.CompileBaseline", "func_index", func_index, "body_size", + "wasm.CompileBaseline", "funcIndex", func_index, "bodySize", func_body_size); Zone zone(allocator, "LiftoffCompilationZone"); auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, func_body.sig); - base::Optional<TimedHistogramScope> liftoff_compile_time_scope; - if (counters) { - liftoff_compile_time_scope.emplace(counters->liftoff_compile_time()); - } size_t code_size_estimate = WasmCodeManager::EstimateLiftoffCodeSize(func_body_size); // Allocate the initial buffer a bit bigger to avoid reallocation during code @@ -4118,18 +4131,14 @@ WasmCompilationResult ExecuteLiftoffCompilation( if (debug_sidetable) { debug_sidetable_builder = std::make_unique<DebugSideTableBuilder>(); } - WasmFullDecoder<Decoder::kValidate, LiftoffCompiler> decoder( + WasmFullDecoder<Decoder::kBooleanValidation, LiftoffCompiler> decoder( &zone, env->module, env->enabled_features, detected, func_body, call_descriptor, env, &zone, instruction_buffer->CreateView(), debug_sidetable_builder.get(), for_debugging, func_index, breakpoints, dead_breakpoint); decoder.Decode(); - liftoff_compile_time_scope.reset(); LiftoffCompiler* compiler = &decoder.interface(); - if (decoder.failed()) { - compiler->OnFirstError(&decoder); - return WasmCompilationResult{}; - } + if (decoder.failed()) compiler->OnFirstError(&decoder); if (counters) { // Check that the histogram for the bailout reasons has the correct size. @@ -4175,7 +4184,7 @@ std::unique_ptr<DebugSideTable> GenerateLiftoffDebugSideTable( auto call_descriptor = compiler::GetWasmCallDescriptor(&zone, func_body.sig); DebugSideTableBuilder debug_sidetable_builder; WasmFeatures detected; - WasmFullDecoder<Decoder::kValidate, LiftoffCompiler> decoder( + WasmFullDecoder<Decoder::kBooleanValidation, LiftoffCompiler> decoder( &zone, env->module, env->enabled_features, &detected, func_body, call_descriptor, env, &zone, NewAssemblerBuffer(AssemblerBase::kDefaultBufferSize), diff --git a/chromium/v8/src/wasm/baseline/liftoff-register.h b/chromium/v8/src/wasm/baseline/liftoff-register.h index 49aac008f07..285af7dac02 100644 --- a/chromium/v8/src/wasm/baseline/liftoff-register.h +++ b/chromium/v8/src/wasm/baseline/liftoff-register.h @@ -137,8 +137,8 @@ static_assert(2 * kBitsPerGpRegCode >= kBitsPerFpRegCode, class LiftoffRegister { static constexpr int needed_bits = - Max(kNeedI64RegPair || kNeedS128RegPair ? kBitsPerRegPair : 0, - kBitsPerLiftoffRegCode); + std::max(kNeedI64RegPair || kNeedS128RegPair ? kBitsPerRegPair : 0, + kBitsPerLiftoffRegCode); using storage_t = std::conditional< needed_bits <= 8, uint8_t, std::conditional<needed_bits <= 16, uint16_t, uint32_t>::type>::type; diff --git a/chromium/v8/src/wasm/baseline/mips/liftoff-assembler-mips.h b/chromium/v8/src/wasm/baseline/mips/liftoff-assembler-mips.h index 97b8487848e..5c78eca3195 100644 --- a/chromium/v8/src/wasm/baseline/mips/liftoff-assembler-mips.h +++ b/chromium/v8/src/wasm/baseline/mips/liftoff-assembler-mips.h @@ -360,16 +360,16 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value, } } -void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, +void LiftoffAssembler::LoadFromInstance(Register dst, int32_t offset, int size) { - DCHECK_LE(offset, kMaxInt); + DCHECK_LE(0, offset); lw(dst, liftoff::GetInstanceOperand()); DCHECK_EQ(4, size); lw(dst, MemOperand(dst, offset)); } void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, - uint32_t offset) { + int32_t offset) { LoadFromInstance(dst, offset, kTaggedSize); } @@ -1883,16 +1883,16 @@ void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, bailout(kSimd, "emit_i8x16_add"); } -void LiftoffAssembler::emit_i8x16_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { - bailout(kSimd, "emit_i8x16_add_saturate_s"); +void LiftoffAssembler::emit_i8x16_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "emit_i8x16_add_sat_s"); } -void LiftoffAssembler::emit_i8x16_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { - bailout(kSimd, "emit_i8x16_add_saturate_u"); +void LiftoffAssembler::emit_i8x16_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "emit_i8x16_add_sat_u"); } void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, @@ -1900,16 +1900,16 @@ void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, bailout(kSimd, "emit_i8x16_sub"); } -void LiftoffAssembler::emit_i8x16_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { - bailout(kSimd, "emit_i8x16_sub_saturate_s"); +void LiftoffAssembler::emit_i8x16_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "emit_i8x16_sub_sat_s"); } -void LiftoffAssembler::emit_i8x16_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { - bailout(kSimd, "emit_i8x16_sub_saturate_u"); +void LiftoffAssembler::emit_i8x16_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "emit_i8x16_sub_sat_u"); } void LiftoffAssembler::emit_i8x16_mul(LiftoffRegister dst, LiftoffRegister lhs, @@ -1998,16 +1998,16 @@ void LiftoffAssembler::emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, bailout(kSimd, "emit_i16x8_add"); } -void LiftoffAssembler::emit_i16x8_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { - bailout(kSimd, "emit_i16x8_add_saturate_s"); +void LiftoffAssembler::emit_i16x8_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "emit_i16x8_add_sat_s"); } -void LiftoffAssembler::emit_i16x8_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { - bailout(kSimd, "emit_i16x8_add_saturate_u"); +void LiftoffAssembler::emit_i16x8_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "emit_i16x8_add_sat_u"); } void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, @@ -2015,16 +2015,16 @@ void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, bailout(kSimd, "emit_i16x8_sub"); } -void LiftoffAssembler::emit_i16x8_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { - bailout(kSimd, "emit_i16x8_sub_saturate_s"); +void LiftoffAssembler::emit_i16x8_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "emit_i16x8_sub_sat_s"); } -void LiftoffAssembler::emit_i16x8_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { - bailout(kSimd, "emit_i16x8_sub_saturate_u"); +void LiftoffAssembler::emit_i16x8_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "emit_i16x8_sub_sat_u"); } void LiftoffAssembler::emit_i16x8_mul(LiftoffRegister dst, LiftoffRegister lhs, @@ -2147,6 +2147,12 @@ void LiftoffAssembler::emit_i32x4_max_u(LiftoffRegister dst, bailout(kSimd, "emit_i32x4_max_u"); } +void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "emit_i32x4_dot_i16x8_s"); +} + void LiftoffAssembler::emit_i64x2_neg(LiftoffRegister dst, LiftoffRegister src) { bailout(kSimd, "emit_i64x2_neg"); diff --git a/chromium/v8/src/wasm/baseline/mips64/liftoff-assembler-mips64.h b/chromium/v8/src/wasm/baseline/mips64/liftoff-assembler-mips64.h index 4c6c1fe1cea..b97c49437f6 100644 --- a/chromium/v8/src/wasm/baseline/mips64/liftoff-assembler-mips64.h +++ b/chromium/v8/src/wasm/baseline/mips64/liftoff-assembler-mips64.h @@ -5,6 +5,7 @@ #ifndef V8_WASM_BASELINE_MIPS64_LIFTOFF_ASSEMBLER_MIPS64_H_ #define V8_WASM_BASELINE_MIPS64_LIFTOFF_ASSEMBLER_MIPS64_H_ +#include "src/heap/memory-chunk.h" #include "src/wasm/baseline/liftoff-assembler.h" namespace v8 { @@ -339,9 +340,9 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value, } } -void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, +void LiftoffAssembler::LoadFromInstance(Register dst, int32_t offset, int size) { - DCHECK_LE(offset, kMaxInt); + DCHECK_LE(0, offset); Ld(dst, liftoff::GetInstanceOperand()); DCHECK(size == 4 || size == 8); if (size == 4) { @@ -352,7 +353,7 @@ void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, } void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, - uint32_t offset) { + int32_t offset) { LoadFromInstance(dst, offset, kTaggedSize); } @@ -378,7 +379,27 @@ void LiftoffAssembler::StoreTaggedPointer(Register dst_addr, int32_t offset_imm, LiftoffRegister src, LiftoffRegList pinned) { - bailout(kRefTypes, "GlobalSet"); + DCHECK_GE(offset_imm, 0); + DCHECK_LE(offset_imm, std::numeric_limits<int32_t>::max()); + STATIC_ASSERT(kTaggedSize == kInt64Size); + Register scratch = pinned.set(GetUnusedRegister(kGpReg, pinned)).gp(); + Sd(src.gp(), MemOperand(dst_addr, offset_imm)); + + Label write_barrier; + Label exit; + CheckPageFlag(dst_addr, scratch, + MemoryChunk::kPointersFromHereAreInterestingMask, ne, + &write_barrier); + b(&exit); + bind(&write_barrier); + JumpIfSmi(src.gp(), &exit); + CheckPageFlag(src.gp(), scratch, + MemoryChunk::kPointersToHereAreInterestingMask, eq, + &exit); + Daddu(scratch, dst_addr, offset_imm); + CallRecordWriteStub(dst_addr, scratch, EMIT_REMEMBERED_SET, kSaveFPRegs, + wasm::WasmCode::kRecordWrite); + bind(&exit); } void LiftoffAssembler::Load(LiftoffRegister dst, Register src_addr, @@ -1487,6 +1508,16 @@ void LiftoffAssembler::LoadTransform(LiftoffRegister dst, Register src_addr, fill_d(dst_msa, scratch); ilvr_w(dst_msa, kSimd128RegZero, dst_msa); } + } else if (transform == LoadTransformationKind::kZeroExtend) { + xor_v(dst_msa, dst_msa, dst_msa); + if (memtype == MachineType::Int32()) { + Lwu(scratch, src_op); + insert_w(dst_msa, 0, scratch); + } else { + DCHECK_EQ(MachineType::Int64(), memtype); + Ld(scratch, src_op); + insert_d(dst_msa, 0, scratch); + } } else { DCHECK_EQ(LoadTransformationKind::kSplat, transform); if (memtype == MachineType::Int8()) { @@ -1841,15 +1872,15 @@ void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, addv_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } -void LiftoffAssembler::emit_i8x16_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { adds_s_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } -void LiftoffAssembler::emit_i8x16_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { adds_u_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } @@ -1858,15 +1889,15 @@ void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, subv_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } -void LiftoffAssembler::emit_i8x16_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { subs_s_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } -void LiftoffAssembler::emit_i8x16_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { subs_u_b(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } @@ -1970,15 +2001,15 @@ void LiftoffAssembler::emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, addv_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } -void LiftoffAssembler::emit_i16x8_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { adds_s_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } -void LiftoffAssembler::emit_i16x8_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { adds_u_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } @@ -1987,15 +2018,15 @@ void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, subv_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } -void LiftoffAssembler::emit_i16x8_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { subs_s_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } -void LiftoffAssembler::emit_i16x8_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { subs_u_h(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } @@ -2131,6 +2162,12 @@ void LiftoffAssembler::emit_i32x4_max_u(LiftoffRegister dst, max_u_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); } +void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + dotp_s_w(dst.fp().toW(), lhs.fp().toW(), rhs.fp().toW()); +} + void LiftoffAssembler::emit_i64x2_neg(LiftoffRegister dst, LiftoffRegister src) { xor_v(kSimd128RegZero, kSimd128RegZero, kSimd128RegZero); @@ -2264,6 +2301,8 @@ void LiftoffAssembler::emit_f32x4_min(LiftoffRegister dst, LiftoffRegister lhs, // dst = (scratch1 <= scratch0) ? scratch1 : scratch0. fsle_w(dst_msa, scratch1, scratch0); bsel_v(dst_msa, scratch0, scratch1); + // Canonicalize the result. + fmin_w(dst_msa, dst_msa, dst_msa); } void LiftoffAssembler::emit_f32x4_max(LiftoffRegister dst, LiftoffRegister lhs, @@ -2284,6 +2323,8 @@ void LiftoffAssembler::emit_f32x4_max(LiftoffRegister dst, LiftoffRegister lhs, // dst = (scratch0 <= scratch1) ? scratch1 : scratch0. fsle_w(dst_msa, scratch0, scratch1); bsel_v(dst_msa, scratch0, scratch1); + // Canonicalize the result. + fmax_w(dst_msa, dst_msa, dst_msa); } void LiftoffAssembler::emit_f32x4_pmin(LiftoffRegister dst, LiftoffRegister lhs, @@ -2383,6 +2424,8 @@ void LiftoffAssembler::emit_f64x2_min(LiftoffRegister dst, LiftoffRegister lhs, // dst = (scratch1 <= scratch0) ? scratch1 : scratch0. fsle_d(dst_msa, scratch1, scratch0); bsel_v(dst_msa, scratch0, scratch1); + // Canonicalize the result. + fmin_d(dst_msa, dst_msa, dst_msa); } void LiftoffAssembler::emit_f64x2_max(LiftoffRegister dst, LiftoffRegister lhs, @@ -2403,6 +2446,8 @@ void LiftoffAssembler::emit_f64x2_max(LiftoffRegister dst, LiftoffRegister lhs, // dst = (scratch0 <= scratch1) ? scratch1 : scratch0. fsle_d(dst_msa, scratch0, scratch1); bsel_v(dst_msa, scratch0, scratch1); + // Canonicalize the result. + fmax_d(dst_msa, dst_msa, dst_msa); } void LiftoffAssembler::emit_f64x2_pmin(LiftoffRegister dst, LiftoffRegister lhs, diff --git a/chromium/v8/src/wasm/baseline/ppc/OWNERS b/chromium/v8/src/wasm/baseline/ppc/OWNERS index 6edd45a6ef2..02c2cd757c9 100644 --- a/chromium/v8/src/wasm/baseline/ppc/OWNERS +++ b/chromium/v8/src/wasm/baseline/ppc/OWNERS @@ -2,3 +2,4 @@ junyan@redhat.com joransiu@ca.ibm.com midawson@redhat.com mfarazma@redhat.com +vasili.skurydzin@ibm.com diff --git a/chromium/v8/src/wasm/baseline/ppc/liftoff-assembler-ppc.h b/chromium/v8/src/wasm/baseline/ppc/liftoff-assembler-ppc.h index ef7b720ea9e..f75e9db459d 100644 --- a/chromium/v8/src/wasm/baseline/ppc/liftoff-assembler-ppc.h +++ b/chromium/v8/src/wasm/baseline/ppc/liftoff-assembler-ppc.h @@ -88,13 +88,11 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value, bailout(kUnsupportedArchitecture, "LoadConstant"); } -void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, - int size) { +void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) { bailout(kUnsupportedArchitecture, "LoadFromInstance"); } -void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, - uint32_t offset) { +void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) { bailout(kUnsupportedArchitecture, "LoadTaggedPointerFromInstance"); } @@ -944,6 +942,12 @@ void LiftoffAssembler::emit_i32x4_max_u(LiftoffRegister dst, bailout(kUnsupportedArchitecture, "emit_i32x4_max_u"); } +void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "i32x4_dot_i16x8_s"); +} + void LiftoffAssembler::emit_i16x8_splat(LiftoffRegister dst, LiftoffRegister src) { bailout(kUnsupportedArchitecture, "emit_i16x8splat"); @@ -1006,9 +1010,9 @@ void LiftoffAssembler::emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i16x8add"); } -void LiftoffAssembler::emit_i16x8_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i16x8addsaturate_s"); } @@ -1017,15 +1021,15 @@ void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i16x8sub"); } -void LiftoffAssembler::emit_i16x8_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i16x8subsaturate_s"); } -void LiftoffAssembler::emit_i16x8_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i16x8subsaturate_u"); } @@ -1034,9 +1038,9 @@ void LiftoffAssembler::emit_i16x8_mul(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i16x8mul"); } -void LiftoffAssembler::emit_i16x8_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i16x8addsaturate_u"); } @@ -1172,9 +1176,9 @@ void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i8x16add"); } -void LiftoffAssembler::emit_i8x16_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i8x16addsaturate_s"); } @@ -1485,15 +1489,15 @@ void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i8x16sub"); } -void LiftoffAssembler::emit_i8x16_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i8x16subsaturate_s"); } -void LiftoffAssembler::emit_i8x16_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i8x16subsaturate_u"); } @@ -1502,9 +1506,9 @@ void LiftoffAssembler::emit_i8x16_mul(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i8x16mul"); } -void LiftoffAssembler::emit_i8x16_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i8x16addsaturate_u"); } diff --git a/chromium/v8/src/wasm/baseline/s390/liftoff-assembler-s390.h b/chromium/v8/src/wasm/baseline/s390/liftoff-assembler-s390.h index dc6ce2f0b3f..a88baa11461 100644 --- a/chromium/v8/src/wasm/baseline/s390/liftoff-assembler-s390.h +++ b/chromium/v8/src/wasm/baseline/s390/liftoff-assembler-s390.h @@ -87,13 +87,11 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value, bailout(kUnsupportedArchitecture, "LoadConstant"); } -void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, - int size) { +void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) { bailout(kUnsupportedArchitecture, "LoadFromInstance"); } -void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, - uint32_t offset) { +void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) { bailout(kUnsupportedArchitecture, "LoadTaggedPointerFromInstance"); } @@ -948,6 +946,12 @@ void LiftoffAssembler::emit_i32x4_max_u(LiftoffRegister dst, bailout(kUnsupportedArchitecture, "emit_i32x4_max_u"); } +void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + bailout(kSimd, "i32x4_dot_i16x8_s"); +} + void LiftoffAssembler::emit_i16x8_splat(LiftoffRegister dst, LiftoffRegister src) { bailout(kUnsupportedArchitecture, "emit_i16x8splat"); @@ -1010,9 +1014,9 @@ void LiftoffAssembler::emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i16x8add"); } -void LiftoffAssembler::emit_i16x8_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i16x8addsaturate_s"); } @@ -1021,15 +1025,15 @@ void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i16x8sub"); } -void LiftoffAssembler::emit_i16x8_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i16x8subsaturate_s"); } -void LiftoffAssembler::emit_i16x8_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i16x8subsaturate_u"); } @@ -1038,9 +1042,9 @@ void LiftoffAssembler::emit_i16x8_mul(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i16x8mul"); } -void LiftoffAssembler::emit_i16x8_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i16x8addsaturate_u"); } @@ -1176,9 +1180,9 @@ void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i8x16add"); } -void LiftoffAssembler::emit_i8x16_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i8x16addsaturate_s"); } @@ -1187,15 +1191,15 @@ void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i8x16sub"); } -void LiftoffAssembler::emit_i8x16_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i8x16subsaturate_s"); } -void LiftoffAssembler::emit_i8x16_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i8x16subsaturate_u"); } @@ -1204,9 +1208,9 @@ void LiftoffAssembler::emit_i8x16_mul(LiftoffRegister dst, LiftoffRegister lhs, bailout(kUnsupportedArchitecture, "emit_i8x16mul"); } -void LiftoffAssembler::emit_i8x16_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { bailout(kUnsupportedArchitecture, "emit_i8x16addsaturate_u"); } diff --git a/chromium/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h b/chromium/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h index 713a1ce72a1..a64b0e2e374 100644 --- a/chromium/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h +++ b/chromium/v8/src/wasm/baseline/x64/liftoff-assembler-x64.h @@ -236,11 +236,10 @@ void LiftoffAssembler::LoadConstant(LiftoffRegister reg, WasmValue value, } } -void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, - int size) { - DCHECK_LE(offset, kMaxInt); - movq(dst, liftoff::GetInstanceOperand()); +void LiftoffAssembler::LoadFromInstance(Register dst, int offset, int size) { + DCHECK_LE(0, offset); DCHECK(size == 4 || size == 8); + movq(dst, liftoff::GetInstanceOperand()); if (size == 4) { movl(dst, Operand(dst, offset)); } else { @@ -248,9 +247,8 @@ void LiftoffAssembler::LoadFromInstance(Register dst, uint32_t offset, } } -void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, - uint32_t offset) { - DCHECK_LE(offset, kMaxInt); +void LiftoffAssembler::LoadTaggedPointerFromInstance(Register dst, int offset) { + DCHECK_LE(0, offset); movq(dst, liftoff::GetInstanceOperand()); LoadTaggedPointerField(dst, Operand(dst, offset)); } @@ -2232,11 +2230,11 @@ void EmitI64x2ShrS(LiftoffAssembler* assm, LiftoffRegister dst, assm->Pextrq(tmp, lhs.fp(), int8_t{0x0}); assm->sarq_cl(tmp); - assm->Pinsrq(dst.fp(), tmp, int8_t{0x0}); + assm->Pinsrq(dst.fp(), tmp, uint8_t{0x0}); assm->Pextrq(tmp, lhs.fp(), int8_t{0x1}); assm->sarq_cl(tmp); - assm->Pinsrq(dst.fp(), tmp, int8_t{0x1}); + assm->Pinsrq(dst.fp(), tmp, uint8_t{0x1}); // restore rcx. if (restore_rcx) { @@ -2289,14 +2287,21 @@ void LiftoffAssembler::LoadTransform(LiftoffRegister dst, Register src_addr, } else if (memtype == MachineType::Uint32()) { Pmovzxdq(dst.fp(), src_op); } + } else if (transform == LoadTransformationKind::kZeroExtend) { + if (memtype == MachineType::Int32()) { + Movss(dst.fp(), src_op); + } else { + DCHECK_EQ(MachineType::Int64(), memtype); + Movsd(dst.fp(), src_op); + } } else { DCHECK_EQ(LoadTransformationKind::kSplat, transform); if (memtype == MachineType::Int8()) { - Pinsrb(dst.fp(), src_op, 0); + Pinsrb(dst.fp(), dst.fp(), src_op, 0); Pxor(kScratchDoubleReg, kScratchDoubleReg); Pshufb(dst.fp(), kScratchDoubleReg); } else if (memtype == MachineType::Int16()) { - Pinsrw(dst.fp(), src_op, 0); + Pinsrw(dst.fp(), dst.fp(), src_op, 0); Pshuflw(dst.fp(), dst.fp(), uint8_t{0}); Punpcklqdq(dst.fp(), dst.fp()); } else if (memtype == MachineType::Int32()) { @@ -2304,8 +2309,8 @@ void LiftoffAssembler::LoadTransform(LiftoffRegister dst, Register src_addr, CpuFeatureScope avx_scope(this, AVX); vbroadcastss(dst.fp(), src_op); } else { - Movss(dst.fp(), src_op); - Shufps(dst.fp(), dst.fp(), byte{0}); + movss(dst.fp(), src_op); + shufps(dst.fp(), dst.fp(), byte{0}); } } else if (memtype == MachineType::Int64()) { Movddup(dst.fp(), src_op); @@ -2324,22 +2329,10 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, wasm::SimdShuffle::Pack16Lanes(imms, shuffle); TurboAssembler::Move(kScratchDoubleReg, make_uint64(imms[3], imms[2]), make_uint64(imms[1], imms[0])); - if (CpuFeatures::IsSupported(AVX)) { - CpuFeatureScope scope(this, AVX); - vpshufb(dst.fp(), lhs.fp(), kScratchDoubleReg); - } else { - if (dst != lhs) { - movups(dst.fp(), lhs.fp()); - } - pshufb(dst.fp(), kScratchDoubleReg); - } + Pshufb(dst.fp(), lhs.fp(), kScratchDoubleReg); return; } - LiftoffRegister tmp_simd = - GetUnusedRegister(kFpReg, LiftoffRegList::ForRegs(dst, lhs, rhs)); - Movups(kScratchDoubleReg, lhs.fp()); - uint64_t mask1[2] = {}; for (int i = 15; i >= 0; i--) { uint8_t lane = shuffle[i]; @@ -2347,10 +2340,8 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, mask1[j] <<= 8; mask1[j] |= lane < kSimd128Size ? lane : 0x80; } - TurboAssembler::Move(tmp_simd.fp(), mask1[0]); - movq(kScratchRegister, mask1[1]); - Pinsrq(tmp_simd.fp(), kScratchRegister, int8_t{1}); - Pshufb(kScratchDoubleReg, tmp_simd.fp()); + TurboAssembler::Move(liftoff::kScratchDoubleReg2, mask1[1], mask1[0]); + Pshufb(kScratchDoubleReg, lhs.fp(), liftoff::kScratchDoubleReg2); uint64_t mask2[2] = {}; for (int i = 15; i >= 0; i--) { @@ -2359,14 +2350,9 @@ void LiftoffAssembler::emit_i8x16_shuffle(LiftoffRegister dst, mask2[j] <<= 8; mask2[j] |= lane >= kSimd128Size ? (lane & 0x0F) : 0x80; } - TurboAssembler::Move(tmp_simd.fp(), mask2[0]); - movq(kScratchRegister, mask2[1]); - Pinsrq(tmp_simd.fp(), kScratchRegister, int8_t{1}); + TurboAssembler::Move(liftoff::kScratchDoubleReg2, mask2[1], mask2[0]); - if (dst.fp() != rhs.fp()) { - Movups(dst.fp(), rhs.fp()); - } - Pshufb(dst.fp(), tmp_simd.fp()); + Pshufb(dst.fp(), rhs.fp(), liftoff::kScratchDoubleReg2); Por(dst.fp(), kScratchDoubleReg); } @@ -2379,10 +2365,7 @@ void LiftoffAssembler::emit_i8x16_swizzle(LiftoffRegister dst, TurboAssembler::Move(mask, uint32_t{0x70707070}); Pshufd(mask, mask, uint8_t{0x0}); Paddusb(mask, rhs.fp()); - if (lhs != dst) { - Movaps(dst.fp(), lhs.fp()); - } - Pshufb(dst.fp(), mask); + Pshufb(dst.fp(), lhs.fp(), mask); } void LiftoffAssembler::emit_i8x16_splat(LiftoffRegister dst, @@ -2413,10 +2396,7 @@ void LiftoffAssembler::emit_i64x2_splat(LiftoffRegister dst, void LiftoffAssembler::emit_f32x4_splat(LiftoffRegister dst, LiftoffRegister src) { - if (dst.fp() != src.fp()) { - Movss(dst.fp(), src.fp()); - } - Shufps(dst.fp(), src.fp(), static_cast<byte>(0)); + Shufps(dst.fp(), src.fp(), 0); } void LiftoffAssembler::emit_f64x2_splat(LiftoffRegister dst, @@ -2659,7 +2639,7 @@ void LiftoffAssembler::emit_s128_const(LiftoffRegister dst, memcpy(vals, imms, sizeof(vals)); TurboAssembler::Move(dst.fp(), vals[0]); movq(kScratchRegister, vals[1]); - Pinsrq(dst.fp(), kScratchRegister, int8_t{1}); + Pinsrq(dst.fp(), kScratchRegister, uint8_t{1}); } void LiftoffAssembler::emit_s128_not(LiftoffRegister dst, LiftoffRegister src) { @@ -2827,16 +2807,16 @@ void LiftoffAssembler::emit_i8x16_add(LiftoffRegister dst, LiftoffRegister lhs, this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i8x16_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdCommutativeBinOp<&Assembler::vpaddsb, &Assembler::paddsb>( this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i8x16_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdCommutativeBinOp<&Assembler::vpaddusb, &Assembler::paddusb>( this, dst, lhs, rhs); } @@ -2847,16 +2827,16 @@ void LiftoffAssembler::emit_i8x16_sub(LiftoffRegister dst, LiftoffRegister lhs, this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i8x16_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdNonCommutativeBinOp<&Assembler::vpsubsb, &Assembler::psubsb>( this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i8x16_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i8x16_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdNonCommutativeBinOp<&Assembler::vpsubusb, &Assembler::psubusb>(this, dst, lhs, rhs); @@ -3025,16 +3005,16 @@ void LiftoffAssembler::emit_i16x8_add(LiftoffRegister dst, LiftoffRegister lhs, this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i16x8_add_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdCommutativeBinOp<&Assembler::vpaddsw, &Assembler::paddsw>( this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i16x8_add_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_add_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdCommutativeBinOp<&Assembler::vpaddusw, &Assembler::paddusw>( this, dst, lhs, rhs); } @@ -3045,16 +3025,16 @@ void LiftoffAssembler::emit_i16x8_sub(LiftoffRegister dst, LiftoffRegister lhs, this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i16x8_sub_saturate_s(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdNonCommutativeBinOp<&Assembler::vpsubsw, &Assembler::psubsw>( this, dst, lhs, rhs); } -void LiftoffAssembler::emit_i16x8_sub_saturate_u(LiftoffRegister dst, - LiftoffRegister lhs, - LiftoffRegister rhs) { +void LiftoffAssembler::emit_i16x8_sub_sat_u(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { liftoff::EmitSimdNonCommutativeBinOp<&Assembler::vpsubusw, &Assembler::psubusw>(this, dst, lhs, rhs); @@ -3204,6 +3184,13 @@ void LiftoffAssembler::emit_i32x4_max_u(LiftoffRegister dst, this, dst, lhs, rhs, base::Optional<CpuFeature>(SSE4_1)); } +void LiftoffAssembler::emit_i32x4_dot_i16x8_s(LiftoffRegister dst, + LiftoffRegister lhs, + LiftoffRegister rhs) { + liftoff::EmitSimdCommutativeBinOp<&Assembler::vpmaddwd, &Assembler::pmaddwd>( + this, dst, lhs, rhs); +} + void LiftoffAssembler::emit_i64x2_neg(LiftoffRegister dst, LiftoffRegister src) { DoubleRegister reg = dst.fp() == src.fp() ? kScratchDoubleReg : dst.fp(); |