diff options
Diffstat (limited to 'chromium/v8/src/compiler/backend/riscv64')
4 files changed, 1555 insertions, 932 deletions
diff --git a/chromium/v8/src/compiler/backend/riscv64/code-generator-riscv64.cc b/chromium/v8/src/compiler/backend/riscv64/code-generator-riscv64.cc index 2d92ae1567e..c95299ee1d2 100644 --- a/chromium/v8/src/compiler/backend/riscv64/code-generator-riscv64.cc +++ b/chromium/v8/src/compiler/backend/riscv64/code-generator-riscv64.cc @@ -106,7 +106,6 @@ class RiscvOperandConverter final : public InstructionOperandConverter { constant.ToDelayedStringConstant()); case Constant::kRpoNumber: UNREACHABLE(); // TODO(titzer): RPO immediates - break; } UNREACHABLE(); } @@ -307,17 +306,6 @@ FPUCondition FlagsConditionToConditionCmpFPU(bool* predicate, UNREACHABLE(); } -void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, - InstructionCode opcode, Instruction* instr, - RiscvOperandConverter const& i) { - const MemoryAccessMode access_mode = - static_cast<MemoryAccessMode>(MiscField::decode(opcode)); - if (access_mode == kMemoryAccessPoisoned) { - Register value = i.OutputRegister(); - codegen->tasm()->And(value, value, kSpeculationPoisonRegister); - } -} - } // namespace #define ASSEMBLE_ATOMIC_LOAD_INTEGER(asm_instr) \ @@ -336,7 +324,7 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, #define ASSEMBLE_ATOMIC_BINOP(load_linked, store_conditional, bin_instr) \ do { \ Label binop; \ - __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ + __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ __ sync(); \ __ bind(&binop); \ __ load_linked(i.OutputRegister(0), MemOperand(i.TempRegister(0), 0)); \ @@ -351,7 +339,7 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, size, bin_instr, representation) \ do { \ Label binop; \ - __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ + __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ if (representation == 32) { \ __ And(i.TempRegister(3), i.TempRegister(0), 0x3); \ } else { \ @@ -380,7 +368,7 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, Label exchange; \ __ sync(); \ __ bind(&exchange); \ - __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ + __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ __ load_linked(i.OutputRegister(0), MemOperand(i.TempRegister(0), 0)); \ __ Move(i.TempRegister(1), i.InputRegister(2)); \ __ store_conditional(i.TempRegister(1), MemOperand(i.TempRegister(0), 0)); \ @@ -392,7 +380,7 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, load_linked, store_conditional, sign_extend, size, representation) \ do { \ Label exchange; \ - __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ + __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ if (representation == 32) { \ __ And(i.TempRegister(1), i.TempRegister(0), 0x3); \ } else { \ @@ -419,7 +407,7 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, do { \ Label compareExchange; \ Label exit; \ - __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ + __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ __ sync(); \ __ bind(&compareExchange); \ __ load_linked(i.OutputRegister(0), MemOperand(i.TempRegister(0), 0)); \ @@ -438,7 +426,7 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, do { \ Label compareExchange; \ Label exit; \ - __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ + __ Add64(i.TempRegister(0), i.InputRegister(0), i.InputRegister(1)); \ if (representation == 32) { \ __ And(i.TempRegister(1), i.TempRegister(0), 0x3); \ } else { \ @@ -453,8 +441,8 @@ void EmitWordLoadPoisoningIfNeeded(CodeGenerator* codegen, __ load_linked(i.TempRegister(2), MemOperand(i.TempRegister(0), 0)); \ __ ExtractBits(i.OutputRegister(0), i.TempRegister(2), i.TempRegister(1), \ size, sign_extend); \ - __ ExtractBits(i.InputRegister(2), i.InputRegister(2), i.TempRegister(1), \ - size, sign_extend); \ + __ ExtractBits(i.InputRegister(2), i.InputRegister(2), 0, size, \ + sign_extend); \ __ BranchShort(&exit, ne, i.InputRegister(2), \ Operand(i.OutputRegister(0))); \ __ InsertBits(i.TempRegister(2), i.InputRegister(3), i.TempRegister(1), \ @@ -570,31 +558,6 @@ void CodeGenerator::BailoutIfDeoptimized() { RelocInfo::CODE_TARGET, ne, kScratchReg, Operand(zero_reg)); } -void CodeGenerator::GenerateSpeculationPoisonFromCodeStartRegister() { - // Calculate a mask which has all bits set in the normal case, but has all - // bits cleared if we are speculatively executing the wrong PC. - // difference = (current - expected) | (expected - current) - // poison = ~(difference >> (kBitsPerSystemPointer - 1)) - __ ComputeCodeStartAddress(kScratchReg); - __ Move(kSpeculationPoisonRegister, kScratchReg); - __ Sub32(kSpeculationPoisonRegister, kSpeculationPoisonRegister, - kJavaScriptCallCodeStartRegister); - __ Sub32(kJavaScriptCallCodeStartRegister, kJavaScriptCallCodeStartRegister, - kScratchReg); - __ or_(kSpeculationPoisonRegister, kSpeculationPoisonRegister, - kJavaScriptCallCodeStartRegister); - __ Sra64(kSpeculationPoisonRegister, kSpeculationPoisonRegister, - kBitsPerSystemPointer - 1); - __ Nor(kSpeculationPoisonRegister, kSpeculationPoisonRegister, - kSpeculationPoisonRegister); -} - -void CodeGenerator::AssembleRegisterArgumentPoisoning() { - __ And(kJSFunctionRegister, kJSFunctionRegister, kSpeculationPoisonRegister); - __ And(kContextRegister, kContextRegister, kSpeculationPoisonRegister); - __ And(sp, sp, kSpeculationPoisonRegister); -} - // Assembles an instruction after register allocation, producing machine code. CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( Instruction* instr) { @@ -780,13 +743,13 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( case kArchTableSwitch: AssembleArchTableSwitch(instr); break; - case kArchAbortCSAAssert: + case kArchAbortCSADcheck: DCHECK(i.InputRegister(0) == a0); { // We don't actually want to generate a pile of code for this, so just // claim there is a stack frame, without generating one. - FrameScope scope(tasm(), StackFrame::NONE); - __ Call(isolate()->builtins()->code_handle(Builtin::kAbortCSAAssert), + FrameScope scope(tasm(), StackFrame::NO_FRAME_TYPE); + __ Call(isolate()->builtins()->code_handle(Builtin::kAbortCSADcheck), RelocInfo::CODE_TARGET); } __ stop(); @@ -887,10 +850,6 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( break; } - case kArchWordPoisonOnSpeculation: - __ And(i.OutputRegister(), i.InputRegister(0), - kSpeculationPoisonRegister); - break; case kIeee754Float64Acos: ASSEMBLE_IEEE754_UNOP(acos); break; @@ -1094,17 +1053,16 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( case kRiscvPopcnt32: { Register src = i.InputRegister(0); Register dst = i.OutputRegister(); - __ Popcnt32(dst, src); + __ Popcnt32(dst, src, kScratchReg); } break; case kRiscvPopcnt64: { Register src = i.InputRegister(0); Register dst = i.OutputRegister(); - __ Popcnt64(dst, src); + __ Popcnt64(dst, src, kScratchReg); } break; case kRiscvShl32: if (instr->InputAt(1)->IsRegister()) { - __ Sll32(i.OutputRegister(), i.InputRegister(0), - i.InputRegister(1)); + __ Sll32(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); } else { int64_t imm = i.InputOperand(1).immediate(); __ Sll32(i.OutputRegister(), i.InputRegister(0), @@ -1113,8 +1071,7 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( break; case kRiscvShr32: if (instr->InputAt(1)->IsRegister()) { - __ Srl32(i.OutputRegister(), i.InputRegister(0), - i.InputRegister(1)); + __ Srl32(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); } else { int64_t imm = i.InputOperand(1).immediate(); __ Srl32(i.OutputRegister(), i.InputRegister(0), @@ -1123,8 +1080,7 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( break; case kRiscvSar32: if (instr->InputAt(1)->IsRegister()) { - __ Sra32(i.OutputRegister(), i.InputRegister(0), - i.InputRegister(1)); + __ Sra32(i.OutputRegister(), i.InputRegister(0), i.InputRegister(1)); } else { int64_t imm = i.InputOperand(1).immediate(); __ Sra32(i.OutputRegister(), i.InputRegister(0), @@ -1553,30 +1509,24 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( break; case kRiscvLbu: __ Lbu(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvLb: __ Lb(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvSb: __ Sb(i.InputOrZeroRegister(2), i.MemoryOperand()); break; case kRiscvLhu: __ Lhu(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvUlhu: __ Ulhu(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvLh: __ Lh(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvUlh: __ Ulh(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvSh: __ Sh(i.InputOrZeroRegister(2), i.MemoryOperand()); @@ -1586,27 +1536,21 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( break; case kRiscvLw: __ Lw(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvUlw: __ Ulw(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvLwu: __ Lwu(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvUlwu: __ Ulwu(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvLd: __ Ld(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvUld: __ Uld(i.OutputRegister(), i.MemoryOperand()); - EmitWordLoadPoisoningIfNeeded(this, opcode, instr, i); break; case kRiscvSw: __ Sw(i.InputOrZeroRegister(2), i.MemoryOperand()); @@ -1625,7 +1569,7 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( break; } case kRiscvULoadFloat: { - __ ULoadFloat(i.OutputSingleRegister(), i.MemoryOperand()); + __ ULoadFloat(i.OutputSingleRegister(), i.MemoryOperand(), kScratchReg); break; } case kRiscvStoreFloat: { @@ -1645,14 +1589,14 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( if (ft == kDoubleRegZero && !__ IsSingleZeroRegSet()) { __ LoadFPRImmediate(kDoubleRegZero, 0.0f); } - __ UStoreFloat(ft, operand); + __ UStoreFloat(ft, operand, kScratchReg); break; } case kRiscvLoadDouble: __ LoadDouble(i.OutputDoubleRegister(), i.MemoryOperand()); break; case kRiscvULoadDouble: - __ ULoadDouble(i.OutputDoubleRegister(), i.MemoryOperand()); + __ ULoadDouble(i.OutputDoubleRegister(), i.MemoryOperand(), kScratchReg); break; case kRiscvStoreDouble: { FPURegister ft = i.InputOrZeroDoubleRegister(2); @@ -1667,7 +1611,7 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( if (ft == kDoubleRegZero && !__ IsDoubleZeroRegSet()) { __ LoadFPRImmediate(kDoubleRegZero, 0.0); } - __ UStoreDouble(ft, i.MemoryOperand()); + __ UStoreDouble(ft, i.MemoryOperand(), kScratchReg); break; } case kRiscvSync: { @@ -1723,156 +1667,175 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( break; } case kRiscvByteSwap64: { - __ ByteSwap(i.OutputRegister(0), i.InputRegister(0), 8); + __ ByteSwap(i.OutputRegister(0), i.InputRegister(0), 8, kScratchReg); break; } case kRiscvByteSwap32: { - __ ByteSwap(i.OutputRegister(0), i.InputRegister(0), 4); + __ ByteSwap(i.OutputRegister(0), i.InputRegister(0), 4, kScratchReg); break; } - case kWord32AtomicLoadInt8: + case kAtomicLoadInt8: + DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); ASSEMBLE_ATOMIC_LOAD_INTEGER(Lb); break; - case kWord32AtomicLoadUint8: + case kAtomicLoadUint8: ASSEMBLE_ATOMIC_LOAD_INTEGER(Lbu); break; - case kWord32AtomicLoadInt16: + case kAtomicLoadInt16: + DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); ASSEMBLE_ATOMIC_LOAD_INTEGER(Lh); break; - case kWord32AtomicLoadUint16: + case kAtomicLoadUint16: ASSEMBLE_ATOMIC_LOAD_INTEGER(Lhu); break; - case kWord32AtomicLoadWord32: + case kAtomicLoadWord32: ASSEMBLE_ATOMIC_LOAD_INTEGER(Lw); break; - case kRiscvWord64AtomicLoadUint8: - ASSEMBLE_ATOMIC_LOAD_INTEGER(Lbu); - break; - case kRiscvWord64AtomicLoadUint16: - ASSEMBLE_ATOMIC_LOAD_INTEGER(Lhu); - break; - case kRiscvWord64AtomicLoadUint32: - ASSEMBLE_ATOMIC_LOAD_INTEGER(Lwu); - break; case kRiscvWord64AtomicLoadUint64: ASSEMBLE_ATOMIC_LOAD_INTEGER(Ld); break; - case kWord32AtomicStoreWord8: - ASSEMBLE_ATOMIC_STORE_INTEGER(Sb); - break; - case kWord32AtomicStoreWord16: - ASSEMBLE_ATOMIC_STORE_INTEGER(Sh); - break; - case kWord32AtomicStoreWord32: - ASSEMBLE_ATOMIC_STORE_INTEGER(Sw); - break; - case kRiscvWord64AtomicStoreWord8: + case kAtomicStoreWord8: ASSEMBLE_ATOMIC_STORE_INTEGER(Sb); break; - case kRiscvWord64AtomicStoreWord16: + case kAtomicStoreWord16: ASSEMBLE_ATOMIC_STORE_INTEGER(Sh); break; - case kRiscvWord64AtomicStoreWord32: + case kAtomicStoreWord32: ASSEMBLE_ATOMIC_STORE_INTEGER(Sw); break; case kRiscvWord64AtomicStoreWord64: ASSEMBLE_ATOMIC_STORE_INTEGER(Sd); break; - case kWord32AtomicExchangeInt8: + case kAtomicExchangeInt8: + DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 8, 32); break; - case kWord32AtomicExchangeUint8: - ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 8, 32); + case kAtomicExchangeUint8: + switch (AtomicWidthField::decode(opcode)) { + case AtomicWidth::kWord32: + ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 8, 32); + break; + case AtomicWidth::kWord64: + ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 8, 64); + break; + } break; - case kWord32AtomicExchangeInt16: + case kAtomicExchangeInt16: + DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 16, 32); break; - case kWord32AtomicExchangeUint16: - ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 16, 32); - break; - case kWord32AtomicExchangeWord32: - ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(Ll, Sc); - break; - case kRiscvWord64AtomicExchangeUint8: - ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 8, 64); - break; - case kRiscvWord64AtomicExchangeUint16: - ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 16, 64); + case kAtomicExchangeUint16: + switch (AtomicWidthField::decode(opcode)) { + case AtomicWidth::kWord32: + ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 16, 32); + break; + case AtomicWidth::kWord64: + ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 16, 64); + break; + } break; - case kRiscvWord64AtomicExchangeUint32: - ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 32, 64); + case kAtomicExchangeWord32: + switch (AtomicWidthField::decode(opcode)) { + case AtomicWidth::kWord32: + ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(Ll, Sc); + break; + case AtomicWidth::kWord64: + ASSEMBLE_ATOMIC_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 32, 64); + break; + } break; case kRiscvWord64AtomicExchangeUint64: ASSEMBLE_ATOMIC_EXCHANGE_INTEGER(Lld, Scd); break; - case kWord32AtomicCompareExchangeInt8: + case kAtomicCompareExchangeInt8: + DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 8, 32); break; - case kWord32AtomicCompareExchangeUint8: - ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 8, 32); + case kAtomicCompareExchangeUint8: + switch (AtomicWidthField::decode(opcode)) { + case AtomicWidth::kWord32: + ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 8, 32); + break; + case AtomicWidth::kWord64: + ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 8, 64); + break; + } break; - case kWord32AtomicCompareExchangeInt16: + case kAtomicCompareExchangeInt16: + DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, true, 16, 32); break; - case kWord32AtomicCompareExchangeUint16: - ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 16, 32); - break; - case kWord32AtomicCompareExchangeWord32: - __ Sll32(i.InputRegister(2), i.InputRegister(2), 0); - ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(Ll, Sc); - break; - case kRiscvWord64AtomicCompareExchangeUint8: - ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 8, 64); - break; - case kRiscvWord64AtomicCompareExchangeUint16: - ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 16, 64); + case kAtomicCompareExchangeUint16: + switch (AtomicWidthField::decode(opcode)) { + case AtomicWidth::kWord32: + ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Ll, Sc, false, 16, 32); + break; + case AtomicWidth::kWord64: + ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 16, 64); + break; + } break; - case kRiscvWord64AtomicCompareExchangeUint32: - ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 32, 64); + case kAtomicCompareExchangeWord32: + switch (AtomicWidthField::decode(opcode)) { + case AtomicWidth::kWord32: + __ Sll32(i.InputRegister(2), i.InputRegister(2), 0); + ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(Ll, Sc); + break; + case AtomicWidth::kWord64: + ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER_EXT(Lld, Scd, false, 32, 64); + break; + } break; case kRiscvWord64AtomicCompareExchangeUint64: ASSEMBLE_ATOMIC_COMPARE_EXCHANGE_INTEGER(Lld, Scd); break; -#define ATOMIC_BINOP_CASE(op, inst) \ - case kWord32Atomic##op##Int8: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, true, 8, inst, 32); \ - break; \ - case kWord32Atomic##op##Uint8: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, false, 8, inst, 32); \ - break; \ - case kWord32Atomic##op##Int16: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, true, 16, inst, 32); \ - break; \ - case kWord32Atomic##op##Uint16: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, false, 16, inst, 32); \ - break; \ - case kWord32Atomic##op##Word32: \ - ASSEMBLE_ATOMIC_BINOP(Ll, Sc, inst); \ - break; - ATOMIC_BINOP_CASE(Add, Add32) - ATOMIC_BINOP_CASE(Sub, Sub32) - ATOMIC_BINOP_CASE(And, And) - ATOMIC_BINOP_CASE(Or, Or) - ATOMIC_BINOP_CASE(Xor, Xor) -#undef ATOMIC_BINOP_CASE -#define ATOMIC_BINOP_CASE(op, inst) \ - case kRiscvWord64Atomic##op##Uint8: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 8, inst, 64); \ - break; \ - case kRiscvWord64Atomic##op##Uint16: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 16, inst, 64); \ - break; \ - case kRiscvWord64Atomic##op##Uint32: \ - ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 32, inst, 64); \ - break; \ - case kRiscvWord64Atomic##op##Uint64: \ - ASSEMBLE_ATOMIC_BINOP(Lld, Scd, inst); \ +#define ATOMIC_BINOP_CASE(op, inst32, inst64) \ + case kAtomic##op##Int8: \ + DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, true, 8, inst32, 32); \ + break; \ + case kAtomic##op##Uint8: \ + switch (AtomicWidthField::decode(opcode)) { \ + case AtomicWidth::kWord32: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, false, 8, inst32, 32); \ + break; \ + case AtomicWidth::kWord64: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 8, inst64, 64); \ + break; \ + } \ + break; \ + case kAtomic##op##Int16: \ + DCHECK_EQ(AtomicWidthField::decode(opcode), AtomicWidth::kWord32); \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, true, 16, inst32, 32); \ + break; \ + case kAtomic##op##Uint16: \ + switch (AtomicWidthField::decode(opcode)) { \ + case AtomicWidth::kWord32: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Ll, Sc, false, 16, inst32, 32); \ + break; \ + case AtomicWidth::kWord64: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 16, inst64, 64); \ + break; \ + } \ + break; \ + case kAtomic##op##Word32: \ + switch (AtomicWidthField::decode(opcode)) { \ + case AtomicWidth::kWord32: \ + ASSEMBLE_ATOMIC_BINOP(Ll, Sc, inst32); \ + break; \ + case AtomicWidth::kWord64: \ + ASSEMBLE_ATOMIC_BINOP_EXT(Lld, Scd, false, 32, inst64, 64); \ + break; \ + } \ + break; \ + case kRiscvWord64Atomic##op##Uint64: \ + ASSEMBLE_ATOMIC_BINOP(Lld, Scd, inst64); \ break; - ATOMIC_BINOP_CASE(Add, Add64) - ATOMIC_BINOP_CASE(Sub, Sub64) - ATOMIC_BINOP_CASE(And, And) - ATOMIC_BINOP_CASE(Or, Or) - ATOMIC_BINOP_CASE(Xor, Xor) + ATOMIC_BINOP_CASE(Add, Add32, Add64) + ATOMIC_BINOP_CASE(Sub, Sub32, Sub64) + ATOMIC_BINOP_CASE(And, And, And) + ATOMIC_BINOP_CASE(Or, Or, Or) + ATOMIC_BINOP_CASE(Xor, Xor, Xor) #undef ATOMIC_BINOP_CASE case kRiscvAssertEqual: __ Assert(eq, static_cast<AbortReason>(i.InputOperand(2).immediate()), @@ -1905,7 +1868,720 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( __ DecompressAnyTagged(result, operand); break; } + case kRiscvRvvSt: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + Register dst = i.MemoryOperand().offset() == 0 ? i.MemoryOperand().rm() + : kScratchReg; + if (i.MemoryOperand().offset() != 0) { + __ Add64(dst, i.MemoryOperand().rm(), i.MemoryOperand().offset()); + } + __ vs(i.InputSimd128Register(2), dst, 0, VSew::E8); + break; + } + case kRiscvRvvLd: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + Register src = i.MemoryOperand().offset() == 0 ? i.MemoryOperand().rm() + : kScratchReg; + if (i.MemoryOperand().offset() != 0) { + __ Add64(src, i.MemoryOperand().rm(), i.MemoryOperand().offset()); + } + __ vl(i.OutputSimd128Register(), src, 0, VSew::E8); + break; + } + case kRiscvS128Const: { + Simd128Register dst = i.OutputSimd128Register(); + uint8_t imm[16]; + *reinterpret_cast<uint64_t*>(imm) = + make_uint64(i.InputUint32(1), i.InputUint32(0)); + *(reinterpret_cast<uint64_t*>(imm) + 1) = + make_uint64(i.InputUint32(3), i.InputUint32(2)); + __ WasmRvvS128const(dst, imm); + break; + } + case kRiscvI64x2Add: { + (__ VU).set(kScratchReg, VSew::E64, Vlmul::m1); + __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI32x4Add: { + (__ VU).set(kScratchReg, VSew::E32, Vlmul::m1); + __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI16x8Add: { + (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1); + __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI16x8AddSatS: { + (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1); + __ vsadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI16x8AddSatU: { + (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1); + __ vsaddu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI8x16Add: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI8x16AddSatS: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vsadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI8x16AddSatU: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vsaddu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI64x2Sub: { + (__ VU).set(kScratchReg, VSew::E64, Vlmul::m1); + __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI32x4Sub: { + (__ VU).set(kScratchReg, VSew::E32, Vlmul::m1); + __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI16x8Sub: { + (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1); + __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI16x8SubSatS: { + (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1); + __ vssub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI16x8SubSatU: { + (__ VU).set(kScratchReg, VSew::E16, Vlmul::m1); + __ vssubu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI8x16Sub: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI8x16SubSatS: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vssub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvI8x16SubSatU: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vssubu_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvS128And: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vand_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvS128Or: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vor_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvS128Xor: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vxor_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvS128Not: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vnot_vv(i.OutputSimd128Register(), i.InputSimd128Register(0)); + break; + } + case kRiscvS128AndNot: { + (__ VU).set(kScratchReg, VSew::E8, Vlmul::m1); + __ vnot_vv(i.OutputSimd128Register(), i.InputSimd128Register(1)); + __ vand_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.OutputSimd128Register()); + break; + } + case kRiscvI32x4ExtractLane: { + __ WasmRvvExtractLane(i.OutputRegister(), i.InputSimd128Register(0), + i.InputInt8(1), E32, m1); + break; + } + case kRiscvI8x16Splat: { + (__ VU).set(kScratchReg, E8, m1); + __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0)); + break; + } + case kRiscvI16x8Splat: { + (__ VU).set(kScratchReg, E16, m1); + __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0)); + break; + } + case kRiscvI32x4Splat: { + (__ VU).set(kScratchReg, E32, m1); + __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0)); + break; + } + case kRiscvI64x2Splat: { + (__ VU).set(kScratchReg, E64, m1); + __ vmv_vx(i.OutputSimd128Register(), i.InputRegister(0)); + break; + } + case kRiscvF32x4Splat: { + (__ VU).set(kScratchReg, E32, m1); + __ fmv_x_w(kScratchReg, i.InputSingleRegister(0)); + __ vmv_vx(i.OutputSimd128Register(), kScratchReg); + break; + } + case kRiscvF64x2Splat: { + (__ VU).set(kScratchReg, E64, m1); + __ fmv_x_d(kScratchReg, i.InputDoubleRegister(0)); + __ vmv_vx(i.OutputSimd128Register(), kScratchReg); + break; + } + case kRiscvI32x4Abs: { + __ VU.set(kScratchReg, E32, m1); + __ vmv_vx(kSimd128RegZero, zero_reg); + __ vmv_vv(i.OutputSimd128Register(), i.InputSimd128Register(0)); + __ vmslt_vv(v0, i.InputSimd128Register(0), kSimd128RegZero); + __ vsub_vv(i.OutputSimd128Register(), kSimd128RegZero, + i.InputSimd128Register(0), Mask); + break; + } + case kRiscvI8x16Eq: { + __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E8, m1); + break; + } + case kRiscvI16x8Eq: { + __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E16, m1); + break; + } + case kRiscvI32x4Eq: { + __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E32, m1); + break; + } + case kRiscvI64x2Eq: { + __ WasmRvvEq(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E64, m1); + break; + } + case kRiscvI8x16Ne: { + __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E8, m1); + break; + } + case kRiscvI16x8Ne: { + __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E16, m1); + break; + } + case kRiscvI32x4Ne: { + __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E32, m1); + break; + } + case kRiscvI64x2Ne: { + __ WasmRvvNe(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E64, m1); + break; + } + case kRiscvI8x16GeS: { + __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E8, m1); + break; + } + case kRiscvI16x8GeS: { + __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E16, m1); + break; + } + case kRiscvI32x4GeS: { + __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E32, m1); + break; + } + case kRiscvI64x2GeS: { + __ WasmRvvGeS(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E64, m1); + break; + } + case kRiscvI8x16GeU: { + __ WasmRvvGeU(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E8, m1); + break; + } + case kRiscvI16x8GeU: { + __ WasmRvvGeU(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E16, m1); + break; + } + case kRiscvI32x4GeU: { + __ WasmRvvGeU(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E32, m1); + break; + } + case kRiscvI8x16GtS: { + __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E8, m1); + break; + } + case kRiscvI16x8GtS: { + __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E16, m1); + break; + } + case kRiscvI32x4GtS: { + __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E32, m1); + break; + } + case kRiscvI64x2GtS: { + __ WasmRvvGtS(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E64, m1); + break; + } + case kRiscvI8x16GtU: { + __ WasmRvvGtU(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E8, m1); + break; + } + case kRiscvI16x8GtU: { + __ WasmRvvGtU(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E16, m1); + break; + } + case kRiscvI32x4GtU: { + __ WasmRvvGtU(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1), E32, m1); + break; + } + case kRiscvI8x16Shl: { + __ VU.set(kScratchReg, E8, m1); + if (instr->InputAt(1)->IsRegister()) { + __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputRegister(1)); + } else { + __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputInt3(1)); + } + break; + } + case kRiscvI16x8Shl: { + __ VU.set(kScratchReg, E16, m1); + if (instr->InputAt(1)->IsRegister()) { + __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputRegister(1)); + } else { + __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputInt4(1)); + } + break; + } + case kRiscvI32x4Shl: { + __ VU.set(kScratchReg, E32, m1); + if (instr->InputAt(1)->IsRegister()) { + __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputRegister(1)); + } else { + __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputInt5(1)); + } + break; + } + case kRiscvI64x2Shl: { + __ VU.set(kScratchReg, E64, m1); + if (instr->InputAt(1)->IsRegister()) { + __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputRegister(1)); + } else { + if (is_int5(i.InputInt6(1))) { + __ vsll_vi(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputInt6(1)); + } else { + __ li(kScratchReg, i.InputInt6(1)); + __ vsll_vx(i.OutputSimd128Register(), i.InputSimd128Register(0), + kScratchReg); + } + } + break; + } + case kRiscvI8x16ReplaceLane: { + Simd128Register src = i.InputSimd128Register(0); + Simd128Register dst = i.OutputSimd128Register(); + __ VU.set(kScratchReg, E32, m1); + __ li(kScratchReg, 0x1 << i.InputInt8(1)); + __ vmv_sx(v0, kScratchReg); + __ vmerge_vx(dst, i.InputRegister(2), src); + break; + } + case kRiscvI16x8ReplaceLane: { + Simd128Register src = i.InputSimd128Register(0); + Simd128Register dst = i.OutputSimd128Register(); + __ VU.set(kScratchReg, E16, m1); + __ li(kScratchReg, 0x1 << i.InputInt8(1)); + __ vmv_sx(v0, kScratchReg); + __ vmerge_vx(dst, i.InputRegister(2), src); + break; + } + case kRiscvI64x2ReplaceLane: { + Simd128Register src = i.InputSimd128Register(0); + Simd128Register dst = i.OutputSimd128Register(); + __ VU.set(kScratchReg, E64, m1); + __ li(kScratchReg, 0x1 << i.InputInt8(1)); + __ vmv_sx(v0, kScratchReg); + __ vmerge_vx(dst, i.InputRegister(2), src); + break; + } + case kRiscvI32x4ReplaceLane: { + Simd128Register src = i.InputSimd128Register(0); + Simd128Register dst = i.OutputSimd128Register(); + __ VU.set(kScratchReg, E32, m1); + __ li(kScratchReg, 0x1 << i.InputInt8(1)); + __ vmv_sx(v0, kScratchReg); + __ vmerge_vx(dst, i.InputRegister(2), src); + break; + } + case kRiscvI8x16BitMask: { + Register dst = i.OutputRegister(); + Simd128Register src = i.InputSimd128Register(0); + __ VU.set(kScratchReg, E8, m1); + __ vmv_vx(kSimd128RegZero, zero_reg); + __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero); + __ VU.set(kScratchReg, E32, m1); + __ vmv_xs(dst, kSimd128ScratchReg); + break; + } + case kRiscvI16x8BitMask: { + Register dst = i.OutputRegister(); + Simd128Register src = i.InputSimd128Register(0); + __ VU.set(kScratchReg, E16, m1); + __ vmv_vx(kSimd128RegZero, zero_reg); + __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero); + __ VU.set(kScratchReg, E32, m1); + __ vmv_xs(dst, kSimd128ScratchReg); + break; + } + case kRiscvI32x4BitMask: { + Register dst = i.OutputRegister(); + Simd128Register src = i.InputSimd128Register(0); + __ VU.set(kScratchReg, E32, m1); + __ vmv_vx(kSimd128RegZero, zero_reg); + __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero); + __ vmv_xs(dst, kSimd128ScratchReg); + break; + } + case kRiscvI64x2BitMask: { + Register dst = i.OutputRegister(); + Simd128Register src = i.InputSimd128Register(0); + __ VU.set(kScratchReg, E64, m1); + __ vmv_vx(kSimd128RegZero, zero_reg); + __ vmslt_vv(kSimd128ScratchReg, src, kSimd128RegZero); + __ VU.set(kScratchReg, E32, m1); + __ vmv_xs(dst, kSimd128ScratchReg); + break; + } + case kRiscvV128AnyTrue: { + __ VU.set(kScratchReg, E8, m1); + Register dst = i.OutputRegister(); + Label t; + __ vmv_sx(kSimd128ScratchReg, zero_reg); + __ vredmaxu_vs(kSimd128ScratchReg, i.InputSimd128Register(0), + kSimd128ScratchReg); + __ vmv_xs(dst, kSimd128ScratchReg); + __ beq(dst, zero_reg, &t); + __ li(dst, 1); + __ bind(&t); + break; + } + case kRiscvI64x2AllTrue: { + __ VU.set(kScratchReg, E64, m1); + Register dst = i.OutputRegister(); + Label all_true; + __ li(kScratchReg, -1); + __ vmv_sx(kSimd128ScratchReg, kScratchReg); + __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0), + kSimd128ScratchReg); + __ vmv_xs(dst, kSimd128ScratchReg); + __ beqz(dst, &all_true); + __ li(dst, 1); + __ bind(&all_true); + break; + } + case kRiscvI32x4AllTrue: { + __ VU.set(kScratchReg, E32, m1); + Register dst = i.OutputRegister(); + Label all_true; + __ li(kScratchReg, -1); + __ vmv_sx(kSimd128ScratchReg, kScratchReg); + __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0), + kSimd128ScratchReg); + __ vmv_xs(dst, kSimd128ScratchReg); + __ beqz(dst, &all_true); + __ li(dst, 1); + __ bind(&all_true); + break; + } + case kRiscvI16x8AllTrue: { + __ VU.set(kScratchReg, E16, m1); + Register dst = i.OutputRegister(); + Label all_true; + __ li(kScratchReg, -1); + __ vmv_sx(kSimd128ScratchReg, kScratchReg); + __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0), + kSimd128ScratchReg); + __ vmv_xs(dst, kSimd128ScratchReg); + __ beqz(dst, &all_true); + __ li(dst, 1); + __ bind(&all_true); + break; + } + case kRiscvI8x16AllTrue: { + __ VU.set(kScratchReg, E8, m1); + Register dst = i.OutputRegister(); + Label all_true; + __ li(kScratchReg, -1); + __ vmv_sx(kSimd128ScratchReg, kScratchReg); + __ vredminu_vs(kSimd128ScratchReg, i.InputSimd128Register(0), + kSimd128ScratchReg); + __ vmv_xs(dst, kSimd128ScratchReg); + __ beqz(dst, &all_true); + __ li(dst, 1); + __ bind(&all_true); + break; + } + case kRiscvI8x16Shuffle: { + VRegister dst = i.OutputSimd128Register(), + src0 = i.InputSimd128Register(0), + src1 = i.InputSimd128Register(1); + + int64_t imm1 = make_uint64(i.InputInt32(3), i.InputInt32(2)); + int64_t imm2 = make_uint64(i.InputInt32(5), i.InputInt32(4)); + __ VU.set(kScratchReg, VSew::E64, Vlmul::m1); + __ li(kScratchReg, 1); + __ vmv_vx(v0, kScratchReg); + __ li(kScratchReg, imm1); + __ vmerge_vx(kSimd128ScratchReg, kScratchReg, kSimd128ScratchReg); + __ li(kScratchReg, imm2); + __ vsll_vi(v0, v0, 1); + __ vmerge_vx(kSimd128ScratchReg, kScratchReg, kSimd128ScratchReg); + + __ VU.set(kScratchReg, E8, m1); + if (dst == src0) { + __ vmv_vv(kSimd128ScratchReg2, src0); + src0 = kSimd128ScratchReg2; + } else if (dst == src1) { + __ vmv_vv(kSimd128ScratchReg2, src1); + src1 = kSimd128ScratchReg2; + } + __ vrgather_vv(dst, src0, kSimd128ScratchReg); + __ vadd_vi(kSimd128ScratchReg, kSimd128ScratchReg, -16); + __ vrgather_vv(kSimd128ScratchReg, src1, kSimd128ScratchReg); + __ vor_vv(dst, dst, kSimd128ScratchReg); + break; + } + case kRiscvF32x4Abs: { + __ VU.set(kScratchReg, VSew::E32, Vlmul::m1); + __ vfabs_vv(i.OutputSimd128Register(), i.InputSimd128Register(0)); + break; + } + case kRiscvF64x2Abs: { + __ VU.set(kScratchReg, VSew::E64, Vlmul::m1); + __ vfabs_vv(i.OutputSimd128Register(), i.InputSimd128Register(0)); + break; + } + case kRiscvF32x4Neg: { + __ VU.set(kScratchReg, VSew::E32, Vlmul::m1); + __ vfneg_vv(i.OutputSimd128Register(), i.InputSimd128Register(0)); + break; + } + case kRiscvF64x2Neg: { + __ VU.set(kScratchReg, VSew::E64, Vlmul::m1); + __ vfneg_vv(i.OutputSimd128Register(), i.InputSimd128Register(0)); + break; + } + case kRiscvF32x4DemoteF64x2Zero: { + __ VU.set(kScratchReg, E32, m1); + __ vfncvt_f_f_w(i.OutputSimd128Register(), i.InputSimd128Register(0)); + __ vmv_vi(v0, 12); + __ vmerge_vx(i.OutputSimd128Register(), zero_reg, + i.OutputSimd128Register()); + break; + } + case kRiscvF32x4Add: { + __ VU.set(kScratchReg, E32, m1); + __ vfadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvF32x4Sub: { + __ VU.set(kScratchReg, E32, m1); + __ vfsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvF64x2Add: { + __ VU.set(kScratchReg, E64, m1); + __ vfadd_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvF64x2Sub: { + __ VU.set(kScratchReg, E64, m1); + __ vfsub_vv(i.OutputSimd128Register(), i.InputSimd128Register(0), + i.InputSimd128Register(1)); + break; + } + case kRiscvF32x4Ceil: { + __ Ceil_f(i.OutputSimd128Register(), i.InputSimd128Register(0), + kScratchReg, kSimd128ScratchReg); + break; + } + case kRiscvF64x2Ceil: { + __ Ceil_d(i.OutputSimd128Register(), i.InputSimd128Register(0), + kScratchReg, kSimd128ScratchReg); + break; + } + case kRiscvF32x4Floor: { + __ Floor_f(i.OutputSimd128Register(), i.InputSimd128Register(0), + kScratchReg, kSimd128ScratchReg); + break; + } + case kRiscvF64x2Floor: { + __ Floor_d(i.OutputSimd128Register(), i.InputSimd128Register(0), + kScratchReg, kSimd128ScratchReg); + break; + } + case kRiscvS128Select: { + __ VU.set(kScratchReg, E8, m1); + __ vand_vv(kSimd128ScratchReg, i.InputSimd128Register(1), + i.InputSimd128Register(0)); + __ vnot_vv(kSimd128ScratchReg2, i.InputSimd128Register(0)); + __ vand_vv(kSimd128ScratchReg2, i.InputSimd128Register(2), + kSimd128ScratchReg2); + __ vor_vv(i.OutputSimd128Register(), kSimd128ScratchReg, + kSimd128ScratchReg2); + break; + } + case kRiscvF32x4UConvertI32x4: { + __ VU.set(kScratchReg, E32, m1); + __ VU.set(RoundingMode::RTZ); + __ vfcvt_f_xu_v(i.OutputSimd128Register(), i.InputSimd128Register(0)); + break; + } + case kRiscvF32x4SConvertI32x4: { + __ VU.set(kScratchReg, E32, m1); + __ VU.set(RoundingMode::RTZ); + __ vfcvt_f_x_v(i.OutputSimd128Register(), i.InputSimd128Register(0)); + break; + } + case kRiscvF32x4Div: { + __ VU.set(kScratchReg, E32, m1); + __ VU.set(RoundingMode::RTZ); + __ vfdiv_vv(i.OutputSimd128Register(), i.InputSimd128Register(1), + i.InputSimd128Register(0)); + break; + } + case kRiscvF32x4Mul: { + __ VU.set(kScratchReg, E32, m1); + __ VU.set(RoundingMode::RTZ); + __ vfmul_vv(i.OutputSimd128Register(), i.InputSimd128Register(1), + i.InputSimd128Register(0)); + break; + } + case kRiscvF32x4Eq: { + __ VU.set(kScratchReg, E32, m1); + __ vmfeq_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0)); + __ vmv_vx(i.OutputSimd128Register(), zero_reg); + __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register()); + break; + } + case kRiscvF32x4Ne: { + __ VU.set(kScratchReg, E32, m1); + __ vmfne_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0)); + __ vmv_vx(i.OutputSimd128Register(), zero_reg); + __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register()); + break; + } + case kRiscvF32x4Lt: { + __ VU.set(kScratchReg, E32, m1); + __ vmflt_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0)); + __ vmv_vx(i.OutputSimd128Register(), zero_reg); + __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register()); + break; + } + case kRiscvF32x4Le: { + __ VU.set(kScratchReg, E32, m1); + __ vmfle_vv(v0, i.InputSimd128Register(1), i.InputSimd128Register(0)); + __ vmv_vx(i.OutputSimd128Register(), zero_reg); + __ vmerge_vi(i.OutputSimd128Register(), -1, i.OutputSimd128Register()); + break; + } + case kRiscvF32x4Max: { + __ VU.set(kScratchReg, E32, m1); + const int32_t kNaN = 0x7FC00000; + __ vmfeq_vv(v0, i.InputSimd128Register(0), i.InputSimd128Register(0)); + __ vmfeq_vv(kSimd128ScratchReg, i.InputSimd128Register(1), + i.InputSimd128Register(1)); + __ vand_vv(v0, v0, kSimd128ScratchReg); + __ li(kScratchReg, kNaN); + __ vmv_vx(kSimd128ScratchReg, kScratchReg); + __ vfmax_vv(kSimd128ScratchReg, i.InputSimd128Register(1), + i.InputSimd128Register(0), Mask); + __ vmv_vv(i.OutputSimd128Register(), kSimd128ScratchReg); + break; + } + case kRiscvF32x4Min: { + __ VU.set(kScratchReg, E32, m1); + const int32_t kNaN = 0x7FC00000; + __ vmfeq_vv(v0, i.InputSimd128Register(0), i.InputSimd128Register(0)); + __ vmfeq_vv(kSimd128ScratchReg, i.InputSimd128Register(1), + i.InputSimd128Register(1)); + __ vand_vv(v0, v0, kSimd128ScratchReg); + __ li(kScratchReg, kNaN); + __ vmv_vx(kSimd128ScratchReg, kScratchReg); + __ vfmin_vv(kSimd128ScratchReg, i.InputSimd128Register(1), + i.InputSimd128Register(0), Mask); + __ vmv_vv(i.OutputSimd128Register(), kSimd128ScratchReg); + break; + } default: +#ifdef DEBUG + switch (arch_opcode) { +#define Print(name) \ + case k##name: \ + printf("k%s", #name); \ + break; + TARGET_ARCH_OPCODE_LIST(Print); +#undef Print + default: + break; + } +#endif UNIMPLEMENTED(); } return kSuccess; @@ -1916,6 +2592,19 @@ CodeGenerator::CodeGenResult CodeGenerator::AssembleArchInstruction( << "\""; \ UNIMPLEMENTED(); +bool IsInludeEqual(Condition cc) { + switch (cc) { + case equal: + case greater_equal: + case less_equal: + case Uless_equal: + case Ugreater_equal: + return true; + default: + return false; + } +} + void AssembleBranchToLabels(CodeGenerator* gen, TurboAssembler* tasm, Instruction* instr, FlagsCondition condition, Label* tlabel, Label* flabel, bool fallthru) { @@ -1952,7 +2641,6 @@ void AssembleBranchToLabels(CodeGenerator* gen, TurboAssembler* tasm, break; default: UNSUPPORTED_COND(instr->arch_opcode(), condition); - break; } } else if (instr->arch_opcode() == kRiscvMulOvf32) { // Overflow occurs if overflow register is not zero @@ -1965,14 +2653,17 @@ void AssembleBranchToLabels(CodeGenerator* gen, TurboAssembler* tasm, break; default: UNSUPPORTED_COND(kRiscvMulOvf32, condition); - break; } } else if (instr->arch_opcode() == kRiscvCmp) { cc = FlagsConditionToConditionCmp(condition); __ Branch(tlabel, cc, i.InputRegister(0), i.InputOperand(1)); } else if (instr->arch_opcode() == kRiscvCmpZero) { cc = FlagsConditionToConditionCmp(condition); - __ Branch(tlabel, cc, i.InputRegister(0), Operand(zero_reg)); + if (i.InputOrZeroRegister(0) == zero_reg && IsInludeEqual(cc)) { + __ Branch(tlabel); + } else if (i.InputOrZeroRegister(0) != zero_reg) { + __ Branch(tlabel, cc, i.InputRegister(0), Operand(zero_reg)); + } } else if (instr->arch_opcode() == kArchStackPointerGreaterThan) { cc = FlagsConditionToConditionCmp(condition); Register lhs_register = sp; @@ -2011,110 +2702,6 @@ void CodeGenerator::AssembleArchBranch(Instruction* instr, BranchInfo* branch) { branch->fallthru); } -void CodeGenerator::AssembleBranchPoisoning(FlagsCondition condition, - Instruction* instr) { - // TODO(jarin) Handle float comparisons (kUnordered[Not]Equal). - if (condition == kUnorderedEqual || condition == kUnorderedNotEqual) { - return; - } - - RiscvOperandConverter i(this, instr); - condition = NegateFlagsCondition(condition); - - switch (instr->arch_opcode()) { - case kRiscvCmp: { - __ CompareI(kScratchReg, i.InputRegister(0), i.InputOperand(1), - FlagsConditionToConditionCmp(condition)); - __ LoadZeroIfConditionNotZero(kSpeculationPoisonRegister, kScratchReg); - } - return; - case kRiscvCmpZero: { - __ CompareI(kScratchReg, i.InputRegister(0), Operand(zero_reg), - FlagsConditionToConditionCmp(condition)); - __ LoadZeroIfConditionNotZero(kSpeculationPoisonRegister, kScratchReg); - } - return; - case kRiscvTst: { - switch (condition) { - case kEqual: - __ LoadZeroIfConditionZero(kSpeculationPoisonRegister, kScratchReg); - break; - case kNotEqual: - __ LoadZeroIfConditionNotZero(kSpeculationPoisonRegister, - kScratchReg); - break; - default: - UNREACHABLE(); - } - } - return; - case kRiscvAdd64: - case kRiscvSub64: { - // Check for overflow creates 1 or 0 for result. - __ Srl64(kScratchReg, i.OutputRegister(), 63); - __ Srl32(kScratchReg2, i.OutputRegister(), 31); - __ Xor(kScratchReg2, kScratchReg, kScratchReg2); - switch (condition) { - case kOverflow: - __ LoadZeroIfConditionNotZero(kSpeculationPoisonRegister, - kScratchReg2); - break; - case kNotOverflow: - __ LoadZeroIfConditionZero(kSpeculationPoisonRegister, kScratchReg2); - break; - default: - UNSUPPORTED_COND(instr->arch_opcode(), condition); - } - } - return; - case kRiscvAddOvf64: - case kRiscvSubOvf64: { - // Overflow occurs if overflow register is negative - __ Slt(kScratchReg2, kScratchReg, zero_reg); - switch (condition) { - case kOverflow: - __ LoadZeroIfConditionNotZero(kSpeculationPoisonRegister, - kScratchReg2); - break; - case kNotOverflow: - __ LoadZeroIfConditionZero(kSpeculationPoisonRegister, kScratchReg2); - break; - default: - UNSUPPORTED_COND(instr->arch_opcode(), condition); - } - } - return; - case kRiscvMulOvf32: { - // Overflow occurs if overflow register is not zero - switch (condition) { - case kOverflow: - __ LoadZeroIfConditionNotZero(kSpeculationPoisonRegister, - kScratchReg); - break; - case kNotOverflow: - __ LoadZeroIfConditionZero(kSpeculationPoisonRegister, kScratchReg); - break; - default: - UNSUPPORTED_COND(instr->arch_opcode(), condition); - } - } - return; - case kRiscvCmpS: - case kRiscvCmpD: { - bool predicate; - FlagsConditionToConditionCmpFPU(&predicate, condition); - if (predicate) { - __ LoadZeroIfConditionNotZero(kSpeculationPoisonRegister, kScratchReg); - } else { - __ LoadZeroIfConditionZero(kSpeculationPoisonRegister, kScratchReg); - } - } - return; - default: - UNREACHABLE(); - } -} - #undef UNSUPPORTED_COND void CodeGenerator::AssembleArchDeoptBranch(Instruction* instr, @@ -2489,7 +3076,6 @@ void CodeGenerator::AssembleConstructFrame() { __ RecordComment("-- OSR entrypoint --"); osr_pc_offset_ = __ pc_offset(); required_slots -= osr_helper()->UnoptimizedFrameSlots(); - ResetSpeculationPoison(); } const RegList saves = call_descriptor->CalleeSavedRegisters(); @@ -2652,7 +3238,18 @@ void CodeGenerator::AssembleReturn(InstructionOperand* additional_pop_count) { void CodeGenerator::FinishCode() { __ ForceConstantPoolEmissionWithoutJump(); } void CodeGenerator::PrepareForDeoptimizationExits( - ZoneDeque<DeoptimizationExit*>* exits) {} + ZoneDeque<DeoptimizationExit*>* exits) { + __ ForceConstantPoolEmissionWithoutJump(); + int total_size = 0; + for (DeoptimizationExit* exit : deoptimization_exits_) { + total_size += (exit->kind() == DeoptimizeKind::kLazy) + ? Deoptimizer::kLazyDeoptExitSize + : Deoptimizer::kNonLazyDeoptExitSize; + } + + __ CheckTrampolinePoolQuick(total_size); + DCHECK(Deoptimizer::kSupportsFixedDeoptExitSizes); +} void CodeGenerator::AssembleMove(InstructionOperand* source, InstructionOperand* destination) { @@ -2735,7 +3332,6 @@ void CodeGenerator::AssembleMove(InstructionOperand* source, } case Constant::kRpoNumber: UNREACHABLE(); // TODO(titzer): loading RPO numbers - break; } if (destination->IsStackSlot()) __ Sd(dst, g.ToMemOperand(destination)); } else if (src.type() == Constant::kFloat32) { @@ -2765,7 +3361,21 @@ void CodeGenerator::AssembleMove(InstructionOperand* source, } else if (source->IsFPRegister()) { MachineRepresentation rep = LocationOperand::cast(source)->representation(); if (rep == MachineRepresentation::kSimd128) { - UNIMPLEMENTED(); + VRegister src = g.ToSimd128Register(source); + if (destination->IsSimd128Register()) { + VRegister dst = g.ToSimd128Register(destination); + __ vmv_vv(dst, src); + } else { + DCHECK(destination->IsSimd128StackSlot()); + Register dst = g.ToMemOperand(destination).offset() == 0 + ? g.ToMemOperand(destination).rm() + : kScratchReg; + if (g.ToMemOperand(destination).offset() != 0) { + __ Add64(dst, g.ToMemOperand(destination).rm(), + g.ToMemOperand(destination).offset()); + } + __ vs(src, dst, 0, E8); + } } else { FPURegister src = g.ToDoubleRegister(source); if (destination->IsFPRegister()) { @@ -2786,7 +3396,25 @@ void CodeGenerator::AssembleMove(InstructionOperand* source, MemOperand src = g.ToMemOperand(source); MachineRepresentation rep = LocationOperand::cast(source)->representation(); if (rep == MachineRepresentation::kSimd128) { - UNIMPLEMENTED(); + Register src_reg = src.offset() == 0 ? src.rm() : kScratchReg; + if (src.offset() != 0) { + __ Add64(src_reg, src.rm(), src.offset()); + } + if (destination->IsSimd128Register()) { + __ vl(g.ToSimd128Register(destination), src_reg, 0, E8); + } else { + DCHECK(destination->IsSimd128StackSlot()); + VRegister temp = kSimd128ScratchReg; + Register dst = g.ToMemOperand(destination).offset() == 0 + ? g.ToMemOperand(destination).rm() + : kScratchReg; + if (g.ToMemOperand(destination).offset() != 0) { + __ Add64(dst, g.ToMemOperand(destination).rm(), + g.ToMemOperand(destination).offset()); + } + __ vl(temp, src_reg, 0, E8); + __ vs(temp, dst, 0, E8); + } } else { if (destination->IsFPRegister()) { if (rep == MachineRepresentation::kFloat32) { diff --git a/chromium/v8/src/compiler/backend/riscv64/instruction-codes-riscv64.h b/chromium/v8/src/compiler/backend/riscv64/instruction-codes-riscv64.h index 2f51c2b1c79..f3aa0f29a83 100644 --- a/chromium/v8/src/compiler/backend/riscv64/instruction-codes-riscv64.h +++ b/chromium/v8/src/compiler/backend/riscv64/instruction-codes-riscv64.h @@ -9,423 +9,400 @@ namespace v8 { namespace internal { namespace compiler { +// Opcodes that support a MemoryAccessMode. +#define TARGET_ARCH_OPCODE_WITH_MEMORY_ACCESS_MODE_LIST(V) // None. + // RISC-V-specific opcodes that specify which assembly sequence to emit. // Most opcodes specify a single instruction. -#define TARGET_ARCH_OPCODE_LIST(V) \ - V(RiscvAdd32) \ - V(RiscvAdd64) \ - V(RiscvAddOvf64) \ - V(RiscvSub32) \ - V(RiscvSub64) \ - V(RiscvSubOvf64) \ - V(RiscvMul32) \ - V(RiscvMulOvf32) \ - V(RiscvMulHigh32) \ - V(RiscvMulHigh64) \ - V(RiscvMulHighU32) \ - V(RiscvMul64) \ - V(RiscvDiv32) \ - V(RiscvDiv64) \ - V(RiscvDivU32) \ - V(RiscvDivU64) \ - V(RiscvMod32) \ - V(RiscvMod64) \ - V(RiscvModU32) \ - V(RiscvModU64) \ - V(RiscvAnd) \ - V(RiscvAnd32) \ - V(RiscvOr) \ - V(RiscvOr32) \ - V(RiscvNor) \ - V(RiscvNor32) \ - V(RiscvXor) \ - V(RiscvXor32) \ - V(RiscvClz32) \ - V(RiscvShl32) \ - V(RiscvShr32) \ - V(RiscvSar32) \ - V(RiscvZeroExtendWord) \ - V(RiscvSignExtendWord) \ - V(RiscvClz64) \ - V(RiscvCtz32) \ - V(RiscvCtz64) \ - V(RiscvPopcnt32) \ - V(RiscvPopcnt64) \ - V(RiscvShl64) \ - V(RiscvShr64) \ - V(RiscvSar64) \ - V(RiscvRor32) \ - V(RiscvRor64) \ - V(RiscvMov) \ - V(RiscvTst) \ - V(RiscvCmp) \ - V(RiscvCmpZero) \ - V(RiscvCmpS) \ - V(RiscvAddS) \ - V(RiscvSubS) \ - V(RiscvMulS) \ - V(RiscvDivS) \ - V(RiscvModS) \ - V(RiscvAbsS) \ - V(RiscvNegS) \ - V(RiscvSqrtS) \ - V(RiscvMaxS) \ - V(RiscvMinS) \ - V(RiscvCmpD) \ - V(RiscvAddD) \ - V(RiscvSubD) \ - V(RiscvMulD) \ - V(RiscvDivD) \ - V(RiscvModD) \ - V(RiscvAbsD) \ - V(RiscvNegD) \ - V(RiscvSqrtD) \ - V(RiscvMaxD) \ - V(RiscvMinD) \ - V(RiscvFloat64RoundDown) \ - V(RiscvFloat64RoundTruncate) \ - V(RiscvFloat64RoundUp) \ - V(RiscvFloat64RoundTiesEven) \ - V(RiscvFloat32RoundDown) \ - V(RiscvFloat32RoundTruncate) \ - V(RiscvFloat32RoundUp) \ - V(RiscvFloat32RoundTiesEven) \ - V(RiscvCvtSD) \ - V(RiscvCvtDS) \ - V(RiscvTruncWD) \ - V(RiscvRoundWD) \ - V(RiscvFloorWD) \ - V(RiscvCeilWD) \ - V(RiscvTruncWS) \ - V(RiscvRoundWS) \ - V(RiscvFloorWS) \ - V(RiscvCeilWS) \ - V(RiscvTruncLS) \ - V(RiscvTruncLD) \ - V(RiscvTruncUwD) \ - V(RiscvTruncUwS) \ - V(RiscvTruncUlS) \ - V(RiscvTruncUlD) \ - V(RiscvCvtDW) \ - V(RiscvCvtSL) \ - V(RiscvCvtSW) \ - V(RiscvCvtSUw) \ - V(RiscvCvtSUl) \ - V(RiscvCvtDL) \ - V(RiscvCvtDUw) \ - V(RiscvCvtDUl) \ - V(RiscvLb) \ - V(RiscvLbu) \ - V(RiscvSb) \ - V(RiscvLh) \ - V(RiscvUlh) \ - V(RiscvLhu) \ - V(RiscvUlhu) \ - V(RiscvSh) \ - V(RiscvUsh) \ - V(RiscvLd) \ - V(RiscvUld) \ - V(RiscvLw) \ - V(RiscvUlw) \ - V(RiscvLwu) \ - V(RiscvUlwu) \ - V(RiscvSw) \ - V(RiscvUsw) \ - V(RiscvSd) \ - V(RiscvUsd) \ - V(RiscvLoadFloat) \ - V(RiscvULoadFloat) \ - V(RiscvStoreFloat) \ - V(RiscvUStoreFloat) \ - V(RiscvLoadDouble) \ - V(RiscvULoadDouble) \ - V(RiscvStoreDouble) \ - V(RiscvUStoreDouble) \ - V(RiscvBitcastDL) \ - V(RiscvBitcastLD) \ - V(RiscvBitcastInt32ToFloat32) \ - V(RiscvBitcastFloat32ToInt32) \ - V(RiscvFloat64ExtractLowWord32) \ - V(RiscvFloat64ExtractHighWord32) \ - V(RiscvFloat64InsertLowWord32) \ - V(RiscvFloat64InsertHighWord32) \ - V(RiscvFloat32Max) \ - V(RiscvFloat64Max) \ - V(RiscvFloat32Min) \ - V(RiscvFloat64Min) \ - V(RiscvFloat64SilenceNaN) \ - V(RiscvPush) \ - V(RiscvPeek) \ - V(RiscvByteSwap64) \ - V(RiscvByteSwap32) \ - V(RiscvStoreToStackSlot) \ - V(RiscvStackClaim) \ - V(RiscvSignExtendByte) \ - V(RiscvSignExtendShort) \ - V(RiscvSync) \ - V(RiscvAssertEqual) \ - V(RiscvS128Const) \ - V(RiscvS128Zero) \ - V(RiscvS128AllOnes) \ - V(RiscvI32x4Splat) \ - V(RiscvI32x4ExtractLane) \ - V(RiscvI32x4ReplaceLane) \ - V(RiscvI32x4Add) \ - V(RiscvI32x4Sub) \ - V(RiscvF64x2Abs) \ - V(RiscvF64x2Neg) \ - V(RiscvF32x4Splat) \ - V(RiscvF32x4ExtractLane) \ - V(RiscvF32x4ReplaceLane) \ - V(RiscvF32x4SConvertI32x4) \ - V(RiscvF32x4UConvertI32x4) \ - V(RiscvI64x2SConvertI32x4Low) \ - V(RiscvI64x2SConvertI32x4High) \ - V(RiscvI64x2UConvertI32x4Low) \ - V(RiscvI64x2UConvertI32x4High) \ - V(RiscvI32x4Mul) \ - V(RiscvI32x4MaxS) \ - V(RiscvI32x4MinS) \ - V(RiscvI32x4Eq) \ - V(RiscvI32x4Ne) \ - V(RiscvI32x4Shl) \ - V(RiscvI32x4ShrS) \ - V(RiscvI32x4ShrU) \ - V(RiscvI32x4MaxU) \ - V(RiscvI32x4MinU) \ - V(RiscvI64x2GtS) \ - V(RiscvI64x2GeS) \ - V(RiscvI64x2Eq) \ - V(RiscvI64x2Ne) \ - V(RiscvF64x2Sqrt) \ - V(RiscvF64x2Add) \ - V(RiscvF64x2Sub) \ - V(RiscvF64x2Mul) \ - V(RiscvF64x2Div) \ - V(RiscvF64x2Min) \ - V(RiscvF64x2Max) \ - V(RiscvF64x2ConvertLowI32x4S) \ - V(RiscvF64x2ConvertLowI32x4U) \ - V(RiscvF64x2PromoteLowF32x4) \ - V(RiscvF64x2Eq) \ - V(RiscvF64x2Ne) \ - V(RiscvF64x2Lt) \ - V(RiscvF64x2Le) \ - V(RiscvF64x2Splat) \ - V(RiscvF64x2ExtractLane) \ - V(RiscvF64x2ReplaceLane) \ - V(RiscvF64x2Pmin) \ - V(RiscvF64x2Pmax) \ - V(RiscvF64x2Ceil) \ - V(RiscvF64x2Floor) \ - V(RiscvF64x2Trunc) \ - V(RiscvF64x2NearestInt) \ - V(RiscvI64x2Splat) \ - V(RiscvI64x2ExtractLane) \ - V(RiscvI64x2ReplaceLane) \ - V(RiscvI64x2Add) \ - V(RiscvI64x2Sub) \ - V(RiscvI64x2Mul) \ - V(RiscvI64x2Abs) \ - V(RiscvI64x2Neg) \ - V(RiscvI64x2Shl) \ - V(RiscvI64x2ShrS) \ - V(RiscvI64x2ShrU) \ - V(RiscvI64x2BitMask) \ - V(RiscvF32x4Abs) \ - V(RiscvF32x4Neg) \ - V(RiscvF32x4Sqrt) \ - V(RiscvF32x4RecipApprox) \ - V(RiscvF32x4RecipSqrtApprox) \ - V(RiscvF32x4Add) \ - V(RiscvF32x4Sub) \ - V(RiscvF32x4Mul) \ - V(RiscvF32x4Div) \ - V(RiscvF32x4Max) \ - V(RiscvF32x4Min) \ - V(RiscvF32x4Eq) \ - V(RiscvF32x4Ne) \ - V(RiscvF32x4Lt) \ - V(RiscvF32x4Le) \ - V(RiscvF32x4Pmin) \ - V(RiscvF32x4Pmax) \ - V(RiscvF32x4DemoteF64x2Zero) \ - V(RiscvF32x4Ceil) \ - V(RiscvF32x4Floor) \ - V(RiscvF32x4Trunc) \ - V(RiscvF32x4NearestInt) \ - V(RiscvI32x4SConvertF32x4) \ - V(RiscvI32x4UConvertF32x4) \ - V(RiscvI32x4Neg) \ - V(RiscvI32x4GtS) \ - V(RiscvI32x4GeS) \ - V(RiscvI32x4GtU) \ - V(RiscvI32x4GeU) \ - V(RiscvI32x4Abs) \ - V(RiscvI32x4BitMask) \ - V(RiscvI32x4DotI16x8S) \ - V(RiscvI32x4TruncSatF64x2SZero) \ - V(RiscvI32x4TruncSatF64x2UZero) \ - V(RiscvI16x8Splat) \ - V(RiscvI16x8ExtractLaneU) \ - V(RiscvI16x8ExtractLaneS) \ - V(RiscvI16x8ReplaceLane) \ - V(RiscvI16x8Neg) \ - V(RiscvI16x8Shl) \ - V(RiscvI16x8ShrS) \ - V(RiscvI16x8ShrU) \ - V(RiscvI16x8Add) \ - V(RiscvI16x8AddSatS) \ - V(RiscvI16x8Sub) \ - V(RiscvI16x8SubSatS) \ - V(RiscvI16x8Mul) \ - V(RiscvI16x8MaxS) \ - V(RiscvI16x8MinS) \ - V(RiscvI16x8Eq) \ - V(RiscvI16x8Ne) \ - V(RiscvI16x8GtS) \ - V(RiscvI16x8GeS) \ - V(RiscvI16x8AddSatU) \ - V(RiscvI16x8SubSatU) \ - V(RiscvI16x8MaxU) \ - V(RiscvI16x8MinU) \ - V(RiscvI16x8GtU) \ - V(RiscvI16x8GeU) \ - V(RiscvI16x8RoundingAverageU) \ - V(RiscvI16x8Q15MulRSatS) \ - V(RiscvI16x8Abs) \ - V(RiscvI16x8BitMask) \ - V(RiscvI8x16Splat) \ - V(RiscvI8x16ExtractLaneU) \ - V(RiscvI8x16ExtractLaneS) \ - V(RiscvI8x16ReplaceLane) \ - V(RiscvI8x16Neg) \ - V(RiscvI8x16Shl) \ - V(RiscvI8x16ShrS) \ - V(RiscvI8x16Add) \ - V(RiscvI8x16AddSatS) \ - V(RiscvI8x16Sub) \ - V(RiscvI8x16SubSatS) \ - V(RiscvI8x16MaxS) \ - V(RiscvI8x16MinS) \ - V(RiscvI8x16Eq) \ - V(RiscvI8x16Ne) \ - V(RiscvI8x16GtS) \ - V(RiscvI8x16GeS) \ - V(RiscvI8x16ShrU) \ - V(RiscvI8x16AddSatU) \ - V(RiscvI8x16SubSatU) \ - V(RiscvI8x16MaxU) \ - V(RiscvI8x16MinU) \ - V(RiscvI8x16GtU) \ - V(RiscvI8x16GeU) \ - V(RiscvI8x16RoundingAverageU) \ - V(RiscvI8x16Abs) \ - V(RiscvI8x16BitMask) \ - V(RiscvI8x16Popcnt) \ - V(RiscvS128And) \ - V(RiscvS128Or) \ - V(RiscvS128Xor) \ - V(RiscvS128Not) \ - V(RiscvS128Select) \ - V(RiscvS128AndNot) \ - V(RiscvI32x4AllTrue) \ - V(RiscvI16x8AllTrue) \ - V(RiscvV128AnyTrue) \ - V(RiscvI8x16AllTrue) \ - V(RiscvI64x2AllTrue) \ - V(RiscvS32x4InterleaveRight) \ - V(RiscvS32x4InterleaveLeft) \ - V(RiscvS32x4PackEven) \ - V(RiscvS32x4PackOdd) \ - V(RiscvS32x4InterleaveEven) \ - V(RiscvS32x4InterleaveOdd) \ - V(RiscvS32x4Shuffle) \ - V(RiscvS16x8InterleaveRight) \ - V(RiscvS16x8InterleaveLeft) \ - V(RiscvS16x8PackEven) \ - V(RiscvS16x8PackOdd) \ - V(RiscvS16x8InterleaveEven) \ - V(RiscvS16x8InterleaveOdd) \ - V(RiscvS16x4Reverse) \ - V(RiscvS16x2Reverse) \ - V(RiscvS8x16InterleaveRight) \ - V(RiscvS8x16InterleaveLeft) \ - V(RiscvS8x16PackEven) \ - V(RiscvS8x16PackOdd) \ - V(RiscvS8x16InterleaveEven) \ - V(RiscvS8x16InterleaveOdd) \ - V(RiscvS8x16Shuffle) \ - V(RiscvI8x16Swizzle) \ - V(RiscvS8x16Concat) \ - V(RiscvS8x8Reverse) \ - V(RiscvS8x4Reverse) \ - V(RiscvS8x2Reverse) \ - V(RiscvS128Load8Splat) \ - V(RiscvS128Load16Splat) \ - V(RiscvS128Load32Splat) \ - V(RiscvS128Load64Splat) \ - V(RiscvS128Load8x8S) \ - V(RiscvS128Load8x8U) \ - V(RiscvS128Load16x4S) \ - V(RiscvS128Load16x4U) \ - V(RiscvS128Load32x2S) \ - V(RiscvS128Load32x2U) \ - V(RiscvS128LoadLane) \ - V(RiscvS128StoreLane) \ - V(RiscvMsaLd) \ - V(RiscvMsaSt) \ - V(RiscvI32x4SConvertI16x8Low) \ - V(RiscvI32x4SConvertI16x8High) \ - V(RiscvI32x4UConvertI16x8Low) \ - V(RiscvI32x4UConvertI16x8High) \ - V(RiscvI16x8SConvertI8x16Low) \ - V(RiscvI16x8SConvertI8x16High) \ - V(RiscvI16x8SConvertI32x4) \ - V(RiscvI16x8UConvertI32x4) \ - V(RiscvI16x8UConvertI8x16Low) \ - V(RiscvI16x8UConvertI8x16High) \ - V(RiscvI8x16SConvertI16x8) \ - V(RiscvI8x16UConvertI16x8) \ - V(RiscvWord64AtomicLoadUint8) \ - V(RiscvWord64AtomicLoadUint16) \ - V(RiscvWord64AtomicLoadUint32) \ - V(RiscvWord64AtomicLoadUint64) \ - V(RiscvWord64AtomicStoreWord8) \ - V(RiscvWord64AtomicStoreWord16) \ - V(RiscvWord64AtomicStoreWord32) \ - V(RiscvWord64AtomicStoreWord64) \ - V(RiscvWord64AtomicAddUint8) \ - V(RiscvWord64AtomicAddUint16) \ - V(RiscvWord64AtomicAddUint32) \ - V(RiscvWord64AtomicAddUint64) \ - V(RiscvWord64AtomicSubUint8) \ - V(RiscvWord64AtomicSubUint16) \ - V(RiscvWord64AtomicSubUint32) \ - V(RiscvWord64AtomicSubUint64) \ - V(RiscvWord64AtomicAndUint8) \ - V(RiscvWord64AtomicAndUint16) \ - V(RiscvWord64AtomicAndUint32) \ - V(RiscvWord64AtomicAndUint64) \ - V(RiscvWord64AtomicOrUint8) \ - V(RiscvWord64AtomicOrUint16) \ - V(RiscvWord64AtomicOrUint32) \ - V(RiscvWord64AtomicOrUint64) \ - V(RiscvWord64AtomicXorUint8) \ - V(RiscvWord64AtomicXorUint16) \ - V(RiscvWord64AtomicXorUint32) \ - V(RiscvWord64AtomicXorUint64) \ - V(RiscvWord64AtomicExchangeUint8) \ - V(RiscvWord64AtomicExchangeUint16) \ - V(RiscvWord64AtomicExchangeUint32) \ - V(RiscvWord64AtomicExchangeUint64) \ - V(RiscvWord64AtomicCompareExchangeUint8) \ - V(RiscvWord64AtomicCompareExchangeUint16) \ - V(RiscvWord64AtomicCompareExchangeUint32) \ - V(RiscvWord64AtomicCompareExchangeUint64) \ - V(RiscvStoreCompressTagged) \ - V(RiscvLoadDecompressTaggedSigned) \ - V(RiscvLoadDecompressTaggedPointer) \ +#define TARGET_ARCH_OPCODE_LIST(V) \ + TARGET_ARCH_OPCODE_WITH_MEMORY_ACCESS_MODE_LIST(V) \ + V(RiscvAdd32) \ + V(RiscvAdd64) \ + V(RiscvAddOvf64) \ + V(RiscvSub32) \ + V(RiscvSub64) \ + V(RiscvSubOvf64) \ + V(RiscvMul32) \ + V(RiscvMulOvf32) \ + V(RiscvMulHigh32) \ + V(RiscvMulHigh64) \ + V(RiscvMulHighU32) \ + V(RiscvMul64) \ + V(RiscvDiv32) \ + V(RiscvDiv64) \ + V(RiscvDivU32) \ + V(RiscvDivU64) \ + V(RiscvMod32) \ + V(RiscvMod64) \ + V(RiscvModU32) \ + V(RiscvModU64) \ + V(RiscvAnd) \ + V(RiscvAnd32) \ + V(RiscvOr) \ + V(RiscvOr32) \ + V(RiscvNor) \ + V(RiscvNor32) \ + V(RiscvXor) \ + V(RiscvXor32) \ + V(RiscvClz32) \ + V(RiscvShl32) \ + V(RiscvShr32) \ + V(RiscvSar32) \ + V(RiscvZeroExtendWord) \ + V(RiscvSignExtendWord) \ + V(RiscvClz64) \ + V(RiscvCtz32) \ + V(RiscvCtz64) \ + V(RiscvPopcnt32) \ + V(RiscvPopcnt64) \ + V(RiscvShl64) \ + V(RiscvShr64) \ + V(RiscvSar64) \ + V(RiscvRor32) \ + V(RiscvRor64) \ + V(RiscvMov) \ + V(RiscvTst) \ + V(RiscvCmp) \ + V(RiscvCmpZero) \ + V(RiscvCmpS) \ + V(RiscvAddS) \ + V(RiscvSubS) \ + V(RiscvMulS) \ + V(RiscvDivS) \ + V(RiscvModS) \ + V(RiscvAbsS) \ + V(RiscvNegS) \ + V(RiscvSqrtS) \ + V(RiscvMaxS) \ + V(RiscvMinS) \ + V(RiscvCmpD) \ + V(RiscvAddD) \ + V(RiscvSubD) \ + V(RiscvMulD) \ + V(RiscvDivD) \ + V(RiscvModD) \ + V(RiscvAbsD) \ + V(RiscvNegD) \ + V(RiscvSqrtD) \ + V(RiscvMaxD) \ + V(RiscvMinD) \ + V(RiscvFloat64RoundDown) \ + V(RiscvFloat64RoundTruncate) \ + V(RiscvFloat64RoundUp) \ + V(RiscvFloat64RoundTiesEven) \ + V(RiscvFloat32RoundDown) \ + V(RiscvFloat32RoundTruncate) \ + V(RiscvFloat32RoundUp) \ + V(RiscvFloat32RoundTiesEven) \ + V(RiscvCvtSD) \ + V(RiscvCvtDS) \ + V(RiscvTruncWD) \ + V(RiscvRoundWD) \ + V(RiscvFloorWD) \ + V(RiscvCeilWD) \ + V(RiscvTruncWS) \ + V(RiscvRoundWS) \ + V(RiscvFloorWS) \ + V(RiscvCeilWS) \ + V(RiscvTruncLS) \ + V(RiscvTruncLD) \ + V(RiscvTruncUwD) \ + V(RiscvTruncUwS) \ + V(RiscvTruncUlS) \ + V(RiscvTruncUlD) \ + V(RiscvCvtDW) \ + V(RiscvCvtSL) \ + V(RiscvCvtSW) \ + V(RiscvCvtSUw) \ + V(RiscvCvtSUl) \ + V(RiscvCvtDL) \ + V(RiscvCvtDUw) \ + V(RiscvCvtDUl) \ + V(RiscvLb) \ + V(RiscvLbu) \ + V(RiscvSb) \ + V(RiscvLh) \ + V(RiscvUlh) \ + V(RiscvLhu) \ + V(RiscvUlhu) \ + V(RiscvSh) \ + V(RiscvUsh) \ + V(RiscvLd) \ + V(RiscvUld) \ + V(RiscvLw) \ + V(RiscvUlw) \ + V(RiscvLwu) \ + V(RiscvUlwu) \ + V(RiscvSw) \ + V(RiscvUsw) \ + V(RiscvSd) \ + V(RiscvUsd) \ + V(RiscvLoadFloat) \ + V(RiscvULoadFloat) \ + V(RiscvStoreFloat) \ + V(RiscvUStoreFloat) \ + V(RiscvLoadDouble) \ + V(RiscvULoadDouble) \ + V(RiscvStoreDouble) \ + V(RiscvUStoreDouble) \ + V(RiscvBitcastDL) \ + V(RiscvBitcastLD) \ + V(RiscvBitcastInt32ToFloat32) \ + V(RiscvBitcastFloat32ToInt32) \ + V(RiscvFloat64ExtractLowWord32) \ + V(RiscvFloat64ExtractHighWord32) \ + V(RiscvFloat64InsertLowWord32) \ + V(RiscvFloat64InsertHighWord32) \ + V(RiscvFloat32Max) \ + V(RiscvFloat64Max) \ + V(RiscvFloat32Min) \ + V(RiscvFloat64Min) \ + V(RiscvFloat64SilenceNaN) \ + V(RiscvPush) \ + V(RiscvPeek) \ + V(RiscvByteSwap64) \ + V(RiscvByteSwap32) \ + V(RiscvStoreToStackSlot) \ + V(RiscvStackClaim) \ + V(RiscvSignExtendByte) \ + V(RiscvSignExtendShort) \ + V(RiscvSync) \ + V(RiscvAssertEqual) \ + V(RiscvS128Const) \ + V(RiscvS128Zero) \ + V(RiscvS128AllOnes) \ + V(RiscvI32x4Splat) \ + V(RiscvI32x4ExtractLane) \ + V(RiscvI32x4ReplaceLane) \ + V(RiscvI32x4Add) \ + V(RiscvI32x4Sub) \ + V(RiscvF64x2Abs) \ + V(RiscvF64x2Neg) \ + V(RiscvF32x4Splat) \ + V(RiscvF32x4ExtractLane) \ + V(RiscvF32x4ReplaceLane) \ + V(RiscvF32x4SConvertI32x4) \ + V(RiscvF32x4UConvertI32x4) \ + V(RiscvI64x2SConvertI32x4Low) \ + V(RiscvI64x2SConvertI32x4High) \ + V(RiscvI64x2UConvertI32x4Low) \ + V(RiscvI64x2UConvertI32x4High) \ + V(RiscvI32x4Mul) \ + V(RiscvI32x4MaxS) \ + V(RiscvI32x4MinS) \ + V(RiscvI32x4Eq) \ + V(RiscvI32x4Ne) \ + V(RiscvI32x4Shl) \ + V(RiscvI32x4ShrS) \ + V(RiscvI32x4ShrU) \ + V(RiscvI32x4MaxU) \ + V(RiscvI32x4MinU) \ + V(RiscvI64x2GtS) \ + V(RiscvI64x2GeS) \ + V(RiscvI64x2Eq) \ + V(RiscvI64x2Ne) \ + V(RiscvF64x2Sqrt) \ + V(RiscvF64x2Add) \ + V(RiscvF64x2Sub) \ + V(RiscvF64x2Mul) \ + V(RiscvF64x2Div) \ + V(RiscvF64x2Min) \ + V(RiscvF64x2Max) \ + V(RiscvF64x2ConvertLowI32x4S) \ + V(RiscvF64x2ConvertLowI32x4U) \ + V(RiscvF64x2PromoteLowF32x4) \ + V(RiscvF64x2Eq) \ + V(RiscvF64x2Ne) \ + V(RiscvF64x2Lt) \ + V(RiscvF64x2Le) \ + V(RiscvF64x2Splat) \ + V(RiscvF64x2ExtractLane) \ + V(RiscvF64x2ReplaceLane) \ + V(RiscvF64x2Pmin) \ + V(RiscvF64x2Pmax) \ + V(RiscvF64x2Ceil) \ + V(RiscvF64x2Floor) \ + V(RiscvF64x2Trunc) \ + V(RiscvF64x2NearestInt) \ + V(RiscvI64x2Splat) \ + V(RiscvI64x2ExtractLane) \ + V(RiscvI64x2ReplaceLane) \ + V(RiscvI64x2Add) \ + V(RiscvI64x2Sub) \ + V(RiscvI64x2Mul) \ + V(RiscvI64x2Abs) \ + V(RiscvI64x2Neg) \ + V(RiscvI64x2Shl) \ + V(RiscvI64x2ShrS) \ + V(RiscvI64x2ShrU) \ + V(RiscvI64x2BitMask) \ + V(RiscvF32x4Abs) \ + V(RiscvF32x4Neg) \ + V(RiscvF32x4Sqrt) \ + V(RiscvF32x4RecipApprox) \ + V(RiscvF32x4RecipSqrtApprox) \ + V(RiscvF32x4Add) \ + V(RiscvF32x4Sub) \ + V(RiscvF32x4Mul) \ + V(RiscvF32x4Div) \ + V(RiscvF32x4Max) \ + V(RiscvF32x4Min) \ + V(RiscvF32x4Eq) \ + V(RiscvF32x4Ne) \ + V(RiscvF32x4Lt) \ + V(RiscvF32x4Le) \ + V(RiscvF32x4Pmin) \ + V(RiscvF32x4Pmax) \ + V(RiscvF32x4DemoteF64x2Zero) \ + V(RiscvF32x4Ceil) \ + V(RiscvF32x4Floor) \ + V(RiscvF32x4Trunc) \ + V(RiscvF32x4NearestInt) \ + V(RiscvI32x4SConvertF32x4) \ + V(RiscvI32x4UConvertF32x4) \ + V(RiscvI32x4Neg) \ + V(RiscvI32x4GtS) \ + V(RiscvI32x4GeS) \ + V(RiscvI32x4GtU) \ + V(RiscvI32x4GeU) \ + V(RiscvI32x4Abs) \ + V(RiscvI32x4BitMask) \ + V(RiscvI32x4DotI16x8S) \ + V(RiscvI32x4TruncSatF64x2SZero) \ + V(RiscvI32x4TruncSatF64x2UZero) \ + V(RiscvI16x8Splat) \ + V(RiscvI16x8ExtractLaneU) \ + V(RiscvI16x8ExtractLaneS) \ + V(RiscvI16x8ReplaceLane) \ + V(RiscvI16x8Neg) \ + V(RiscvI16x8Shl) \ + V(RiscvI16x8ShrS) \ + V(RiscvI16x8ShrU) \ + V(RiscvI16x8Add) \ + V(RiscvI16x8AddSatS) \ + V(RiscvI16x8Sub) \ + V(RiscvI16x8SubSatS) \ + V(RiscvI16x8Mul) \ + V(RiscvI16x8MaxS) \ + V(RiscvI16x8MinS) \ + V(RiscvI16x8Eq) \ + V(RiscvI16x8Ne) \ + V(RiscvI16x8GtS) \ + V(RiscvI16x8GeS) \ + V(RiscvI16x8AddSatU) \ + V(RiscvI16x8SubSatU) \ + V(RiscvI16x8MaxU) \ + V(RiscvI16x8MinU) \ + V(RiscvI16x8GtU) \ + V(RiscvI16x8GeU) \ + V(RiscvI16x8RoundingAverageU) \ + V(RiscvI16x8Q15MulRSatS) \ + V(RiscvI16x8Abs) \ + V(RiscvI16x8BitMask) \ + V(RiscvI8x16Splat) \ + V(RiscvI8x16ExtractLaneU) \ + V(RiscvI8x16ExtractLaneS) \ + V(RiscvI8x16ReplaceLane) \ + V(RiscvI8x16Neg) \ + V(RiscvI8x16Shl) \ + V(RiscvI8x16ShrS) \ + V(RiscvI8x16Add) \ + V(RiscvI8x16AddSatS) \ + V(RiscvI8x16Sub) \ + V(RiscvI8x16SubSatS) \ + V(RiscvI8x16MaxS) \ + V(RiscvI8x16MinS) \ + V(RiscvI8x16Eq) \ + V(RiscvI8x16Ne) \ + V(RiscvI8x16GtS) \ + V(RiscvI8x16GeS) \ + V(RiscvI8x16ShrU) \ + V(RiscvI8x16AddSatU) \ + V(RiscvI8x16SubSatU) \ + V(RiscvI8x16MaxU) \ + V(RiscvI8x16MinU) \ + V(RiscvI8x16GtU) \ + V(RiscvI8x16GeU) \ + V(RiscvI8x16RoundingAverageU) \ + V(RiscvI8x16Abs) \ + V(RiscvI8x16BitMask) \ + V(RiscvI8x16Popcnt) \ + V(RiscvS128And) \ + V(RiscvS128Or) \ + V(RiscvS128Xor) \ + V(RiscvS128Not) \ + V(RiscvS128Select) \ + V(RiscvS128AndNot) \ + V(RiscvI32x4AllTrue) \ + V(RiscvI16x8AllTrue) \ + V(RiscvV128AnyTrue) \ + V(RiscvI8x16AllTrue) \ + V(RiscvI64x2AllTrue) \ + V(RiscvS32x4InterleaveRight) \ + V(RiscvS32x4InterleaveLeft) \ + V(RiscvS32x4PackEven) \ + V(RiscvS32x4PackOdd) \ + V(RiscvS32x4InterleaveEven) \ + V(RiscvS32x4InterleaveOdd) \ + V(RiscvS32x4Shuffle) \ + V(RiscvS16x8InterleaveRight) \ + V(RiscvS16x8InterleaveLeft) \ + V(RiscvS16x8PackEven) \ + V(RiscvS16x8PackOdd) \ + V(RiscvS16x8InterleaveEven) \ + V(RiscvS16x8InterleaveOdd) \ + V(RiscvS16x4Reverse) \ + V(RiscvS16x2Reverse) \ + V(RiscvS8x16InterleaveRight) \ + V(RiscvS8x16InterleaveLeft) \ + V(RiscvS8x16PackEven) \ + V(RiscvS8x16PackOdd) \ + V(RiscvS8x16InterleaveEven) \ + V(RiscvS8x16InterleaveOdd) \ + V(RiscvI8x16Shuffle) \ + V(RiscvI8x16Swizzle) \ + V(RiscvS8x16Concat) \ + V(RiscvS8x8Reverse) \ + V(RiscvS8x4Reverse) \ + V(RiscvS8x2Reverse) \ + V(RiscvS128Load8Splat) \ + V(RiscvS128Load16Splat) \ + V(RiscvS128Load32Splat) \ + V(RiscvS128Load64Splat) \ + V(RiscvS128Load8x8S) \ + V(RiscvS128Load8x8U) \ + V(RiscvS128Load16x4S) \ + V(RiscvS128Load16x4U) \ + V(RiscvS128Load32x2S) \ + V(RiscvS128Load32x2U) \ + V(RiscvS128LoadLane) \ + V(RiscvS128StoreLane) \ + V(RiscvRvvLd) \ + V(RiscvRvvSt) \ + V(RiscvI32x4SConvertI16x8Low) \ + V(RiscvI32x4SConvertI16x8High) \ + V(RiscvI32x4UConvertI16x8Low) \ + V(RiscvI32x4UConvertI16x8High) \ + V(RiscvI16x8SConvertI8x16Low) \ + V(RiscvI16x8SConvertI8x16High) \ + V(RiscvI16x8SConvertI32x4) \ + V(RiscvI16x8UConvertI32x4) \ + V(RiscvI16x8UConvertI8x16Low) \ + V(RiscvI16x8UConvertI8x16High) \ + V(RiscvI8x16SConvertI16x8) \ + V(RiscvI8x16UConvertI16x8) \ + V(RiscvWord64AtomicLoadUint64) \ + V(RiscvWord64AtomicStoreWord64) \ + V(RiscvWord64AtomicAddUint64) \ + V(RiscvWord64AtomicSubUint64) \ + V(RiscvWord64AtomicAndUint64) \ + V(RiscvWord64AtomicOrUint64) \ + V(RiscvWord64AtomicXorUint64) \ + V(RiscvWord64AtomicExchangeUint64) \ + V(RiscvWord64AtomicCompareExchangeUint64) \ + V(RiscvStoreCompressTagged) \ + V(RiscvLoadDecompressTaggedSigned) \ + V(RiscvLoadDecompressTaggedPointer) \ V(RiscvLoadDecompressAnyTagged) // Addressing modes represent the "shape" of inputs to an instruction. diff --git a/chromium/v8/src/compiler/backend/riscv64/instruction-scheduler-riscv64.cc b/chromium/v8/src/compiler/backend/riscv64/instruction-scheduler-riscv64.cc index 157b11c9308..54d9a98663f 100644 --- a/chromium/v8/src/compiler/backend/riscv64/instruction-scheduler-riscv64.cc +++ b/chromium/v8/src/compiler/backend/riscv64/instruction-scheduler-riscv64.cc @@ -318,7 +318,7 @@ int InstructionScheduler::GetTargetInstructionFlags( case kRiscvS8x2Reverse: case kRiscvS8x4Reverse: case kRiscvS8x8Reverse: - case kRiscvS8x16Shuffle: + case kRiscvI8x16Shuffle: case kRiscvI8x16Swizzle: case kRiscvSar32: case kRiscvSignExtendByte: @@ -352,7 +352,7 @@ int InstructionScheduler::GetTargetInstructionFlags( case kRiscvLw: case kRiscvLoadFloat: case kRiscvLwu: - case kRiscvMsaLd: + case kRiscvRvvLd: case kRiscvPeek: case kRiscvUld: case kRiscvULoadDouble: @@ -372,9 +372,6 @@ int InstructionScheduler::GetTargetInstructionFlags( case kRiscvS128Load32x2S: case kRiscvS128Load32x2U: case kRiscvS128LoadLane: - case kRiscvWord64AtomicLoadUint8: - case kRiscvWord64AtomicLoadUint16: - case kRiscvWord64AtomicLoadUint32: case kRiscvWord64AtomicLoadUint64: case kRiscvLoadDecompressTaggedSigned: case kRiscvLoadDecompressTaggedPointer: @@ -383,7 +380,7 @@ int InstructionScheduler::GetTargetInstructionFlags( case kRiscvModD: case kRiscvModS: - case kRiscvMsaSt: + case kRiscvRvvSt: case kRiscvPush: case kRiscvSb: case kRiscvSd: @@ -399,37 +396,13 @@ int InstructionScheduler::GetTargetInstructionFlags( case kRiscvUsw: case kRiscvUStoreFloat: case kRiscvSync: - case kRiscvWord64AtomicStoreWord8: - case kRiscvWord64AtomicStoreWord16: - case kRiscvWord64AtomicStoreWord32: case kRiscvWord64AtomicStoreWord64: - case kRiscvWord64AtomicAddUint8: - case kRiscvWord64AtomicAddUint16: - case kRiscvWord64AtomicAddUint32: case kRiscvWord64AtomicAddUint64: - case kRiscvWord64AtomicSubUint8: - case kRiscvWord64AtomicSubUint16: - case kRiscvWord64AtomicSubUint32: case kRiscvWord64AtomicSubUint64: - case kRiscvWord64AtomicAndUint8: - case kRiscvWord64AtomicAndUint16: - case kRiscvWord64AtomicAndUint32: case kRiscvWord64AtomicAndUint64: - case kRiscvWord64AtomicOrUint8: - case kRiscvWord64AtomicOrUint16: - case kRiscvWord64AtomicOrUint32: case kRiscvWord64AtomicOrUint64: - case kRiscvWord64AtomicXorUint8: - case kRiscvWord64AtomicXorUint16: - case kRiscvWord64AtomicXorUint32: case kRiscvWord64AtomicXorUint64: - case kRiscvWord64AtomicExchangeUint8: - case kRiscvWord64AtomicExchangeUint16: - case kRiscvWord64AtomicExchangeUint32: case kRiscvWord64AtomicExchangeUint64: - case kRiscvWord64AtomicCompareExchangeUint8: - case kRiscvWord64AtomicCompareExchangeUint16: - case kRiscvWord64AtomicCompareExchangeUint32: case kRiscvWord64AtomicCompareExchangeUint64: case kRiscvStoreCompressTagged: case kRiscvS128StoreLane: @@ -1144,7 +1117,7 @@ int InstructionScheduler::GetInstructionLatency(const Instruction* instr) { return AssembleArchJumpLatency(); case kArchTableSwitch: return AssembleArchTableSwitchLatency(); - case kArchAbortCSAAssert: + case kArchAbortCSADcheck: return CallLatency() + 1; case kArchDebugBreak: return 1; @@ -1169,8 +1142,6 @@ int InstructionScheduler::GetInstructionLatency(const Instruction* instr) { return Add64Latency(false) + AndLatency(false) + AssertLatency() + Add64Latency(false) + AndLatency(false) + BranchShortLatency() + 1 + Sub64Latency() + Add64Latency(); - case kArchWordPoisonOnSpeculation: - return AndLatency(); case kIeee754Float64Acos: case kIeee754Float64Acosh: case kIeee754Float64Asin: @@ -1541,35 +1512,35 @@ int InstructionScheduler::GetInstructionLatency(const Instruction* instr) { return ByteSwapSignedLatency(); case kRiscvByteSwap32: return ByteSwapSignedLatency(); - case kWord32AtomicLoadInt8: - case kWord32AtomicLoadUint8: - case kWord32AtomicLoadInt16: - case kWord32AtomicLoadUint16: - case kWord32AtomicLoadWord32: + case kAtomicLoadInt8: + case kAtomicLoadUint8: + case kAtomicLoadInt16: + case kAtomicLoadUint16: + case kAtomicLoadWord32: return 2; - case kWord32AtomicStoreWord8: - case kWord32AtomicStoreWord16: - case kWord32AtomicStoreWord32: + case kAtomicStoreWord8: + case kAtomicStoreWord16: + case kAtomicStoreWord32: return 3; - case kWord32AtomicExchangeInt8: + case kAtomicExchangeInt8: return Word32AtomicExchangeLatency(true, 8); - case kWord32AtomicExchangeUint8: + case kAtomicExchangeUint8: return Word32AtomicExchangeLatency(false, 8); - case kWord32AtomicExchangeInt16: + case kAtomicExchangeInt16: return Word32AtomicExchangeLatency(true, 16); - case kWord32AtomicExchangeUint16: + case kAtomicExchangeUint16: return Word32AtomicExchangeLatency(false, 16); - case kWord32AtomicExchangeWord32: + case kAtomicExchangeWord32: return 2 + LlLatency(0) + 1 + ScLatency(0) + BranchShortLatency() + 1; - case kWord32AtomicCompareExchangeInt8: + case kAtomicCompareExchangeInt8: return Word32AtomicCompareExchangeLatency(true, 8); - case kWord32AtomicCompareExchangeUint8: + case kAtomicCompareExchangeUint8: return Word32AtomicCompareExchangeLatency(false, 8); - case kWord32AtomicCompareExchangeInt16: + case kAtomicCompareExchangeInt16: return Word32AtomicCompareExchangeLatency(true, 16); - case kWord32AtomicCompareExchangeUint16: + case kAtomicCompareExchangeUint16: return Word32AtomicCompareExchangeLatency(false, 16); - case kWord32AtomicCompareExchangeWord32: + case kAtomicCompareExchangeWord32: return 3 + LlLatency(0) + BranchShortLatency() + 1 + ScLatency(0) + BranchShortLatency() + 1; case kRiscvAssertEqual: diff --git a/chromium/v8/src/compiler/backend/riscv64/instruction-selector-riscv64.cc b/chromium/v8/src/compiler/backend/riscv64/instruction-selector-riscv64.cc index 72706201e2a..6fc64256ec7 100644 --- a/chromium/v8/src/compiler/backend/riscv64/instruction-selector-riscv64.cc +++ b/chromium/v8/src/compiler/backend/riscv64/instruction-selector-riscv64.cc @@ -363,9 +363,9 @@ void InstructionSelector::VisitStackSlot(Node* node) { sequence()->AddImmediate(Constant(alignment)), 0, nullptr); } -void InstructionSelector::VisitAbortCSAAssert(Node* node) { +void InstructionSelector::VisitAbortCSADcheck(Node* node) { RiscvOperandGenerator g(this); - Emit(kArchAbortCSAAssert, g.NoOutput(), g.UseFixed(node->InputAt(0), a0)); + Emit(kArchAbortCSADcheck, g.NoOutput(), g.UseFixed(node->InputAt(0), a0)); } void EmitLoad(InstructionSelector* selector, Node* node, InstructionCode opcode, @@ -454,7 +454,7 @@ void InstructionSelector::VisitLoad(Node* node) { opcode = load_rep.IsUnsigned() ? kRiscvLhu : kRiscvLh; break; case MachineRepresentation::kWord32: - opcode = load_rep.IsUnsigned() ? kRiscvLwu : kRiscvLw; + opcode = kRiscvLw; break; #ifdef V8_COMPRESS_POINTERS case MachineRepresentation::kTaggedSigned: @@ -475,7 +475,7 @@ void InstructionSelector::VisitLoad(Node* node) { opcode = kRiscvLd; break; case MachineRepresentation::kSimd128: - opcode = kRiscvMsaLd; + opcode = kRiscvRvvLd; break; case MachineRepresentation::kCompressedPointer: case MachineRepresentation::kCompressed: @@ -489,16 +489,10 @@ void InstructionSelector::VisitLoad(Node* node) { case MachineRepresentation::kNone: UNREACHABLE(); } - if (node->opcode() == IrOpcode::kPoisonedLoad) { - CHECK_NE(poisoning_level_, PoisoningMitigationLevel::kDontPoison); - opcode |= MiscField::encode(kMemoryAccessPoisoned); - } EmitLoad(this, node, opcode); } -void InstructionSelector::VisitPoisonedLoad(Node* node) { VisitLoad(node); } - void InstructionSelector::VisitProtectedLoad(Node* node) { // TODO(eholk) UNIMPLEMENTED(); @@ -560,7 +554,7 @@ void InstructionSelector::VisitStore(Node* node) { opcode = kRiscvSd; break; case MachineRepresentation::kSimd128: - opcode = kRiscvMsaSt; + opcode = kRiscvRvvSt; break; case MachineRepresentation::kCompressedPointer: // Fall through. case MachineRepresentation::kCompressed: @@ -569,7 +563,6 @@ void InstructionSelector::VisitStore(Node* node) { break; #else UNREACHABLE(); - break; #endif case MachineRepresentation::kMapWord: // Fall through. case MachineRepresentation::kNone: @@ -1294,7 +1287,6 @@ bool InstructionSelector::ZeroExtendsWord32ToWord64NoPhis(Node* node) { switch (load_rep.representation()) { case MachineRepresentation::kWord8: case MachineRepresentation::kWord16: - case MachineRepresentation::kWord32: return true; default: return false; @@ -1630,7 +1622,7 @@ void InstructionSelector::VisitUnalignedLoad(Node* node) { opcode = load_rep.IsUnsigned() ? kRiscvUlhu : kRiscvUlh; break; case MachineRepresentation::kWord32: - opcode = load_rep.IsUnsigned() ? kRiscvUlwu : kRiscvUlw; + opcode = kRiscvUlw; break; case MachineRepresentation::kTaggedSigned: // Fall through. case MachineRepresentation::kTaggedPointer: // Fall through. @@ -1639,7 +1631,7 @@ void InstructionSelector::VisitUnalignedLoad(Node* node) { opcode = kRiscvUld; break; case MachineRepresentation::kSimd128: - opcode = kRiscvMsaLd; + opcode = kRiscvRvvLd; break; case MachineRepresentation::kBit: // Fall through. case MachineRepresentation::kCompressedPointer: // Fall through. @@ -1693,7 +1685,7 @@ void InstructionSelector::VisitUnalignedStore(Node* node) { opcode = kRiscvUsd; break; case MachineRepresentation::kSimd128: - opcode = kRiscvMsaSt; + opcode = kRiscvRvvSt; break; case MachineRepresentation::kBit: // Fall through. case MachineRepresentation::kCompressedPointer: // Fall through. @@ -1789,7 +1781,8 @@ void VisitWordCompare(InstructionSelector* selector, Node* node, Int32BinopMatcher m(node, true); NumberBinopMatcher n(node, true); if (m.right().Is(0) || n.right().IsZero()) { - VisitWordCompareZero(selector, g.UseRegister(left), cont); + VisitWordCompareZero(selector, g.UseRegisterOrImmediateZero(left), + cont); } else { VisitCompare(selector, opcode, g.UseRegister(left), g.UseRegister(right), cont); @@ -1802,7 +1795,8 @@ void VisitWordCompare(InstructionSelector* selector, Node* node, case kUnsignedGreaterThanOrEqual: { Int32BinopMatcher m(node, true); if (m.right().Is(0)) { - VisitWordCompareZero(selector, g.UseRegister(left), cont); + VisitWordCompareZero(selector, g.UseRegisterOrImmediateZero(left), + cont); } else { VisitCompare(selector, opcode, g.UseRegister(left), g.UseImmediate(right), cont); @@ -1811,7 +1805,8 @@ void VisitWordCompare(InstructionSelector* selector, Node* node, default: Int32BinopMatcher m(node, true); if (m.right().Is(0)) { - VisitWordCompareZero(selector, g.UseRegister(left), cont); + VisitWordCompareZero(selector, g.UseRegisterOrImmediateZero(left), + cont); } else { VisitCompare(selector, opcode, g.UseRegister(left), g.UseRegister(right), cont); @@ -1827,10 +1822,13 @@ void VisitWordCompare(InstructionSelector* selector, Node* node, bool IsNodeUnsigned(Node* n) { NodeMatcher m(n); - if (m.IsLoad() || m.IsUnalignedLoad() || m.IsPoisonedLoad() || - m.IsProtectedLoad() || m.IsWord32AtomicLoad() || m.IsWord64AtomicLoad()) { + if (m.IsLoad() || m.IsUnalignedLoad() || m.IsProtectedLoad()) { LoadRepresentation load_rep = LoadRepresentationOf(n->op()); return load_rep.IsUnsigned(); + } else if (m.IsWord32AtomicLoad() || m.IsWord64AtomicLoad()) { + AtomicLoadParameters atomic_load_params = AtomicLoadParametersOf(n->op()); + LoadRepresentation load_rep = atomic_load_params.representation(); + return load_rep.IsUnsigned(); } else { return m.IsUint32Div() || m.IsUint32LessThan() || m.IsUint32LessThanOrEqual() || m.IsUint32Mod() || @@ -1930,16 +1928,18 @@ void VisitWord64Compare(InstructionSelector* selector, Node* node, void EmitWordCompareZero(InstructionSelector* selector, Node* value, FlagsContinuation* cont) { RiscvOperandGenerator g(selector); - selector->EmitWithContinuation(kRiscvCmpZero, g.UseRegister(value), cont); + selector->EmitWithContinuation(kRiscvCmpZero, + g.UseRegisterOrImmediateZero(value), cont); } void VisitAtomicLoad(InstructionSelector* selector, Node* node, - ArchOpcode opcode) { + ArchOpcode opcode, AtomicWidth width) { RiscvOperandGenerator g(selector); Node* base = node->InputAt(0); Node* index = node->InputAt(1); if (g.CanBeImmediate(index, opcode)) { - selector->Emit(opcode | AddressingModeField::encode(kMode_MRI), + selector->Emit(opcode | AddressingModeField::encode(kMode_MRI) | + AtomicWidthField::encode(width), g.DefineAsRegister(node), g.UseRegister(base), g.UseImmediate(index)); } else { @@ -1947,20 +1947,22 @@ void VisitAtomicLoad(InstructionSelector* selector, Node* node, selector->Emit(kRiscvAdd64 | AddressingModeField::encode(kMode_None), addr_reg, g.UseRegister(index), g.UseRegister(base)); // Emit desired load opcode, using temp addr_reg. - selector->Emit(opcode | AddressingModeField::encode(kMode_MRI), + selector->Emit(opcode | AddressingModeField::encode(kMode_MRI) | + AtomicWidthField::encode(width), g.DefineAsRegister(node), addr_reg, g.TempImmediate(0)); } } void VisitAtomicStore(InstructionSelector* selector, Node* node, - ArchOpcode opcode) { + ArchOpcode opcode, AtomicWidth width) { RiscvOperandGenerator g(selector); Node* base = node->InputAt(0); Node* index = node->InputAt(1); Node* value = node->InputAt(2); if (g.CanBeImmediate(index, opcode)) { - selector->Emit(opcode | AddressingModeField::encode(kMode_MRI), + selector->Emit(opcode | AddressingModeField::encode(kMode_MRI) | + AtomicWidthField::encode(width), g.NoOutput(), g.UseRegister(base), g.UseImmediate(index), g.UseRegisterOrImmediateZero(value)); } else { @@ -1968,14 +1970,15 @@ void VisitAtomicStore(InstructionSelector* selector, Node* node, selector->Emit(kRiscvAdd64 | AddressingModeField::encode(kMode_None), addr_reg, g.UseRegister(index), g.UseRegister(base)); // Emit desired store opcode, using temp addr_reg. - selector->Emit(opcode | AddressingModeField::encode(kMode_MRI), + selector->Emit(opcode | AddressingModeField::encode(kMode_MRI) | + AtomicWidthField::encode(width), g.NoOutput(), addr_reg, g.TempImmediate(0), g.UseRegisterOrImmediateZero(value)); } } void VisitAtomicExchange(InstructionSelector* selector, Node* node, - ArchOpcode opcode) { + ArchOpcode opcode, AtomicWidth width) { RiscvOperandGenerator g(selector); Node* base = node->InputAt(0); Node* index = node->InputAt(1); @@ -1993,12 +1996,13 @@ void VisitAtomicExchange(InstructionSelector* selector, Node* node, temp[0] = g.TempRegister(); temp[1] = g.TempRegister(); temp[2] = g.TempRegister(); - InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); + InstructionCode code = opcode | AddressingModeField::encode(addressing_mode) | + AtomicWidthField::encode(width); selector->Emit(code, 1, outputs, input_count, inputs, 3, temp); } void VisitAtomicCompareExchange(InstructionSelector* selector, Node* node, - ArchOpcode opcode) { + ArchOpcode opcode, AtomicWidth width) { RiscvOperandGenerator g(selector); Node* base = node->InputAt(0); Node* index = node->InputAt(1); @@ -2018,12 +2022,13 @@ void VisitAtomicCompareExchange(InstructionSelector* selector, Node* node, temp[0] = g.TempRegister(); temp[1] = g.TempRegister(); temp[2] = g.TempRegister(); - InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); + InstructionCode code = opcode | AddressingModeField::encode(addressing_mode) | + AtomicWidthField::encode(width); selector->Emit(code, 1, outputs, input_count, inputs, 3, temp); } void VisitAtomicBinop(InstructionSelector* selector, Node* node, - ArchOpcode opcode) { + ArchOpcode opcode, AtomicWidth width) { RiscvOperandGenerator g(selector); Node* base = node->InputAt(0); Node* index = node->InputAt(1); @@ -2042,7 +2047,8 @@ void VisitAtomicBinop(InstructionSelector* selector, Node* node, temps[1] = g.TempRegister(); temps[2] = g.TempRegister(); temps[3] = g.TempRegister(); - InstructionCode code = opcode | AddressingModeField::encode(addressing_mode); + InstructionCode code = opcode | AddressingModeField::encode(addressing_mode) | + AtomicWidthField::encode(width); selector->Emit(code, 1, outputs, input_count, inputs, 4, temps); } @@ -2404,163 +2410,201 @@ void InstructionSelector::VisitMemoryBarrier(Node* node) { } void InstructionSelector::VisitWord32AtomicLoad(Node* node) { - LoadRepresentation load_rep = LoadRepresentationOf(node->op()); + AtomicLoadParameters atomic_load_params = AtomicLoadParametersOf(node->op()); + LoadRepresentation load_rep = atomic_load_params.representation(); ArchOpcode opcode; switch (load_rep.representation()) { case MachineRepresentation::kWord8: - opcode = - load_rep.IsSigned() ? kWord32AtomicLoadInt8 : kWord32AtomicLoadUint8; + opcode = load_rep.IsSigned() ? kAtomicLoadInt8 : kAtomicLoadUint8; break; case MachineRepresentation::kWord16: - opcode = load_rep.IsSigned() ? kWord32AtomicLoadInt16 - : kWord32AtomicLoadUint16; + opcode = load_rep.IsSigned() ? kAtomicLoadInt16 : kAtomicLoadUint16; break; case MachineRepresentation::kWord32: - opcode = kWord32AtomicLoadWord32; + opcode = kAtomicLoadWord32; break; default: UNREACHABLE(); } - VisitAtomicLoad(this, node, opcode); + VisitAtomicLoad(this, node, opcode, AtomicWidth::kWord32); } void InstructionSelector::VisitWord32AtomicStore(Node* node) { - MachineRepresentation rep = AtomicStoreRepresentationOf(node->op()); + AtomicStoreParameters store_params = AtomicStoreParametersOf(node->op()); + MachineRepresentation rep = store_params.representation(); ArchOpcode opcode; switch (rep) { case MachineRepresentation::kWord8: - opcode = kWord32AtomicStoreWord8; + opcode = kAtomicStoreWord8; break; case MachineRepresentation::kWord16: - opcode = kWord32AtomicStoreWord16; + opcode = kAtomicStoreWord16; break; case MachineRepresentation::kWord32: - opcode = kWord32AtomicStoreWord32; + opcode = kAtomicStoreWord32; break; default: UNREACHABLE(); } - VisitAtomicStore(this, node, opcode); + VisitAtomicStore(this, node, opcode, AtomicWidth::kWord32); } void InstructionSelector::VisitWord64AtomicLoad(Node* node) { - LoadRepresentation load_rep = LoadRepresentationOf(node->op()); + AtomicLoadParameters atomic_load_params = AtomicLoadParametersOf(node->op()); + LoadRepresentation load_rep = atomic_load_params.representation(); ArchOpcode opcode; switch (load_rep.representation()) { case MachineRepresentation::kWord8: - opcode = kRiscvWord64AtomicLoadUint8; + opcode = kAtomicLoadUint8; break; case MachineRepresentation::kWord16: - opcode = kRiscvWord64AtomicLoadUint16; + opcode = kAtomicLoadUint16; break; case MachineRepresentation::kWord32: - opcode = kRiscvWord64AtomicLoadUint32; + opcode = kAtomicLoadWord32; break; case MachineRepresentation::kWord64: opcode = kRiscvWord64AtomicLoadUint64; break; +#ifdef V8_COMPRESS_POINTERS + case MachineRepresentation::kTaggedSigned: + opcode = kRiscv64LdDecompressTaggedSigned; + break; + case MachineRepresentation::kTaggedPointer: + opcode = kRiscv64LdDecompressTaggedPointer; + break; + case MachineRepresentation::kTagged: + opcode = kRiscv64LdDecompressAnyTagged; + break; +#else + case MachineRepresentation::kTaggedSigned: // Fall through. + case MachineRepresentation::kTaggedPointer: // Fall through. + case MachineRepresentation::kTagged: + if (kTaggedSize == 8) { + opcode = kRiscvWord64AtomicLoadUint64; + } else { + opcode = kAtomicLoadWord32; + } + break; +#endif + case MachineRepresentation::kCompressedPointer: // Fall through. + case MachineRepresentation::kCompressed: + DCHECK(COMPRESS_POINTERS_BOOL); + opcode = kAtomicLoadWord32; + break; default: UNREACHABLE(); } - VisitAtomicLoad(this, node, opcode); + VisitAtomicLoad(this, node, opcode, AtomicWidth::kWord64); } void InstructionSelector::VisitWord64AtomicStore(Node* node) { - MachineRepresentation rep = AtomicStoreRepresentationOf(node->op()); + AtomicStoreParameters store_params = AtomicStoreParametersOf(node->op()); + MachineRepresentation rep = store_params.representation(); ArchOpcode opcode; switch (rep) { case MachineRepresentation::kWord8: - opcode = kRiscvWord64AtomicStoreWord8; + opcode = kAtomicStoreWord8; break; case MachineRepresentation::kWord16: - opcode = kRiscvWord64AtomicStoreWord16; + opcode = kAtomicStoreWord16; break; case MachineRepresentation::kWord32: - opcode = kRiscvWord64AtomicStoreWord32; + opcode = kAtomicStoreWord32; break; case MachineRepresentation::kWord64: opcode = kRiscvWord64AtomicStoreWord64; break; + case MachineRepresentation::kTaggedSigned: // Fall through. + case MachineRepresentation::kTaggedPointer: // Fall through. + case MachineRepresentation::kTagged: + opcode = kRiscvWord64AtomicStoreWord64; + break; + case MachineRepresentation::kCompressedPointer: // Fall through. + case MachineRepresentation::kCompressed: + CHECK(COMPRESS_POINTERS_BOOL); + opcode = kAtomicStoreWord32; + break; default: UNREACHABLE(); } - VisitAtomicStore(this, node, opcode); + VisitAtomicStore(this, node, opcode, AtomicWidth::kWord64); } void InstructionSelector::VisitWord32AtomicExchange(Node* node) { ArchOpcode opcode; MachineType type = AtomicOpType(node->op()); if (type == MachineType::Int8()) { - opcode = kWord32AtomicExchangeInt8; + opcode = kAtomicExchangeInt8; } else if (type == MachineType::Uint8()) { - opcode = kWord32AtomicExchangeUint8; + opcode = kAtomicExchangeUint8; } else if (type == MachineType::Int16()) { - opcode = kWord32AtomicExchangeInt16; + opcode = kAtomicExchangeInt16; } else if (type == MachineType::Uint16()) { - opcode = kWord32AtomicExchangeUint16; + opcode = kAtomicExchangeUint16; } else if (type == MachineType::Int32() || type == MachineType::Uint32()) { - opcode = kWord32AtomicExchangeWord32; + opcode = kAtomicExchangeWord32; } else { UNREACHABLE(); } - VisitAtomicExchange(this, node, opcode); + VisitAtomicExchange(this, node, opcode, AtomicWidth::kWord32); } void InstructionSelector::VisitWord64AtomicExchange(Node* node) { ArchOpcode opcode; MachineType type = AtomicOpType(node->op()); if (type == MachineType::Uint8()) { - opcode = kRiscvWord64AtomicExchangeUint8; + opcode = kAtomicExchangeUint8; } else if (type == MachineType::Uint16()) { - opcode = kRiscvWord64AtomicExchangeUint16; + opcode = kAtomicExchangeUint16; } else if (type == MachineType::Uint32()) { - opcode = kRiscvWord64AtomicExchangeUint32; + opcode = kAtomicExchangeWord32; } else if (type == MachineType::Uint64()) { opcode = kRiscvWord64AtomicExchangeUint64; } else { UNREACHABLE(); } - VisitAtomicExchange(this, node, opcode); + VisitAtomicExchange(this, node, opcode, AtomicWidth::kWord64); } void InstructionSelector::VisitWord32AtomicCompareExchange(Node* node) { ArchOpcode opcode; MachineType type = AtomicOpType(node->op()); if (type == MachineType::Int8()) { - opcode = kWord32AtomicCompareExchangeInt8; + opcode = kAtomicCompareExchangeInt8; } else if (type == MachineType::Uint8()) { - opcode = kWord32AtomicCompareExchangeUint8; + opcode = kAtomicCompareExchangeUint8; } else if (type == MachineType::Int16()) { - opcode = kWord32AtomicCompareExchangeInt16; + opcode = kAtomicCompareExchangeInt16; } else if (type == MachineType::Uint16()) { - opcode = kWord32AtomicCompareExchangeUint16; + opcode = kAtomicCompareExchangeUint16; } else if (type == MachineType::Int32() || type == MachineType::Uint32()) { - opcode = kWord32AtomicCompareExchangeWord32; + opcode = kAtomicCompareExchangeWord32; } else { UNREACHABLE(); } - VisitAtomicCompareExchange(this, node, opcode); + VisitAtomicCompareExchange(this, node, opcode, AtomicWidth::kWord32); } void InstructionSelector::VisitWord64AtomicCompareExchange(Node* node) { ArchOpcode opcode; MachineType type = AtomicOpType(node->op()); if (type == MachineType::Uint8()) { - opcode = kRiscvWord64AtomicCompareExchangeUint8; + opcode = kAtomicCompareExchangeUint8; } else if (type == MachineType::Uint16()) { - opcode = kRiscvWord64AtomicCompareExchangeUint16; + opcode = kAtomicCompareExchangeUint16; } else if (type == MachineType::Uint32()) { - opcode = kRiscvWord64AtomicCompareExchangeUint32; + opcode = kAtomicCompareExchangeWord32; } else if (type == MachineType::Uint64()) { opcode = kRiscvWord64AtomicCompareExchangeUint64; } else { UNREACHABLE(); } - VisitAtomicCompareExchange(this, node, opcode); + VisitAtomicCompareExchange(this, node, opcode, AtomicWidth::kWord64); } void InstructionSelector::VisitWord32AtomicBinaryOperation( Node* node, ArchOpcode int8_op, ArchOpcode uint8_op, ArchOpcode int16_op, @@ -2581,15 +2625,14 @@ void InstructionSelector::VisitWord32AtomicBinaryOperation( UNREACHABLE(); } - VisitAtomicBinop(this, node, opcode); + VisitAtomicBinop(this, node, opcode, AtomicWidth::kWord32); } -#define VISIT_ATOMIC_BINOP(op) \ - void InstructionSelector::VisitWord32Atomic##op(Node* node) { \ - VisitWord32AtomicBinaryOperation( \ - node, kWord32Atomic##op##Int8, kWord32Atomic##op##Uint8, \ - kWord32Atomic##op##Int16, kWord32Atomic##op##Uint16, \ - kWord32Atomic##op##Word32); \ +#define VISIT_ATOMIC_BINOP(op) \ + void InstructionSelector::VisitWord32Atomic##op(Node* node) { \ + VisitWord32AtomicBinaryOperation( \ + node, kAtomic##op##Int8, kAtomic##op##Uint8, kAtomic##op##Int16, \ + kAtomic##op##Uint16, kAtomic##op##Word32); \ } VISIT_ATOMIC_BINOP(Add) VISIT_ATOMIC_BINOP(Sub) @@ -2614,14 +2657,14 @@ void InstructionSelector::VisitWord64AtomicBinaryOperation( } else { UNREACHABLE(); } - VisitAtomicBinop(this, node, opcode); + VisitAtomicBinop(this, node, opcode, AtomicWidth::kWord64); } -#define VISIT_ATOMIC_BINOP(op) \ - void InstructionSelector::VisitWord64Atomic##op(Node* node) { \ - VisitWord64AtomicBinaryOperation( \ - node, kRiscvWord64Atomic##op##Uint8, kRiscvWord64Atomic##op##Uint16, \ - kRiscvWord64Atomic##op##Uint32, kRiscvWord64Atomic##op##Uint64); \ +#define VISIT_ATOMIC_BINOP(op) \ + void InstructionSelector::VisitWord64Atomic##op(Node* node) { \ + VisitWord64AtomicBinaryOperation(node, kAtomic##op##Uint8, \ + kAtomic##op##Uint16, kAtomic##op##Word32, \ + kRiscvWord64Atomic##op##Uint64); \ } VISIT_ATOMIC_BINOP(Add) VISIT_ATOMIC_BINOP(Sub) @@ -2640,6 +2683,7 @@ void InstructionSelector::VisitInt64AbsWithOverflow(Node* node) { #define SIMD_TYPE_LIST(V) \ V(F32x4) \ + V(I64x2) \ V(I32x4) \ V(I16x8) \ V(I8x16) @@ -2844,6 +2888,7 @@ SIMD_VISIT_SPLAT(F64x2) SIMD_VISIT_EXTRACT_LANE(F64x2, ) SIMD_VISIT_EXTRACT_LANE(F32x4, ) SIMD_VISIT_EXTRACT_LANE(I32x4, ) +SIMD_VISIT_EXTRACT_LANE(I64x2, ) SIMD_VISIT_EXTRACT_LANE(I16x8, U) SIMD_VISIT_EXTRACT_LANE(I16x8, S) SIMD_VISIT_EXTRACT_LANE(I8x16, U) @@ -2890,73 +2935,75 @@ struct ShuffleEntry { ArchOpcode opcode; }; -static const ShuffleEntry arch_shuffles[] = { - {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}, - kRiscvS32x4InterleaveRight}, - {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31}, - kRiscvS32x4InterleaveLeft}, - {{0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27}, - kRiscvS32x4PackEven}, - {{4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31}, - kRiscvS32x4PackOdd}, - {{0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27}, - kRiscvS32x4InterleaveEven}, - {{4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31}, - kRiscvS32x4InterleaveOdd}, - - {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23}, - kRiscvS16x8InterleaveRight}, - {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31}, - kRiscvS16x8InterleaveLeft}, - {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29}, - kRiscvS16x8PackEven}, - {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31}, - kRiscvS16x8PackOdd}, - {{0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29}, - kRiscvS16x8InterleaveEven}, - {{2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31}, - kRiscvS16x8InterleaveOdd}, - {{6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9}, - kRiscvS16x4Reverse}, - {{2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}, - kRiscvS16x2Reverse}, - - {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23}, - kRiscvS8x16InterleaveRight}, - {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31}, - kRiscvS8x16InterleaveLeft}, - {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30}, - kRiscvS8x16PackEven}, - {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31}, - kRiscvS8x16PackOdd}, - {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30}, - kRiscvS8x16InterleaveEven}, - {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31}, - kRiscvS8x16InterleaveOdd}, - {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8}, kRiscvS8x8Reverse}, - {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12}, kRiscvS8x4Reverse}, - {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14}, - kRiscvS8x2Reverse}}; - -bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table, - size_t num_entries, bool is_swizzle, - ArchOpcode* opcode) { - uint8_t mask = is_swizzle ? kSimd128Size - 1 : 2 * kSimd128Size - 1; - for (size_t i = 0; i < num_entries; ++i) { - const ShuffleEntry& entry = table[i]; - int j = 0; - for (; j < kSimd128Size; ++j) { - if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) { - break; - } - } - if (j == kSimd128Size) { - *opcode = entry.opcode; - return true; - } - } - return false; -} +// static const ShuffleEntry arch_shuffles[] = { +// {{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23}, +// kRiscvS32x4InterleaveRight}, +// {{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14, 15, 28, 29, 30, 31}, +// kRiscvS32x4InterleaveLeft}, +// {{0, 1, 2, 3, 8, 9, 10, 11, 16, 17, 18, 19, 24, 25, 26, 27}, +// kRiscvS32x4PackEven}, +// {{4, 5, 6, 7, 12, 13, 14, 15, 20, 21, 22, 23, 28, 29, 30, 31}, +// kRiscvS32x4PackOdd}, +// {{0, 1, 2, 3, 16, 17, 18, 19, 8, 9, 10, 11, 24, 25, 26, 27}, +// kRiscvS32x4InterleaveEven}, +// {{4, 5, 6, 7, 20, 21, 22, 23, 12, 13, 14, 15, 28, 29, 30, 31}, +// kRiscvS32x4InterleaveOdd}, + +// {{0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23}, +// kRiscvS16x8InterleaveRight}, +// {{8, 9, 24, 25, 10, 11, 26, 27, 12, 13, 28, 29, 14, 15, 30, 31}, +// kRiscvS16x8InterleaveLeft}, +// {{0, 1, 4, 5, 8, 9, 12, 13, 16, 17, 20, 21, 24, 25, 28, 29}, +// kRiscvS16x8PackEven}, +// {{2, 3, 6, 7, 10, 11, 14, 15, 18, 19, 22, 23, 26, 27, 30, 31}, +// kRiscvS16x8PackOdd}, +// {{0, 1, 16, 17, 4, 5, 20, 21, 8, 9, 24, 25, 12, 13, 28, 29}, +// kRiscvS16x8InterleaveEven}, +// {{2, 3, 18, 19, 6, 7, 22, 23, 10, 11, 26, 27, 14, 15, 30, 31}, +// kRiscvS16x8InterleaveOdd}, +// {{6, 7, 4, 5, 2, 3, 0, 1, 14, 15, 12, 13, 10, 11, 8, 9}, +// kRiscvS16x4Reverse}, +// {{2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}, +// kRiscvS16x2Reverse}, + +// {{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21, 6, 22, 7, 23}, +// kRiscvS8x16InterleaveRight}, +// {{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13, 29, 14, 30, 15, 31}, +// kRiscvS8x16InterleaveLeft}, +// {{0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30}, +// kRiscvS8x16PackEven}, +// {{1, 3, 5, 7, 9, 11, 13, 15, 17, 19, 21, 23, 25, 27, 29, 31}, +// kRiscvS8x16PackOdd}, +// {{0, 16, 2, 18, 4, 20, 6, 22, 8, 24, 10, 26, 12, 28, 14, 30}, +// kRiscvS8x16InterleaveEven}, +// {{1, 17, 3, 19, 5, 21, 7, 23, 9, 25, 11, 27, 13, 29, 15, 31}, +// kRiscvS8x16InterleaveOdd}, +// {{7, 6, 5, 4, 3, 2, 1, 0, 15, 14, 13, 12, 11, 10, 9, 8}, +// kRiscvS8x8Reverse}, +// {{3, 2, 1, 0, 7, 6, 5, 4, 11, 10, 9, 8, 15, 14, 13, 12}, +// kRiscvS8x4Reverse}, +// {{1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14}, +// kRiscvS8x2Reverse}}; + +// bool TryMatchArchShuffle(const uint8_t* shuffle, const ShuffleEntry* table, +// size_t num_entries, bool is_swizzle, +// ArchOpcode* opcode) { +// uint8_t mask = is_swizzle ? kSimd128Size - 1 : 2 * kSimd128Size - 1; +// for (size_t i = 0; i < num_entries; ++i) { +// const ShuffleEntry& entry = table[i]; +// int j = 0; +// for (; j < kSimd128Size; ++j) { +// if ((entry.shuffle[j] & mask) != (shuffle[j] & mask)) { +// break; +// } +// } +// if (j == kSimd128Size) { +// *opcode = entry.opcode; +// return true; +// } +// } +// return false; +// } } // namespace @@ -2964,29 +3011,29 @@ void InstructionSelector::VisitI8x16Shuffle(Node* node) { uint8_t shuffle[kSimd128Size]; bool is_swizzle; CanonicalizeShuffle(node, shuffle, &is_swizzle); - uint8_t shuffle32x4[4]; - ArchOpcode opcode; - if (TryMatchArchShuffle(shuffle, arch_shuffles, arraysize(arch_shuffles), - is_swizzle, &opcode)) { - VisitRRR(this, opcode, node); - return; - } Node* input0 = node->InputAt(0); Node* input1 = node->InputAt(1); - uint8_t offset; RiscvOperandGenerator g(this); - if (wasm::SimdShuffle::TryMatchConcat(shuffle, &offset)) { - Emit(kRiscvS8x16Concat, g.DefineSameAsFirst(node), g.UseRegister(input1), - g.UseRegister(input0), g.UseImmediate(offset)); - return; - } - if (wasm::SimdShuffle::TryMatch32x4Shuffle(shuffle, shuffle32x4)) { - Emit(kRiscvS32x4Shuffle, g.DefineAsRegister(node), g.UseRegister(input0), - g.UseRegister(input1), - g.UseImmediate(wasm::SimdShuffle::Pack4Lanes(shuffle32x4))); - return; - } - Emit(kRiscvS8x16Shuffle, g.DefineAsRegister(node), g.UseRegister(input0), + // uint8_t shuffle32x4[4]; + // ArchOpcode opcode; + // if (TryMatchArchShuffle(shuffle, arch_shuffles, arraysize(arch_shuffles), + // is_swizzle, &opcode)) { + // VisitRRR(this, opcode, node); + // return; + // } + // uint8_t offset; + // if (wasm::SimdShuffle::TryMatchConcat(shuffle, &offset)) { + // Emit(kRiscvS8x16Concat, g.DefineSameAsFirst(node), g.UseRegister(input1), + // g.UseRegister(input0), g.UseImmediate(offset)); + // return; + // } + // if (wasm::SimdShuffle::TryMatch32x4Shuffle(shuffle, shuffle32x4)) { + // Emit(kRiscvS32x4Shuffle, g.DefineAsRegister(node), g.UseRegister(input0), + // g.UseRegister(input1), + // g.UseImmediate(wasm::SimdShuffle::Pack4Lanes(shuffle32x4))); + // return; + // } + Emit(kRiscvI8x16Shuffle, g.DefineAsRegister(node), g.UseRegister(input0), g.UseRegister(input1), g.UseImmediate(wasm::SimdShuffle::Pack4Lanes(shuffle)), g.UseImmediate(wasm::SimdShuffle::Pack4Lanes(shuffle + 4)), |