diff options
Diffstat (limited to 'Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp')
| -rw-r--r-- | Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp | 4163 |
1 files changed, 2562 insertions, 1601 deletions
diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp index bc21f929f..1048c3b0f 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT32_64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, 2012, 2013, 2014 Apple Inc. All rights reserved. + * Copyright (C) 2011-2016 Apple Inc. All rights reserved. * Copyright (C) 2011 Intel Corporation. All rights reserved. * * Redistribution and use in source and binary forms, with or without @@ -30,14 +30,22 @@ #if ENABLE(DFG_JIT) #include "ArrayPrototype.h" +#include "CallFrameShuffler.h" #include "DFGAbstractInterpreterInlines.h" #include "DFGCallArrayAllocatorSlowPathGenerator.h" #include "DFGOperations.h" #include "DFGSlowPathGenerator.h" -#include "Debugger.h" -#include "JSActivation.h" +#include "DirectArguments.h" +#include "GetterSetter.h" +#include "HasOwnPropertyCache.h" +#include "HashMapImpl.h" +#include "JSEnvironmentRecord.h" +#include "JSPropertyNameEnumerator.h" #include "ObjectPrototype.h" -#include "Operations.h" +#include "JSCInlines.h" +#include "SetupVarargsFrame.h" +#include "TypeProfilerLog.h" +#include "Watchdog.h" namespace JSC { namespace DFG { @@ -57,11 +65,12 @@ bool SpeculativeJIT::fillJSValue(Edge edge, GPRReg& tagGPR, GPRReg& payloadGPR, if (edge->hasConstant()) { tagGPR = allocate(); payloadGPR = allocate(); - m_jit.move(Imm32(valueOfJSConstant(edge.node()).tag()), tagGPR); - m_jit.move(Imm32(valueOfJSConstant(edge.node()).payload()), payloadGPR); + JSValue value = edge->asJSValue(); + m_jit.move(Imm32(value.tag()), tagGPR); + m_jit.move(Imm32(value.payload()), payloadGPR); m_gprs.retain(tagGPR, virtualRegister, SpillOrderConstant); m_gprs.retain(payloadGPR, virtualRegister, SpillOrderConstant); - info.fillJSValue(*m_stream, tagGPR, payloadGPR, isInt32Constant(edge.node()) ? DataFormatJSInt32 : DataFormatJS); + info.fillJSValue(*m_stream, tagGPR, payloadGPR, DataFormatJS); } else { DataFormat spillFormat = info.spillFormat(); ASSERT(spillFormat != DataFormatNone && spillFormat != DataFormatStorage); @@ -106,7 +115,7 @@ bool SpeculativeJIT::fillJSValue(Edge edge, GPRReg& tagGPR, GPRReg& payloadGPR, m_gprs.lock(gpr); } tagGPR = allocate(); - uint32_t tag = JSValue::EmptyValueTag; + int32_t tag = JSValue::EmptyValueTag; DataFormat fillFormat = DataFormatJS; switch (info.registerFormat()) { case DataFormatInt32: @@ -134,20 +143,6 @@ bool SpeculativeJIT::fillJSValue(Edge edge, GPRReg& tagGPR, GPRReg& payloadGPR, } case DataFormatJSDouble: - case DataFormatDouble: { - FPRReg oldFPR = info.fpr(); - m_fprs.lock(oldFPR); - tagGPR = allocate(); - payloadGPR = allocate(); - boxDouble(oldFPR, tagGPR, payloadGPR); - m_fprs.unlock(oldFPR); - m_fprs.release(oldFPR); - m_gprs.retain(tagGPR, virtualRegister, SpillOrderJS); - m_gprs.retain(payloadGPR, virtualRegister, SpillOrderJS); - info.fillJSValue(*m_stream, tagGPR, payloadGPR, DataFormatJS); - return true; - } - case DataFormatJS: case DataFormatJSInt32: case DataFormatJSCell: @@ -160,6 +155,7 @@ bool SpeculativeJIT::fillJSValue(Edge edge, GPRReg& tagGPR, GPRReg& payloadGPR, } case DataFormatStorage: + case DataFormatDouble: // this type currently never occurs RELEASE_ASSERT_NOT_REACHED(); @@ -169,12 +165,40 @@ bool SpeculativeJIT::fillJSValue(Edge edge, GPRReg& tagGPR, GPRReg& payloadGPR, } } -void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseTagGPROrNone, GPRReg basePayloadGPR, GPRReg resultTagGPR, GPRReg resultPayloadGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode) +void SpeculativeJIT::cachedGetById(CodeOrigin origin, JSValueRegs base, JSValueRegs result, unsigned identifierNumber, JITCompiler::Jump slowPathTarget , SpillRegistersMode mode, AccessType type) +{ + cachedGetById(origin, base.tagGPR(), base.payloadGPR(), result.tagGPR(), result.payloadGPR(), identifierNumber, slowPathTarget, mode, type); +} + +void SpeculativeJIT::cachedGetById( + CodeOrigin codeOrigin, GPRReg baseTagGPROrNone, GPRReg basePayloadGPR, GPRReg resultTagGPR, GPRReg resultPayloadGPR, + unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode, AccessType type) { + // This is a hacky fix for when the register allocator decides to alias the base payload with the result tag. This only happens + // in the case of GetByIdFlush, which has a relatively expensive register allocation story already so we probably don't need to + // trip over one move instruction. + if (basePayloadGPR == resultTagGPR) { + RELEASE_ASSERT(basePayloadGPR != resultPayloadGPR); + + if (baseTagGPROrNone == resultPayloadGPR) { + m_jit.swap(basePayloadGPR, baseTagGPROrNone); + baseTagGPROrNone = resultTagGPR; + } else + m_jit.move(basePayloadGPR, resultPayloadGPR); + basePayloadGPR = resultPayloadGPR; + } + + RegisterSet usedRegisters = this->usedRegisters(); + if (spillMode == DontSpill) { + // We've already flushed registers to the stack, we don't need to spill these. + usedRegisters.set(JSValueRegs(baseTagGPROrNone, basePayloadGPR), false); + usedRegisters.set(JSValueRegs(resultTagGPR, resultPayloadGPR), false); + } + + CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(codeOrigin, m_stream->size()); JITGetByIdGenerator gen( - m_jit.codeBlock(), codeOrigin, usedRegisters(), GPRInfo::callFrameRegister, - JSValueRegs(baseTagGPROrNone, basePayloadGPR), - JSValueRegs(resultTagGPR, resultPayloadGPR), spillMode != NeedToSpill); + m_jit.codeBlock(), codeOrigin, callSite, usedRegisters, identifierUID(identifierNumber), + JSValueRegs(baseTagGPROrNone, basePayloadGPR), JSValueRegs(resultTagGPR, resultPayloadGPR), type); gen.generateFastPath(m_jit); @@ -182,31 +206,43 @@ void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseTagGPROrNon if (slowPathTarget.isSet()) slowCases.append(slowPathTarget); slowCases.append(gen.slowPathJump()); - - OwnPtr<SlowPathGenerator> slowPath; + + J_JITOperation_ESsiJI getByIdFunction; + if (type == AccessType::Get) + getByIdFunction = operationGetByIdOptimize; + else + getByIdFunction = operationTryGetByIdOptimize; + + std::unique_ptr<SlowPathGenerator> slowPath; if (baseTagGPROrNone == InvalidGPRReg) { slowPath = slowPathCall( - slowCases, this, operationGetByIdOptimize, + slowCases, this, getByIdFunction, JSValueRegs(resultTagGPR, resultPayloadGPR), gen.stubInfo(), static_cast<int32_t>(JSValue::CellTag), basePayloadGPR, identifierUID(identifierNumber)); } else { slowPath = slowPathCall( - slowCases, this, operationGetByIdOptimize, - JSValueRegs(resultTagGPR, resultPayloadGPR), gen.stubInfo(), baseTagGPROrNone, - basePayloadGPR, identifierUID(identifierNumber)); + slowCases, this, getByIdFunction, + JSValueRegs(resultTagGPR, resultPayloadGPR), gen.stubInfo(), JSValueRegs(baseTagGPROrNone, basePayloadGPR), identifierUID(identifierNumber)); } - + m_jit.addGetById(gen, slowPath.get()); - addSlowPathGenerator(slowPath.release()); + addSlowPathGenerator(WTFMove(slowPath)); } -void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg basePayloadGPR, GPRReg valueTagGPR, GPRReg valuePayloadGPR, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget) +void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg basePayloadGPR, GPRReg valueTagGPR, GPRReg valuePayloadGPR, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode) { + RegisterSet usedRegisters = this->usedRegisters(); + if (spillMode == DontSpill) { + // We've already flushed registers to the stack, we don't need to spill these. + usedRegisters.set(basePayloadGPR, false); + usedRegisters.set(JSValueRegs(valueTagGPR, valuePayloadGPR), false); + } + CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(codeOrigin, m_stream->size()); JITPutByIdGenerator gen( - m_jit.codeBlock(), codeOrigin, usedRegisters(), GPRInfo::callFrameRegister, + m_jit.codeBlock(), codeOrigin, callSite, usedRegisters, JSValueRegs::payloadOnly(basePayloadGPR), JSValueRegs(valueTagGPR, valuePayloadGPR), - scratchGPR, false, m_jit.ecmaModeFor(codeOrigin), putKind); + scratchGPR, m_jit.ecmaModeFor(codeOrigin), putKind); gen.generateFastPath(m_jit); @@ -215,17 +251,17 @@ void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg basePayloadGPR, slowCases.append(slowPathTarget); slowCases.append(gen.slowPathJump()); - OwnPtr<SlowPathGenerator> slowPath = slowPathCall( - slowCases, this, gen.slowPathFunction(), NoResult, gen.stubInfo(), valueTagGPR, - valuePayloadGPR, basePayloadGPR, identifierUID(identifierNumber)); + auto slowPath = slowPathCall( + slowCases, this, gen.slowPathFunction(), NoResult, gen.stubInfo(), JSValueRegs(valueTagGPR, valuePayloadGPR), + basePayloadGPR, identifierUID(identifierNumber)); m_jit.addPutById(gen, slowPath.get()); - addSlowPathGenerator(slowPath.release()); + addSlowPathGenerator(WTFMove(slowPath)); } -void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool invert) +void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNullOrUndefined(Edge operand) { - JSValueOperand arg(this, operand); + JSValueOperand arg(this, operand, ManualOperandSpeculation); GPRReg argTagGPR = arg.tagGPR(); GPRReg argPayloadGPR = arg.payloadGPR(); @@ -236,29 +272,32 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool inv JITCompiler::Jump notMasqueradesAsUndefined; if (masqueradesAsUndefinedWatchpointIsStillValid()) { if (!isKnownCell(operand.node())) - notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag)); - - m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultPayloadGPR); + notCell = m_jit.branchIfNotCell(arg.jsValueRegs()); + + m_jit.move(TrustedImm32(0), resultPayloadGPR); notMasqueradesAsUndefined = m_jit.jump(); } else { GPRTemporary localGlobalObject(this); GPRTemporary remoteGlobalObject(this); if (!isKnownCell(operand.node())) - notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag)); - - m_jit.loadPtr(JITCompiler::Address(argPayloadGPR, JSCell::structureOffset()), resultPayloadGPR); - JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(resultPayloadGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined)); + notCell = m_jit.branchIfNotCell(arg.jsValueRegs()); - m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultPayloadGPR); + JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8( + JITCompiler::NonZero, + JITCompiler::Address(argPayloadGPR, JSCell::typeInfoFlagsOffset()), + JITCompiler::TrustedImm32(MasqueradesAsUndefined)); + + m_jit.move(TrustedImm32(0), resultPayloadGPR); notMasqueradesAsUndefined = m_jit.jump(); isMasqueradesAsUndefined.link(&m_jit); GPRReg localGlobalObjectGPR = localGlobalObject.gpr(); GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr(); - m_jit.move(JITCompiler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)), localGlobalObjectGPR); + m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)), localGlobalObjectGPR); + m_jit.loadPtr(JITCompiler::Address(argPayloadGPR, JSCell::structureIDOffset()), resultPayloadGPR); m_jit.loadPtr(JITCompiler::Address(resultPayloadGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR); - m_jit.compare32(invert ? JITCompiler::NotEqual : JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, resultPayloadGPR); + m_jit.compare32(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, resultPayloadGPR); } if (!isKnownCell(operand.node())) { @@ -267,9 +306,8 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool inv notCell.link(&m_jit); // null or undefined? COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag); - m_jit.move(argTagGPR, resultPayloadGPR); - m_jit.or32(TrustedImm32(1), resultPayloadGPR); - m_jit.compare32(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultPayloadGPR, TrustedImm32(JSValue::NullTag), resultPayloadGPR); + m_jit.or32(TrustedImm32(1), argTagGPR, resultPayloadGPR); + m_jit.compare32(JITCompiler::Equal, resultPayloadGPR, TrustedImm32(JSValue::NullTag), resultPayloadGPR); done.link(&m_jit); } @@ -279,11 +317,12 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool inv booleanResult(resultPayloadGPR, m_currentNode); } -void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branchNode, bool invert) +void SpeculativeJIT::nonSpeculativePeepholeBranchNullOrUndefined(Edge operand, Node* branchNode) { - BasicBlock* taken = branchNode->takenBlock(); - BasicBlock* notTaken = branchNode->notTakenBlock(); - + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; + + bool invert = false; if (taken == nextBlock()) { invert = !invert; BasicBlock* tmp = taken; @@ -291,7 +330,7 @@ void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branch notTaken = tmp; } - JSValueOperand arg(this, operand); + JSValueOperand arg(this, operand, ManualOperandSpeculation); GPRReg argTagGPR = arg.tagGPR(); GPRReg argPayloadGPR = arg.payloadGPR(); @@ -302,22 +341,25 @@ void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branch if (masqueradesAsUndefinedWatchpointIsStillValid()) { if (!isKnownCell(operand.node())) - notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag)); - + notCell = m_jit.branchIfNotCell(arg.jsValueRegs()); + jump(invert ? taken : notTaken, ForceJump); } else { GPRTemporary localGlobalObject(this); GPRTemporary remoteGlobalObject(this); if (!isKnownCell(operand.node())) - notCell = m_jit.branch32(MacroAssembler::NotEqual, argTagGPR, TrustedImm32(JSValue::CellTag)); - - m_jit.loadPtr(JITCompiler::Address(argPayloadGPR, JSCell::structureOffset()), resultGPR); - branchTest8(JITCompiler::Zero, JITCompiler::Address(resultGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined), invert ? taken : notTaken); + notCell = m_jit.branchIfNotCell(arg.jsValueRegs()); + + branchTest8(JITCompiler::Zero, + JITCompiler::Address(argPayloadGPR, JSCell::typeInfoFlagsOffset()), + JITCompiler::TrustedImm32(MasqueradesAsUndefined), + invert ? taken : notTaken); GPRReg localGlobalObjectGPR = localGlobalObject.gpr(); GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr(); - m_jit.move(TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)), localGlobalObjectGPR); + m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)), localGlobalObjectGPR); + m_jit.loadPtr(JITCompiler::Address(argPayloadGPR, JSCell::structureIDOffset()), resultGPR); m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR); branchPtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, invert ? notTaken : taken); } @@ -328,41 +370,17 @@ void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branch notCell.link(&m_jit); // null or undefined? COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag); - m_jit.move(argTagGPR, resultGPR); - m_jit.or32(TrustedImm32(1), resultGPR); + m_jit.or32(TrustedImm32(1), argTagGPR, resultGPR); branch32(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm32(JSValue::NullTag), taken); } jump(notTaken); } -bool SpeculativeJIT::nonSpeculativeCompareNull(Node* node, Edge operand, bool invert) -{ - unsigned branchIndexInBlock = detectPeepHoleBranch(); - if (branchIndexInBlock != UINT_MAX) { - Node* branchNode = m_block->at(branchIndexInBlock); - - ASSERT(node->adjustedRefCount() == 1); - - nonSpeculativePeepholeBranchNull(operand, branchNode, invert); - - use(node->child1()); - use(node->child2()); - m_indexInBlock = branchIndexInBlock; - m_currentNode = branchNode; - - return true; - } - - nonSpeculativeNonPeepholeCompareNull(operand, invert); - - return false; -} - void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction) { - BasicBlock* taken = branchNode->takenBlock(); - BasicBlock* notTaken = branchNode->notTakenBlock(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; JITCompiler::ResultCondition callResultCondition = JITCompiler::NonZero; @@ -378,6 +396,8 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, JSValueOperand arg1(this, node->child1()); JSValueOperand arg2(this, node->child2()); + JSValueRegs arg1Regs = arg1.jsValueRegs(); + JSValueRegs arg2Regs = arg2.jsValueRegs(); GPRReg arg1TagGPR = arg1.tagGPR(); GPRReg arg1PayloadGPR = arg1.payloadGPR(); GPRReg arg2TagGPR = arg2.tagGPR(); @@ -386,14 +406,15 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, JITCompiler::JumpList slowPath; if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) { - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); arg1.use(); arg2.use(); flushRegisters(); - callOperation(helperFunction, resultGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR); + callOperation(helperFunction, resultGPR, arg1Regs, arg2Regs); + m_jit.exceptionCheck(); branchTest32(callResultCondition, resultGPR, taken); } else { @@ -416,7 +437,8 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, slowPath.link(&m_jit); silentSpillAllRegisters(resultGPR); - callOperation(helperFunction, resultGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR); + callOperation(helperFunction, resultGPR, arg1Regs, arg2Regs); + m_jit.exceptionCheck(); silentFillAllRegisters(resultGPR); branchTest32(callResultCondition, resultGPR, taken); @@ -438,7 +460,7 @@ public: S_JITOperation_EJJ function, GPRReg result, GPRReg arg1Tag, GPRReg arg1Payload, GPRReg arg2Tag, GPRReg arg2Payload) : CallSlowPathGenerator<JumpType, S_JITOperation_EJJ, GPRReg>( - from, jit, function, NeedToSpill, result) + from, jit, function, NeedToSpill, ExceptionCheckRequirement::CheckNeeded, result) , m_arg1Tag(arg1Tag) , m_arg1Payload(arg1Payload) , m_arg2Tag(arg2Tag) @@ -452,8 +474,7 @@ protected: this->setUp(jit); this->recordCall( jit->callOperation( - this->m_function, this->m_result, m_arg1Tag, m_arg1Payload, m_arg2Tag, - m_arg2Payload)); + this->m_function, this->m_result, JSValueRegs(m_arg1Tag, m_arg1Payload), JSValueRegs(m_arg2Tag, m_arg2Payload))); jit->m_jit.and32(JITCompiler::TrustedImm32(1), this->m_result); this->tearDown(jit); } @@ -477,14 +498,15 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler JITCompiler::JumpList slowPath; if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) { - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultPayloadGPR = result.gpr(); arg1.use(); arg2.use(); flushRegisters(); - callOperation(helperFunction, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR); + callOperation(helperFunction, resultPayloadGPR, arg1.jsValueRegs(), arg2.jsValueRegs()); + m_jit.exceptionCheck(); booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly); } else { @@ -502,10 +524,9 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler m_jit.compare32(cond, arg1PayloadGPR, arg2PayloadGPR, resultPayloadGPR); if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) { - addSlowPathGenerator(adoptPtr( - new CompareAndBoxBooleanSlowPathGenerator<JITCompiler::JumpList>( + addSlowPathGenerator(std::make_unique<CompareAndBoxBooleanSlowPathGenerator<JITCompiler::JumpList>>( slowPath, this, helperFunction, resultPayloadGPR, arg1TagGPR, - arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR))); + arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR)); } booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly); @@ -514,8 +535,8 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode, bool invert) { - BasicBlock* taken = branchNode->takenBlock(); - BasicBlock* notTaken = branchNode->notTakenBlock(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; // The branch instruction will branch to the taken block. // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through. @@ -528,10 +549,10 @@ void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode JSValueOperand arg1(this, node->child1()); JSValueOperand arg2(this, node->child2()); - GPRReg arg1TagGPR = arg1.tagGPR(); GPRReg arg1PayloadGPR = arg1.payloadGPR(); - GPRReg arg2TagGPR = arg2.tagGPR(); GPRReg arg2PayloadGPR = arg2.payloadGPR(); + JSValueRegs arg1Regs = arg1.jsValueRegs(); + JSValueRegs arg2Regs = arg2.jsValueRegs(); GPRTemporary resultPayload(this, Reuse, arg1, PayloadWord); GPRReg resultPayloadGPR = resultPayload.gpr(); @@ -545,7 +566,8 @@ void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode branchPtr(JITCompiler::Equal, arg1PayloadGPR, arg2PayloadGPR, invert ? notTaken : taken); silentSpillAllRegisters(resultPayloadGPR); - callOperation(operationCompareStrictEqCell, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR); + callOperation(operationCompareStrictEqCell, resultPayloadGPR, arg1Regs, arg2Regs); + m_jit.exceptionCheck(); silentFillAllRegisters(resultPayloadGPR); branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultPayloadGPR, taken); @@ -553,7 +575,8 @@ void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode // FIXME: Add fast paths for twoCells, number etc. silentSpillAllRegisters(resultPayloadGPR); - callOperation(operationCompareStrictEq, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR); + callOperation(operationCompareStrictEq, resultPayloadGPR, arg1Regs, arg2Regs); + m_jit.exceptionCheck(); silentFillAllRegisters(resultPayloadGPR); branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultPayloadGPR, taken); @@ -566,10 +589,10 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert) { JSValueOperand arg1(this, node->child1()); JSValueOperand arg2(this, node->child2()); - GPRReg arg1TagGPR = arg1.tagGPR(); GPRReg arg1PayloadGPR = arg1.payloadGPR(); - GPRReg arg2TagGPR = arg2.tagGPR(); GPRReg arg2PayloadGPR = arg2.payloadGPR(); + JSValueRegs arg1Regs = arg1.jsValueRegs(); + JSValueRegs arg2Regs = arg2.jsValueRegs(); GPRTemporary resultPayload(this, Reuse, arg1, PayloadWord); GPRReg resultPayloadGPR = resultPayload.gpr(); @@ -589,7 +612,8 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert) notEqualCase.link(&m_jit); silentSpillAllRegisters(resultPayloadGPR); - callOperation(operationCompareStrictEqCell, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR); + callOperation(operationCompareStrictEqCell, resultPayloadGPR, arg1Regs, arg2Regs); + m_jit.exceptionCheck(); silentFillAllRegisters(resultPayloadGPR); m_jit.andPtr(JITCompiler::TrustedImm32(1), resultPayloadGPR); @@ -599,8 +623,9 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert) // FIXME: Add fast paths. silentSpillAllRegisters(resultPayloadGPR); - callOperation(operationCompareStrictEq, resultPayloadGPR, arg1TagGPR, arg1PayloadGPR, arg2TagGPR, arg2PayloadGPR); + callOperation(operationCompareStrictEq, resultPayloadGPR, arg1Regs, arg2Regs); silentFillAllRegisters(resultPayloadGPR); + m_jit.exceptionCheck(); m_jit.andPtr(JITCompiler::TrustedImm32(1), resultPayloadGPR); } @@ -608,92 +633,463 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert) booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly); } -void SpeculativeJIT::emitCall(Node* node) +void SpeculativeJIT::compileMiscStrictEq(Node* node) { - if (node->op() != Call) - ASSERT(node->op() == Construct); - - // For constructors, the this argument is not passed but we have to make space - // for it. - int dummyThisArgument = node->op() == Call ? 0 : 1; - - CallLinkInfo::CallType callType = node->op() == Call ? CallLinkInfo::Call : CallLinkInfo::Construct; + JSValueOperand op1(this, node->child1(), ManualOperandSpeculation); + JSValueOperand op2(this, node->child2(), ManualOperandSpeculation); + GPRTemporary result(this); + + if (node->child1().useKind() == MiscUse) + speculateMisc(node->child1(), op1.jsValueRegs()); + if (node->child2().useKind() == MiscUse) + speculateMisc(node->child2(), op2.jsValueRegs()); + + m_jit.move(TrustedImm32(0), result.gpr()); + JITCompiler::Jump notEqual = m_jit.branch32(JITCompiler::NotEqual, op1.tagGPR(), op2.tagGPR()); + m_jit.compare32(JITCompiler::Equal, op1.payloadGPR(), op2.payloadGPR(), result.gpr()); + notEqual.link(&m_jit); + booleanResult(result.gpr(), node); +} - Edge calleeEdge = m_jit.graph().m_varArgChildren[node->firstChild()]; - JSValueOperand callee(this, calleeEdge); - GPRReg calleeTagGPR = callee.tagGPR(); - GPRReg calleePayloadGPR = callee.payloadGPR(); - use(calleeEdge); +void SpeculativeJIT::emitCall(Node* node) +{ + CallLinkInfo::CallType callType; + bool isVarargs = false; + bool isForwardVarargs = false; + bool isTail = false; + bool isDirect = false; + bool isEmulatedTail = false; + switch (node->op()) { + case Call: + case CallEval: + callType = CallLinkInfo::Call; + break; + case TailCall: + callType = CallLinkInfo::TailCall; + isTail = true; + break; + case TailCallInlinedCaller: + callType = CallLinkInfo::Call; + isEmulatedTail = true; + break; + case Construct: + callType = CallLinkInfo::Construct; + break; + case CallVarargs: + callType = CallLinkInfo::CallVarargs; + isVarargs = true; + break; + case TailCallVarargs: + callType = CallLinkInfo::TailCallVarargs; + isVarargs = true; + isTail = true; + break; + case TailCallVarargsInlinedCaller: + callType = CallLinkInfo::CallVarargs; + isVarargs = true; + isEmulatedTail = true; + break; + case ConstructVarargs: + callType = CallLinkInfo::ConstructVarargs; + isVarargs = true; + break; + case CallForwardVarargs: + callType = CallLinkInfo::CallVarargs; + isForwardVarargs = true; + break; + case TailCallForwardVarargs: + callType = CallLinkInfo::TailCallVarargs; + isTail = true; + isForwardVarargs = true; + break; + case TailCallForwardVarargsInlinedCaller: + callType = CallLinkInfo::CallVarargs; + isEmulatedTail = true; + isForwardVarargs = true; + break; + case ConstructForwardVarargs: + callType = CallLinkInfo::ConstructVarargs; + isForwardVarargs = true; + break; + case DirectCall: + callType = CallLinkInfo::DirectCall; + isDirect = true; + break; + case DirectConstruct: + callType = CallLinkInfo::DirectConstruct; + isDirect = true; + break; + case DirectTailCall: + callType = CallLinkInfo::DirectTailCall; + isTail = true; + isDirect = true; + break; + case DirectTailCallInlinedCaller: + callType = CallLinkInfo::DirectCall; + isEmulatedTail = true; + isDirect = true; + break; + default: + DFG_CRASH(m_jit.graph(), node, "bad node type"); + break; + } - // The call instruction's first child is either the function (normal call) or the - // receiver (method call). subsequent children are the arguments. - int numPassedArgs = node->numChildren() - 1; + Edge calleeEdge = m_jit.graph().child(node, 0); + GPRReg calleeTagGPR = InvalidGPRReg; + GPRReg calleePayloadGPR = InvalidGPRReg; + CallFrameShuffleData shuffleData; + + ExecutableBase* executable = nullptr; + FunctionExecutable* functionExecutable = nullptr; + if (isDirect) { + executable = node->castOperand<ExecutableBase*>(); + functionExecutable = jsDynamicCast<FunctionExecutable*>(*m_jit.vm(), executable); + } + + unsigned numPassedArgs = 0; + unsigned numAllocatedArgs = 0; - int numArgs = numPassedArgs + dummyThisArgument; + // Gotta load the arguments somehow. Varargs is trickier. + if (isVarargs || isForwardVarargs) { + RELEASE_ASSERT(!isDirect); + CallVarargsData* data = node->callVarargsData(); - m_jit.store32(MacroAssembler::TrustedImm32(numArgs), calleeFramePayloadSlot(numArgs, JSStack::ArgumentCount)); - m_jit.storePtr(GPRInfo::callFrameRegister, calleeFrameCallerFrame(numArgs)); - m_jit.store32(calleePayloadGPR, calleeFramePayloadSlot(numArgs, JSStack::Callee)); - m_jit.store32(calleeTagGPR, calleeFrameTagSlot(numArgs, JSStack::Callee)); + unsigned numUsedStackSlots = m_jit.graph().m_nextMachineLocal; + + if (isForwardVarargs) { + flushRegisters(); + if (node->child3()) + use(node->child3()); + + GPRReg scratchGPR1; + GPRReg scratchGPR2; + GPRReg scratchGPR3; + + scratchGPR1 = JITCompiler::selectScratchGPR(); + scratchGPR2 = JITCompiler::selectScratchGPR(scratchGPR1); + scratchGPR3 = JITCompiler::selectScratchGPR(scratchGPR1, scratchGPR2); + + m_jit.move(TrustedImm32(numUsedStackSlots), scratchGPR2); + JITCompiler::JumpList slowCase; + InlineCallFrame* inlineCallFrame; + if (node->child3()) + inlineCallFrame = node->child3()->origin.semantic.inlineCallFrame; + else + inlineCallFrame = node->origin.semantic.inlineCallFrame; + // emitSetupVarargsFrameFastCase modifies the stack pointer if it succeeds. + emitSetupVarargsFrameFastCase(m_jit, scratchGPR2, scratchGPR1, scratchGPR2, scratchGPR3, inlineCallFrame, data->firstVarArgOffset, slowCase); + JITCompiler::Jump done = m_jit.jump(); + slowCase.link(&m_jit); + callOperation(operationThrowStackOverflowForVarargs); + m_jit.exceptionCheck(); + m_jit.abortWithReason(DFGVarargsThrowingPathDidNotThrow); + done.link(&m_jit); + } else { + GPRReg argumentsPayloadGPR; + GPRReg argumentsTagGPR; + GPRReg scratchGPR1; + GPRReg scratchGPR2; + GPRReg scratchGPR3; + + auto loadArgumentsGPR = [&] (GPRReg reservedGPR) { + if (reservedGPR != InvalidGPRReg) + lock(reservedGPR); + JSValueOperand arguments(this, node->child3()); + argumentsTagGPR = arguments.tagGPR(); + argumentsPayloadGPR = arguments.payloadGPR(); + if (reservedGPR != InvalidGPRReg) + unlock(reservedGPR); + flushRegisters(); + + scratchGPR1 = JITCompiler::selectScratchGPR(argumentsPayloadGPR, argumentsTagGPR, reservedGPR); + scratchGPR2 = JITCompiler::selectScratchGPR(argumentsPayloadGPR, argumentsTagGPR, scratchGPR1, reservedGPR); + scratchGPR3 = JITCompiler::selectScratchGPR(argumentsPayloadGPR, argumentsTagGPR, scratchGPR1, scratchGPR2, reservedGPR); + }; + + loadArgumentsGPR(InvalidGPRReg); + + DFG_ASSERT(m_jit.graph(), node, isFlushed()); - for (int i = 0; i < numPassedArgs; i++) { - Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i]; - JSValueOperand arg(this, argEdge); - GPRReg argTagGPR = arg.tagGPR(); - GPRReg argPayloadGPR = arg.payloadGPR(); - use(argEdge); + // Right now, arguments is in argumentsTagGPR/argumentsPayloadGPR and the register file is + // flushed. + callOperation(operationSizeFrameForVarargs, GPRInfo::returnValueGPR, JSValueRegs(argumentsTagGPR, argumentsPayloadGPR), numUsedStackSlots, data->firstVarArgOffset); + m_jit.exceptionCheck(); + + // Now we have the argument count of the callee frame, but we've lost the arguments operand. + // Reconstruct the arguments operand while preserving the callee frame. + loadArgumentsGPR(GPRInfo::returnValueGPR); + m_jit.move(TrustedImm32(numUsedStackSlots), scratchGPR1); + emitSetVarargsFrame(m_jit, GPRInfo::returnValueGPR, false, scratchGPR1, scratchGPR1); + m_jit.addPtr(TrustedImm32(-(sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 6 * sizeof(void*)))), scratchGPR1, JITCompiler::stackPointerRegister); + + callOperation(operationSetupVarargsFrame, GPRInfo::returnValueGPR, scratchGPR1, JSValueRegs(argumentsTagGPR, argumentsPayloadGPR), data->firstVarArgOffset, GPRInfo::returnValueGPR); + m_jit.exceptionCheck(); + m_jit.addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), GPRInfo::returnValueGPR, JITCompiler::stackPointerRegister); + } + + DFG_ASSERT(m_jit.graph(), node, isFlushed()); + + // We don't need the arguments array anymore. + if (isVarargs) + use(node->child3()); + + // Now set up the "this" argument. + JSValueOperand thisArgument(this, node->child2()); + GPRReg thisArgumentTagGPR = thisArgument.tagGPR(); + GPRReg thisArgumentPayloadGPR = thisArgument.payloadGPR(); + thisArgument.use(); + + m_jit.store32(thisArgumentTagGPR, JITCompiler::calleeArgumentTagSlot(0)); + m_jit.store32(thisArgumentPayloadGPR, JITCompiler::calleeArgumentPayloadSlot(0)); + } else { + // The call instruction's first child is either the function (normal call) or the + // receiver (method call). subsequent children are the arguments. + numPassedArgs = node->numChildren() - 1; + numAllocatedArgs = numPassedArgs; + + if (functionExecutable) { + // Allocate more args if this would let us avoid arity checks. This is throttled by + // CallLinkInfo's limit. It's probably good to throttle it - if the callee wants a + // ginormous amount of argument space then it's better for them to do it so that when we + // make calls to other things, we don't waste space. + unsigned desiredNumAllocatedArgs = static_cast<unsigned>(functionExecutable->parameterCount()) + 1; + if (desiredNumAllocatedArgs <= Options::maximumDirectCallStackSize()) { + numAllocatedArgs = std::max(numAllocatedArgs, desiredNumAllocatedArgs); + + // Whoever converts to DirectCall should do this adjustment. It's too late for us to + // do this adjustment now since we will have already emitted code that relied on the + // value of m_parameterSlots. + DFG_ASSERT( + m_jit.graph(), node, + Graph::parameterSlotsForArgCount(numAllocatedArgs) + <= m_jit.graph().m_parameterSlots); + } + } - m_jit.store32(argTagGPR, calleeArgumentTagSlot(numArgs, i + dummyThisArgument)); - m_jit.store32(argPayloadGPR, calleeArgumentPayloadSlot(numArgs, i + dummyThisArgument)); + if (isTail) { + JSValueOperand callee(this, calleeEdge); + calleeTagGPR = callee.tagGPR(); + calleePayloadGPR = callee.payloadGPR(); + if (!isDirect) + use(calleeEdge); + + shuffleData.numLocals = m_jit.graph().frameRegisterCount(); + shuffleData.callee = ValueRecovery::inPair(calleeTagGPR, calleePayloadGPR); + shuffleData.args.resize(numAllocatedArgs); + shuffleData.numPassedArgs = numPassedArgs; + + for (unsigned i = 0; i < numPassedArgs; ++i) { + Edge argEdge = m_jit.graph().varArgChild(node, i + 1); + GenerationInfo& info = generationInfo(argEdge.node()); + if (!isDirect) + use(argEdge); + shuffleData.args[i] = info.recovery(argEdge->virtualRegister()); + } + + for (unsigned i = numPassedArgs; i < numAllocatedArgs; ++i) + shuffleData.args[i] = ValueRecovery::constant(jsUndefined()); + } else { + m_jit.store32(MacroAssembler::TrustedImm32(numPassedArgs), m_jit.calleeFramePayloadSlot(CallFrameSlot::argumentCount)); + + for (unsigned i = 0; i < numPassedArgs; i++) { + Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i]; + JSValueOperand arg(this, argEdge); + GPRReg argTagGPR = arg.tagGPR(); + GPRReg argPayloadGPR = arg.payloadGPR(); + use(argEdge); + + m_jit.store32(argTagGPR, m_jit.calleeArgumentTagSlot(i)); + m_jit.store32(argPayloadGPR, m_jit.calleeArgumentPayloadSlot(i)); + } + + for (unsigned i = numPassedArgs; i < numAllocatedArgs; ++i) + m_jit.storeTrustedValue(jsUndefined(), JITCompiler::calleeArgumentSlot(i)); + } } - flushRegisters(); + if (!isTail || isVarargs || isForwardVarargs) { + JSValueOperand callee(this, calleeEdge); + calleeTagGPR = callee.tagGPR(); + calleePayloadGPR = callee.payloadGPR(); + use(calleeEdge); + m_jit.store32(calleePayloadGPR, m_jit.calleeFramePayloadSlot(CallFrameSlot::callee)); + m_jit.store32(calleeTagGPR, m_jit.calleeFrameTagSlot(CallFrameSlot::callee)); - GPRResult resultPayload(this); - GPRResult2 resultTag(this); - GPRReg resultPayloadGPR = resultPayload.gpr(); - GPRReg resultTagGPR = resultTag.gpr(); + if (!isTail) + flushRegisters(); + } JITCompiler::DataLabelPtr targetToCheck; JITCompiler::JumpList slowPath; - m_jit.emitStoreCodeOrigin(node->codeOrigin); + CodeOrigin staticOrigin = node->origin.semantic; + ASSERT(!isTail || !staticOrigin.inlineCallFrame || !staticOrigin.inlineCallFrame->getCallerSkippingTailCalls()); + ASSERT(!isEmulatedTail || (staticOrigin.inlineCallFrame && staticOrigin.inlineCallFrame->getCallerSkippingTailCalls())); + CodeOrigin dynamicOrigin = + isEmulatedTail ? *staticOrigin.inlineCallFrame->getCallerSkippingTailCalls() : staticOrigin; + CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(dynamicOrigin, m_stream->size()); + + CallLinkInfo* info = m_jit.codeBlock()->addCallLinkInfo(); + info->setUpCall(callType, node->origin.semantic, calleePayloadGPR); + + auto setResultAndResetStack = [&] () { + GPRFlushedCallResult resultPayload(this); + GPRFlushedCallResult2 resultTag(this); + GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRReg resultTagGPR = resultTag.gpr(); + + m_jit.setupResults(resultPayloadGPR, resultTagGPR); + + jsValueResult(resultTagGPR, resultPayloadGPR, node, DataFormatJS, UseChildrenCalledExplicitly); + // After the calls are done, we need to reestablish our stack + // pointer. We rely on this for varargs calls, calls with arity + // mismatch (the callframe is slided) and tail calls. + m_jit.addPtr(TrustedImm32(m_jit.graph().stackPointerOffset() * sizeof(Register)), GPRInfo::callFrameRegister, JITCompiler::stackPointerRegister); + }; + + if (node->op() == CallEval) { + // We want to call operationCallEval but we don't want to overwrite the parameter area in + // which we have created a prototypical eval call frame. This means that we have to + // subtract stack to make room for the call. Lucky for us, at this point we have the whole + // register file to ourselves. + + m_jit.emitStoreCallSiteIndex(callSite); + m_jit.addPtr(TrustedImm32(-static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC))), JITCompiler::stackPointerRegister, GPRInfo::regT0); + m_jit.storePtr(GPRInfo::callFrameRegister, JITCompiler::Address(GPRInfo::regT0, CallFrame::callerFrameOffset())); + + // Now we need to make room for: + // - The caller frame and PC of a call to operationCallEval. + // - Potentially two arguments on the stack. + unsigned requiredBytes = sizeof(CallerFrameAndPC) + sizeof(ExecState*) * 2; + requiredBytes = WTF::roundUpToMultipleOf(stackAlignmentBytes(), requiredBytes); + m_jit.subPtr(TrustedImm32(requiredBytes), JITCompiler::stackPointerRegister); + m_jit.setupArgumentsWithExecState(GPRInfo::regT0); + prepareForExternalCall(); + m_jit.appendCall(operationCallEval); + m_jit.exceptionCheck(); + JITCompiler::Jump done = m_jit.branch32(JITCompiler::NotEqual, GPRInfo::returnValueGPR2, TrustedImm32(JSValue::EmptyValueTag)); + + // This is the part where we meant to make a normal call. Oops. + m_jit.addPtr(TrustedImm32(requiredBytes), JITCompiler::stackPointerRegister); + m_jit.load32(JITCompiler::calleeFrameSlot(CallFrameSlot::callee).withOffset(PayloadOffset), GPRInfo::regT0); + m_jit.load32(JITCompiler::calleeFrameSlot(CallFrameSlot::callee).withOffset(TagOffset), GPRInfo::regT1); + m_jit.emitDumbVirtualCall(info); + + done.link(&m_jit); + setResultAndResetStack(); + return; + } + + if (isDirect) { + info->setExecutableDuringCompilation(executable); + info->setMaxNumArguments(numAllocatedArgs); + + if (isTail) { + RELEASE_ASSERT(node->op() == DirectTailCall); + + JITCompiler::PatchableJump patchableJump = m_jit.patchableJump(); + JITCompiler::Label mainPath = m_jit.label(); + + m_jit.emitStoreCallSiteIndex(callSite); + + info->setFrameShuffleData(shuffleData); + CallFrameShuffler(m_jit, shuffleData).prepareForTailCall(); + + JITCompiler::Call call = m_jit.nearTailCall(); + + JITCompiler::Label slowPath = m_jit.label(); + patchableJump.m_jump.linkTo(slowPath, &m_jit); + + silentSpillAllRegisters(InvalidGPRReg); + callOperation(operationLinkDirectCall, info, calleePayloadGPR); + silentFillAllRegisters(InvalidGPRReg); + m_jit.exceptionCheck(); + m_jit.jump().linkTo(mainPath, &m_jit); + + useChildren(node); + + m_jit.addJSDirectTailCall(patchableJump, call, slowPath, info); + return; + } + + JITCompiler::Label mainPath = m_jit.label(); + + m_jit.emitStoreCallSiteIndex(callSite); + + JITCompiler::Call call = m_jit.nearCall(); + JITCompiler::Jump done = m_jit.jump(); + + JITCompiler::Label slowPath = m_jit.label(); + if (isX86()) + m_jit.pop(JITCompiler::selectScratchGPR(calleePayloadGPR)); + + callOperation(operationLinkDirectCall, info, calleePayloadGPR); + m_jit.exceptionCheck(); + m_jit.jump().linkTo(mainPath, &m_jit); + + done.link(&m_jit); + + setResultAndResetStack(); + + m_jit.addJSDirectCall(call, slowPath, info); + return; + } - m_jit.addPtr(TrustedImm32(calleeFrameOffset(numArgs)), GPRInfo::callFrameRegister); + m_jit.emitStoreCallSiteIndex(callSite); - slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, calleeTagGPR, TrustedImm32(JSValue::CellTag))); + slowPath.append(m_jit.branchIfNotCell(JSValueRegs(calleeTagGPR, calleePayloadGPR))); slowPath.append(m_jit.branchPtrWithPatch(MacroAssembler::NotEqual, calleePayloadGPR, targetToCheck)); - m_jit.loadPtr(MacroAssembler::Address(calleePayloadGPR, OBJECT_OFFSETOF(JSFunction, m_scope)), resultPayloadGPR); - m_jit.storePtr(resultPayloadGPR, MacroAssembler::Address(GPRInfo::callFrameRegister, static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload))); - m_jit.store32(MacroAssembler::TrustedImm32(JSValue::CellTag), MacroAssembler::Address(GPRInfo::callFrameRegister, static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag))); - JITCompiler::Call fastCall = m_jit.nearCall(); + if (isTail) { + if (node->op() == TailCall) { + info->setFrameShuffleData(shuffleData); + CallFrameShuffler(m_jit, shuffleData).prepareForTailCall(); + } else { + m_jit.emitRestoreCalleeSaves(); + m_jit.prepareForTailCallSlow(); + } + } + + JITCompiler::Call fastCall = isTail ? m_jit.nearTailCall() : m_jit.nearCall(); JITCompiler::Jump done = m_jit.jump(); slowPath.link(&m_jit); - // Callee payload needs to be in regT0, tag in regT1 - if (calleeTagGPR == GPRInfo::regT0) { - if (calleePayloadGPR == GPRInfo::regT1) - m_jit.swap(GPRInfo::regT1, GPRInfo::regT0); - else { - m_jit.move(calleeTagGPR, GPRInfo::regT1); + if (node->op() == TailCall) { + CallFrameShuffler callFrameShuffler(m_jit, shuffleData); + callFrameShuffler.setCalleeJSValueRegs(JSValueRegs( + GPRInfo::regT1, GPRInfo::regT0)); + callFrameShuffler.prepareForSlowPath(); + } else { + // Callee payload needs to be in regT0, tag in regT1 + if (calleeTagGPR == GPRInfo::regT0) { + if (calleePayloadGPR == GPRInfo::regT1) + m_jit.swap(GPRInfo::regT1, GPRInfo::regT0); + else { + m_jit.move(calleeTagGPR, GPRInfo::regT1); + m_jit.move(calleePayloadGPR, GPRInfo::regT0); + } + } else { m_jit.move(calleePayloadGPR, GPRInfo::regT0); + m_jit.move(calleeTagGPR, GPRInfo::regT1); } - } else { - m_jit.move(calleePayloadGPR, GPRInfo::regT0); - m_jit.move(calleeTagGPR, GPRInfo::regT1); + + if (isTail) + m_jit.emitRestoreCalleeSaves(); } + + m_jit.move(TrustedImmPtr(info), GPRInfo::regT2); JITCompiler::Call slowCall = m_jit.nearCall(); done.link(&m_jit); - m_jit.setupResults(resultPayloadGPR, resultTagGPR); - - jsValueResult(resultTagGPR, resultPayloadGPR, node, DataFormatJS, UseChildrenCalledExplicitly); + if (isTail) + m_jit.abortWithReason(JITDidReturnFromTailCall); + else + setResultAndResetStack(); - m_jit.addJSCall(fastCall, slowCall, targetToCheck, callType, calleePayloadGPR, node->codeOrigin); + m_jit.addJSCall(fastCall, slowCall, targetToCheck, info); } template<bool strict> @@ -701,23 +1097,24 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnF { AbstractValue& value = m_state.forNode(edge); SpeculatedType type = value.m_type; - ASSERT(edge.useKind() != KnownInt32Use || !(value.m_type & ~SpecInt32)); - m_interpreter.filter(value, SpecInt32); + ASSERT(edge.useKind() != KnownInt32Use || !(value.m_type & ~SpecInt32Only)); + + m_interpreter.filter(value, SpecInt32Only); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + returnFormat = DataFormatInt32; + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if ((edge->hasConstant() && !isInt32Constant(edge.node())) || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - returnFormat = DataFormatInt32; - return allocate(); - } - if (edge->hasConstant()) { - ASSERT(isInt32Constant(edge.node())); + ASSERT(edge->isInt32Constant()); GPRReg gpr = allocate(); - m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr); + m_jit.move(MacroAssembler::Imm32(edge->asInt32()), gpr); m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); info.fillInt32(*m_stream, gpr); returnFormat = DataFormatInt32; @@ -725,10 +1122,11 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnF } DataFormat spillFormat = info.spillFormat(); + ASSERT_UNUSED(spillFormat, (spillFormat & DataFormatJS) || spillFormat == DataFormatInt32); // If we know this was spilled as an integer we can fill without checking. - if (type & ~SpecInt32) + if (type & ~SpecInt32Only) speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag))); GPRReg gpr = allocate(); @@ -746,7 +1144,7 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnF GPRReg payloadGPR = info.payloadGPR(); m_gprs.lock(tagGPR); m_gprs.lock(payloadGPR); - if (type & ~SpecInt32) + if (type & ~SpecInt32Only) speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::NotEqual, tagGPR, TrustedImm32(JSValue::Int32Tag))); m_gprs.unlock(tagGPR); m_gprs.release(tagGPR); @@ -765,16 +1163,12 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnF return gpr; } - case DataFormatDouble: case DataFormatCell: case DataFormatBoolean: case DataFormatJSDouble: case DataFormatJSCell: case DataFormatJSBoolean: - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - returnFormat = DataFormatInt32; - return allocate(); - + case DataFormatDouble: case DataFormatStorage: default: RELEASE_ASSERT_NOT_REACHED(); @@ -797,136 +1191,34 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Strict(Edge edge) FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge) { - AbstractValue& value = m_state.forNode(edge); - SpeculatedType type = value.m_type; - ASSERT(edge.useKind() != KnownNumberUse || !(value.m_type & ~SpecFullNumber)); - m_interpreter.filter(value, SpecFullNumber); + ASSERT(isDouble(edge.useKind())); + ASSERT(edge->hasDoubleResult()); VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); if (info.registerFormat() == DataFormatNone) { if (edge->hasConstant()) { - if (isInt32Constant(edge.node())) { - GPRReg gpr = allocate(); - m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr); - m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - info.fillInt32(*m_stream, gpr); - unlock(gpr); - } else if (isNumberConstant(edge.node())) { - FPRReg fpr = fprAllocate(); - m_jit.loadDouble(addressOfDoubleConstant(edge.node()), fpr); - m_fprs.retain(fpr, virtualRegister, SpillOrderConstant); - info.fillDouble(*m_stream, fpr); - return fpr; - } else { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return fprAllocate(); - } - } else { - DataFormat spillFormat = info.spillFormat(); - ASSERT((spillFormat & DataFormatJS) || spillFormat == DataFormatInt32); - if (spillFormat == DataFormatJSDouble || spillFormat == DataFormatDouble) { - FPRReg fpr = fprAllocate(); - m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr); - m_fprs.retain(fpr, virtualRegister, SpillOrderSpilled); - info.fillDouble(*m_stream, fpr); - return fpr; - } - + RELEASE_ASSERT(edge->isNumberConstant()); FPRReg fpr = fprAllocate(); - JITCompiler::Jump hasUnboxedDouble; - - if (spillFormat != DataFormatJSInt32 && spillFormat != DataFormatInt32) { - JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::Int32Tag)); - if (type & ~SpecFullNumber) - speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::AboveOrEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::LowestTag))); - m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr); - hasUnboxedDouble = m_jit.jump(); - - isInteger.link(&m_jit); - } - - m_jit.convertInt32ToDouble(JITCompiler::payloadFor(virtualRegister), fpr); - - if (hasUnboxedDouble.isSet()) - hasUnboxedDouble.link(&m_jit); - - m_fprs.retain(fpr, virtualRegister, SpillOrderSpilled); + m_jit.loadDouble(TrustedImmPtr(m_jit.addressOfDoubleConstant(edge.node())), fpr); + m_fprs.retain(fpr, virtualRegister, SpillOrderConstant); info.fillDouble(*m_stream, fpr); - info.killSpilled(); return fpr; } - } - - switch (info.registerFormat()) { - case DataFormatJS: - case DataFormatJSInt32: { - GPRReg tagGPR = info.tagGPR(); - GPRReg payloadGPR = info.payloadGPR(); + + RELEASE_ASSERT(info.spillFormat() == DataFormatDouble); FPRReg fpr = fprAllocate(); - - m_gprs.lock(tagGPR); - m_gprs.lock(payloadGPR); - - JITCompiler::Jump hasUnboxedDouble; - - if (info.registerFormat() != DataFormatJSInt32) { - FPRTemporary scratch(this); - JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, tagGPR, TrustedImm32(JSValue::Int32Tag)); - if (type & ~SpecFullNumber) - speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::AboveOrEqual, tagGPR, TrustedImm32(JSValue::LowestTag))); - unboxDouble(tagGPR, payloadGPR, fpr, scratch.fpr()); - hasUnboxedDouble = m_jit.jump(); - isInteger.link(&m_jit); - } - - m_jit.convertInt32ToDouble(payloadGPR, fpr); - - if (hasUnboxedDouble.isSet()) - hasUnboxedDouble.link(&m_jit); - - m_gprs.release(tagGPR); - m_gprs.release(payloadGPR); - m_gprs.unlock(tagGPR); - m_gprs.unlock(payloadGPR); - m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); + m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr); + m_fprs.retain(fpr, virtualRegister, SpillOrderSpilled); info.fillDouble(*m_stream, fpr); - info.killSpilled(); return fpr; } - case DataFormatInt32: { - FPRReg fpr = fprAllocate(); - GPRReg gpr = info.gpr(); - m_gprs.lock(gpr); - m_jit.convertInt32ToDouble(gpr, fpr); - m_gprs.unlock(gpr); - return fpr; - } - - case DataFormatJSDouble: - case DataFormatDouble: { - FPRReg fpr = info.fpr(); - m_fprs.lock(fpr); - return fpr; - } - - case DataFormatNone: - case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); - - case DataFormatCell: - case DataFormatJSCell: - case DataFormatBoolean: - case DataFormatJSBoolean: - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return fprAllocate(); - - default: - RELEASE_ASSERT_NOT_REACHED(); - return InvalidFPRReg; - } + RELEASE_ASSERT(info.registerFormat() == DataFormatDouble); + FPRReg fpr = info.fpr(); + m_fprs.lock(fpr); + return fpr; } GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) @@ -934,33 +1226,37 @@ GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) AbstractValue& value = m_state.forNode(edge); SpeculatedType type = value.m_type; ASSERT((edge.useKind() != KnownCellUse && edge.useKind() != KnownStringUse) || !(value.m_type & ~SpecCell)); + m_interpreter.filter(value, SpecCell); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if (info.spillFormat() == DataFormatInt32 || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - if (edge->hasConstant()) { - JSValue jsValue = valueOfJSConstant(edge.node()); GPRReg gpr = allocate(); - if (jsValue.isCell()) { - m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - m_jit.move(MacroAssembler::TrustedImmPtr(jsValue.asCell()), gpr); - info.fillCell(*m_stream, gpr); - return gpr; - } - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); + m_jit.move(TrustedImmPtr(edge->constant()), gpr); + info.fillCell(*m_stream, gpr); return gpr; } ASSERT((info.spillFormat() & DataFormatJS) || info.spillFormat() == DataFormatCell); - if (type & ~SpecCell) - speculationCheck(BadType, JSValueSource(JITCompiler::addressFor(virtualRegister)), edge, m_jit.branch32(MacroAssembler::NotEqual, JITCompiler::tagFor(virtualRegister), TrustedImm32(JSValue::CellTag))); + if (type & ~SpecCell) { + speculationCheck( + BadType, + JSValueSource(JITCompiler::addressFor(virtualRegister)), + edge, + m_jit.branch32( + MacroAssembler::NotEqual, + JITCompiler::tagFor(virtualRegister), + TrustedImm32(JSValue::CellTag))); + } GPRReg gpr = allocate(); m_jit.load32(JITCompiler::payloadFor(virtualRegister), gpr); m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); @@ -980,8 +1276,11 @@ GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) GPRReg payloadGPR = info.payloadGPR(); m_gprs.lock(tagGPR); m_gprs.lock(payloadGPR); - if (type & ~SpecCell) - speculationCheck(BadType, JSValueRegs(tagGPR, payloadGPR), edge, m_jit.branch32(MacroAssembler::NotEqual, tagGPR, TrustedImm32(JSValue::CellTag))); + if (type & ~SpecCell) { + speculationCheck( + BadType, JSValueRegs(tagGPR, payloadGPR), edge, + m_jit.branchIfNotCell(info.jsValueRegs())); + } m_gprs.unlock(tagGPR); m_gprs.release(tagGPR); m_gprs.release(payloadGPR); @@ -993,12 +1292,9 @@ GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) case DataFormatJSInt32: case DataFormatInt32: case DataFormatJSDouble: - case DataFormatDouble: case DataFormatJSBoolean: case DataFormatBoolean: - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - + case DataFormatDouble: case DataFormatStorage: RELEASE_ASSERT_NOT_REACHED(); @@ -1012,27 +1308,25 @@ GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge) { AbstractValue& value = m_state.forNode(edge); SpeculatedType type = value.m_type; + ASSERT(edge.useKind() != KnownBooleanUse || !(value.m_type & ~SpecBoolean)); + m_interpreter.filter(value, SpecBoolean); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if (info.spillFormat() == DataFormatInt32 || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - if (edge->hasConstant()) { - JSValue jsValue = valueOfJSConstant(edge.node()); + JSValue jsValue = edge->asJSValue(); GPRReg gpr = allocate(); - if (jsValue.isBoolean()) { - m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - m_jit.move(MacroAssembler::TrustedImm32(jsValue.asBoolean()), gpr); - info.fillBoolean(*m_stream, gpr); - return gpr; - } - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); + m_jit.move(MacroAssembler::TrustedImm32(jsValue.asBoolean()), gpr); + info.fillBoolean(*m_stream, gpr); return gpr; } @@ -1074,12 +1368,9 @@ GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge) case DataFormatJSInt32: case DataFormatInt32: case DataFormatJSDouble: - case DataFormatDouble: case DataFormatJSCell: case DataFormatCell: - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - + case DataFormatDouble: case DataFormatStorage: RELEASE_ASSERT_NOT_REACHED(); @@ -1089,45 +1380,6 @@ GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge) } } -JITCompiler::Jump SpeculativeJIT::convertToDouble(JSValueOperand& op, FPRReg result) -{ - FPRTemporary scratch(this); - - GPRReg opPayloadGPR = op.payloadGPR(); - GPRReg opTagGPR = op.tagGPR(); - FPRReg scratchFPR = scratch.fpr(); - - JITCompiler::Jump isInteger = m_jit.branch32(MacroAssembler::Equal, opTagGPR, TrustedImm32(JSValue::Int32Tag)); - JITCompiler::Jump notNumber = m_jit.branch32(MacroAssembler::AboveOrEqual, opPayloadGPR, TrustedImm32(JSValue::LowestTag)); - - unboxDouble(opTagGPR, opPayloadGPR, result, scratchFPR); - JITCompiler::Jump done = m_jit.jump(); - - isInteger.link(&m_jit); - m_jit.convertInt32ToDouble(opPayloadGPR, result); - - done.link(&m_jit); - - return notNumber; -} - -void SpeculativeJIT::compileBaseValueStoreBarrier(Edge& baseEdge, Edge& valueEdge) -{ -#if ENABLE(GGC) - ASSERT(!isKnownNotCell(valueEdge.node())); - - SpeculateCellOperand base(this, baseEdge); - JSValueOperand value(this, valueEdge); - GPRTemporary scratch1(this); - GPRTemporary scratch2(this); - - writeBarrier(base.gpr(), value.tagGPR(), valueEdge, scratch1.gpr(), scratch2.gpr()); -#else - UNUSED_PARAM(baseEdge); - UNUSED_PARAM(valueEdge); -#endif -} - void SpeculativeJIT::compileObjectEquality(Node* node) { SpeculateCellOperand op1(this, node->child1()); @@ -1137,41 +1389,24 @@ void SpeculativeJIT::compileObjectEquality(Node* node) if (masqueradesAsUndefinedWatchpointIsStillValid()) { DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchIfNotObject(op1GPR)); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchIfNotObject(op2GPR)); } else { - GPRTemporary structure(this); - GPRReg structureGPR = structure.gpr(); - - m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); - speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), + JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchIfNotObject(op1GPR)); + speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); - m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); - speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), + JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchIfNotObject(op2GPR)); + speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchTest8( - MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::NonZero, + MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } @@ -1188,6 +1423,57 @@ void SpeculativeJIT::compileObjectEquality(Node* node) booleanResult(resultPayloadGPR, node); } +void SpeculativeJIT::compileObjectStrictEquality(Edge objectChild, Edge otherChild) +{ + SpeculateCellOperand op1(this, objectChild); + JSValueOperand op2(this, otherChild); + + GPRReg op1GPR = op1.gpr(); + GPRReg op2GPR = op2.payloadGPR(); + + DFG_TYPE_CHECK(JSValueSource::unboxedCell(op1GPR), objectChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); + + GPRTemporary resultPayload(this, Reuse, op1); + GPRReg resultPayloadGPR = resultPayload.gpr(); + + MacroAssembler::Jump op2CellJump = m_jit.branchIfCell(op2.jsValueRegs()); + + m_jit.move(TrustedImm32(0), resultPayloadGPR); + MacroAssembler::Jump op2NotCellJump = m_jit.jump(); + + // At this point we know that we can perform a straight-forward equality comparison on pointer + // values because we are doing strict equality. + op2CellJump.link(&m_jit); + m_jit.compare32(MacroAssembler::Equal, op1GPR, op2GPR, resultPayloadGPR); + + op2NotCellJump.link(&m_jit); + booleanResult(resultPayloadGPR, m_currentNode); +} + +void SpeculativeJIT::compilePeepHoleObjectStrictEquality(Edge objectChild, Edge otherChild, Node* branchNode) +{ + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; + + SpeculateCellOperand op1(this, objectChild); + JSValueOperand op2(this, otherChild); + + GPRReg op1GPR = op1.gpr(); + GPRReg op2GPR = op2.payloadGPR(); + + DFG_TYPE_CHECK(JSValueSource::unboxedCell(op1GPR), objectChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); + + branch32(MacroAssembler::NotEqual, op2.tagGPR(), TrustedImm32(JSValue::CellTag), notTaken); + + if (taken == nextBlock()) { + branch32(MacroAssembler::NotEqual, op1GPR, op2GPR, notTaken); + jump(taken); + } else { + branch32(MacroAssembler::Equal, op1GPR, op2GPR, taken); + jump(notTaken); + } +} + void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild) { SpeculateCellOperand op1(this, leftChild); @@ -1198,66 +1484,39 @@ void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge r GPRReg op2TagGPR = op2.tagGPR(); GPRReg op2PayloadGPR = op2.payloadGPR(); GPRReg resultGPR = result.gpr(); - GPRTemporary structure; - GPRReg structureGPR = InvalidGPRReg; bool masqueradesAsUndefinedWatchpointValid = masqueradesAsUndefinedWatchpointIsStillValid(); - if (!masqueradesAsUndefinedWatchpointValid) { - // The masquerades as undefined case will use the structure register, so allocate it here. - // Do this at the top of the function to avoid branching around a register allocation. - GPRTemporary realStructure(this); - structure.adopt(realStructure); - structureGPR = structure.gpr(); - } - if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } // It seems that most of the time when programs do a == b where b may be either null/undefined // or an object, b is usually an object. Balance the branches to make that case fast. - MacroAssembler::Jump rightNotCell = - m_jit.branch32(MacroAssembler::NotEqual, op2TagGPR, TrustedImm32(JSValue::CellTag)); + MacroAssembler::Jump rightNotCell = m_jit.branchIfNotCell(op2.jsValueRegs()); // We know that within this branch, rightChild must be a cell. if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( - JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject, - m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2PayloadGPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject, - m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2PayloadGPR)); speculationCheck(BadType, JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op2PayloadGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } @@ -1272,8 +1531,7 @@ void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge r // We know that within this branch, rightChild must not be a cell. Check if that is enough to // prove that it is either null or undefined. if (needsTypeCheck(rightChild, SpecCell | SpecOther)) { - m_jit.move(op2TagGPR, resultGPR); - m_jit.or32(TrustedImm32(1), resultGPR); + m_jit.or32(TrustedImm32(1), op2TagGPR, resultGPR); typeCheck( JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, SpecCell | SpecOther, @@ -1294,8 +1552,8 @@ void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge r void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild, Node* branchNode) { - BasicBlock* taken = branchNode->takenBlock(); - BasicBlock* notTaken = branchNode->notTakenBlock(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; SpeculateCellOperand op1(this, leftChild); JSValueOperand op2(this, rightChild, ManualOperandSpeculation); @@ -1305,65 +1563,40 @@ void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild GPRReg op2TagGPR = op2.tagGPR(); GPRReg op2PayloadGPR = op2.payloadGPR(); GPRReg resultGPR = result.gpr(); - GPRTemporary structure; - GPRReg structureGPR = InvalidGPRReg; bool masqueradesAsUndefinedWatchpointValid = masqueradesAsUndefinedWatchpointIsStillValid(); - if (!masqueradesAsUndefinedWatchpointValid) { - // The masquerades as undefined case will use the structure register, so allocate it here. - // Do this at the top of the function to avoid branching around a register allocation. - GPRTemporary realStructure(this); - structure.adopt(realStructure); - structureGPR = structure.gpr(); - } - if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } // It seems that most of the time when programs do a == b where b may be either null/undefined // or an object, b is usually an object. Balance the branches to make that case fast. - MacroAssembler::Jump rightNotCell = - m_jit.branch32(MacroAssembler::NotEqual, op2TagGPR, TrustedImm32(JSValue::CellTag)); + MacroAssembler::Jump rightNotCell = m_jit.branchIfNotCell(op2.jsValueRegs()); // We know that within this branch, rightChild must be a cell. if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject, - m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + m_jit.branchIfNotObject(op2PayloadGPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op2PayloadGPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, (~SpecCell) | SpecObject, - m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + m_jit.branchIfNotObject(op2PayloadGPR)); speculationCheck(BadType, JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op2PayloadGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } @@ -1380,8 +1613,7 @@ void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild jump(notTaken, ForceJump); rightNotCell.link(&m_jit); - m_jit.move(op2TagGPR, resultGPR); - m_jit.or32(TrustedImm32(1), resultGPR); + m_jit.or32(TrustedImm32(1), op2TagGPR, resultGPR); typeCheck( JSValueRegs(op2TagGPR, op2PayloadGPR), rightChild, SpecCell | SpecOther, @@ -1393,6 +1625,33 @@ void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild jump(notTaken); } +void SpeculativeJIT::compileSymbolUntypedEquality(Node* node, Edge symbolEdge, Edge untypedEdge) +{ + SpeculateCellOperand symbol(this, symbolEdge); + JSValueOperand untyped(this, untypedEdge); + + GPRReg symbolGPR = symbol.gpr(); + GPRReg untypedGPR = untyped.payloadGPR(); + + speculateSymbol(symbolEdge, symbolGPR); + + GPRTemporary resultPayload(this, Reuse, symbol); + GPRReg resultPayloadGPR = resultPayload.gpr(); + + MacroAssembler::Jump untypedCellJump = m_jit.branchIfCell(untyped.jsValueRegs()); + + m_jit.move(TrustedImm32(0), resultPayloadGPR); + MacroAssembler::Jump untypedNotCellJump = m_jit.jump(); + + // At this point we know that we can perform a straight-forward equality comparison on pointer + // values because we are doing strict equality. + untypedCellJump.link(&m_jit); + m_jit.compare32(MacroAssembler::Equal, symbolGPR, untypedGPR, resultPayloadGPR); + + untypedNotCellJump.link(&m_jit); + booleanResult(resultPayloadGPR, node); +} + void SpeculativeJIT::compileInt32Compare(Node* node, MacroAssembler::RelationalCondition condition) { SpeculateInt32Operand op1(this, node->child1()); @@ -1440,35 +1699,28 @@ void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse) structureGPR = structure.gpr(); } - MacroAssembler::Jump notCell = m_jit.branch32(MacroAssembler::NotEqual, valueTagGPR, TrustedImm32(JSValue::CellTag)); + MacroAssembler::Jump notCell = m_jit.branchIfNotCell(value.jsValueRegs()); if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject, - m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + m_jit.branchIfNotObject(valuePayloadGPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()), structureGPR); - DFG_TYPE_CHECK( JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject, - m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + m_jit.branchIfNotObject(valuePayloadGPR)); MacroAssembler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8( MacroAssembler::Zero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(valuePayloadGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined)); + m_jit.loadPtr(MacroAssembler::Address(valuePayloadGPR, JSCell::structureIDOffset()), structureGPR); speculationCheck(BadType, JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, m_jit.branchPtr( MacroAssembler::Equal, MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()), - MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)))); + TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)))); isNotMasqueradesAsUndefined.link(&m_jit); } @@ -1479,8 +1731,7 @@ void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse) COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag); if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) { - m_jit.move(valueTagGPR, resultPayloadGPR); - m_jit.or32(TrustedImm32(1), resultPayloadGPR); + m_jit.or32(TrustedImm32(1), valueTagGPR, resultPayloadGPR); typeCheck( JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, SpecCell | SpecOther, m_jit.branch32( @@ -1498,7 +1749,8 @@ void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse) void SpeculativeJIT::compileLogicalNot(Node* node) { switch (node->child1().useKind()) { - case BooleanUse: { + case BooleanUse: + case KnownBooleanUse: { SpeculateBooleanOperand value(this, node->child1()); GPRTemporary result(this, Reuse, value); m_jit.xor32(TrustedImm32(1), value.gpr(), result.gpr()); @@ -1519,7 +1771,7 @@ void SpeculativeJIT::compileLogicalNot(Node* node) return; } - case NumberUse: { + case DoubleRepUse: { SpeculateDoubleOperand value(this, node->child1()); FPRTemporary scratch(this); GPRTemporary resultPayload(this); @@ -1533,29 +1785,26 @@ void SpeculativeJIT::compileLogicalNot(Node* node) case UntypedUse: { JSValueOperand arg1(this, node->child1()); - GPRTemporary resultPayload(this, Reuse, arg1, PayloadWord); - GPRReg arg1TagGPR = arg1.tagGPR(); - GPRReg arg1PayloadGPR = arg1.payloadGPR(); - GPRReg resultPayloadGPR = resultPayload.gpr(); - - arg1.use(); + GPRTemporary result(this); + GPRTemporary temp(this); + FPRTemporary valueFPR(this); + FPRTemporary tempFPR(this); - JITCompiler::Jump slowCase = m_jit.branch32(JITCompiler::NotEqual, arg1TagGPR, TrustedImm32(JSValue::BooleanTag)); - - m_jit.move(arg1PayloadGPR, resultPayloadGPR); + GPRReg resultGPR = result.gpr(); - addSlowPathGenerator( - slowPathCall( - slowCase, this, operationConvertJSValueToBoolean, resultPayloadGPR, arg1TagGPR, - arg1PayloadGPR)); - - m_jit.xor32(TrustedImm32(1), resultPayloadGPR); - booleanResult(resultPayloadGPR, node, UseChildrenCalledExplicitly); + bool shouldCheckMasqueradesAsUndefined = !masqueradesAsUndefinedWatchpointIsStillValid(); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + bool negateResult = true; + m_jit.emitConvertValueToBoolean(arg1.jsValueRegs(), resultGPR, temp.gpr(), valueFPR.fpr(), tempFPR.fpr(), shouldCheckMasqueradesAsUndefined, globalObject, negateResult); + booleanResult(resultGPR, node); return; } case StringUse: return compileStringZeroLength(node); + case StringOrOtherUse: + return compileLogicalNotStringOrOther(node); + default: RELEASE_ASSERT_NOT_REACHED(); break; @@ -1570,31 +1819,27 @@ void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BasicBlock* taken, Ba GPRReg valuePayloadGPR = value.payloadGPR(); GPRReg scratchGPR = scratch.gpr(); - MacroAssembler::Jump notCell = m_jit.branch32(MacroAssembler::NotEqual, valueTagGPR, TrustedImm32(JSValue::CellTag)); + MacroAssembler::Jump notCell = m_jit.branchIfNotCell(value.jsValueRegs()); if (masqueradesAsUndefinedWatchpointIsStillValid()) { DFG_TYPE_CHECK( JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject, - m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + m_jit.branchIfNotObject(valuePayloadGPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(valuePayloadGPR, JSCell::structureOffset()), scratchGPR); - DFG_TYPE_CHECK( JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, (~SpecCell) | SpecObject, - m_jit.branchPtr( - MacroAssembler::Equal, - scratchGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + m_jit.branchIfNotObject(valuePayloadGPR)); - JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::Zero, MacroAssembler::Address(scratchGPR, Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)); + JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8( + JITCompiler::Zero, + MacroAssembler::Address(valuePayloadGPR, JSCell::typeInfoFlagsOffset()), + TrustedImm32(MasqueradesAsUndefined)); + m_jit.loadPtr(MacroAssembler::Address(valuePayloadGPR, JSCell::structureIDOffset()), scratchGPR); speculationCheck(BadType, JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, m_jit.branchPtr( MacroAssembler::Equal, MacroAssembler::Address(scratchGPR, Structure::globalObjectOffset()), - MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)))); + TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)))); isNotMasqueradesAsUndefined.link(&m_jit); } @@ -1604,8 +1849,7 @@ void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BasicBlock* taken, Ba COMPILE_ASSERT((JSValue::UndefinedTag | 1) == JSValue::NullTag, UndefinedTag_OR_1_EQUALS_NullTag); if (needsTypeCheck(nodeUse, SpecCell | SpecOther)) { - m_jit.move(valueTagGPR, scratchGPR); - m_jit.or32(TrustedImm32(1), scratchGPR); + m_jit.or32(TrustedImm32(1), valueTagGPR, scratchGPR); typeCheck( JSValueRegs(valueTagGPR, valuePayloadGPR), nodeUse, SpecCell | SpecOther, m_jit.branch32(MacroAssembler::NotEqual, scratchGPR, TrustedImm32(JSValue::NullTag))); @@ -1618,11 +1862,12 @@ void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BasicBlock* taken, Ba void SpeculativeJIT::emitBranch(Node* node) { - BasicBlock* taken = node->takenBlock(); - BasicBlock* notTaken = node->notTakenBlock(); + BasicBlock* taken = node->branchData()->taken.block; + BasicBlock* notTaken = node->branchData()->notTaken.block; switch (node->child1().useKind()) { - case BooleanUse: { + case BooleanUse: + case KnownBooleanUse: { SpeculateBooleanOperand value(this, node->child1()); MacroAssembler::ResultCondition condition = MacroAssembler::NonZero; @@ -1644,8 +1889,18 @@ void SpeculativeJIT::emitBranch(Node* node) emitObjectOrOtherBranch(node->child1(), taken, notTaken); return; } - - case NumberUse: + + case StringUse: { + emitStringBranch(node->child1(), taken, notTaken); + return; + } + + case StringOrOtherUse: { + emitStringOrOtherBranch(node->child1(), taken, notTaken); + return; + } + + case DoubleRepUse: case Int32Use: { if (node->child1().useKind() == Int32Use) { bool invert = false; @@ -1673,30 +1928,22 @@ void SpeculativeJIT::emitBranch(Node* node) case UntypedUse: { JSValueOperand value(this, node->child1()); - value.fill(); - GPRReg valueTagGPR = value.tagGPR(); - GPRReg valuePayloadGPR = value.payloadGPR(); - + FPRTemporary valueFPR(this); + FPRTemporary tempFPR(this); GPRTemporary result(this); + GPRTemporary temp(this); + + JSValueRegs valueRegs = value.jsValueRegs(); GPRReg resultGPR = result.gpr(); use(node->child1()); - - JITCompiler::Jump fastPath = m_jit.branch32(JITCompiler::Equal, valueTagGPR, JITCompiler::TrustedImm32(JSValue::Int32Tag)); - JITCompiler::Jump slowPath = m_jit.branch32(JITCompiler::NotEqual, valueTagGPR, JITCompiler::TrustedImm32(JSValue::BooleanTag)); - fastPath.link(&m_jit); - branchTest32(JITCompiler::Zero, valuePayloadGPR, notTaken); + bool shouldCheckMasqueradesAsUndefined = !masqueradesAsUndefinedWatchpointIsStillValid(); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + m_jit.emitConvertValueToBoolean(valueRegs, resultGPR, temp.gpr(), valueFPR.fpr(), tempFPR.fpr(), shouldCheckMasqueradesAsUndefined, globalObject); + branchTest32(JITCompiler::Zero, resultGPR, notTaken); jump(taken, ForceJump); - slowPath.link(&m_jit); - silentSpillAllRegisters(resultGPR); - callOperation(operationConvertJSValueToBoolean, resultGPR, valueTagGPR, valuePayloadGPR); - silentFillAllRegisters(resultGPR); - - branchTest32(JITCompiler::NonZero, resultGPR, taken); - jump(notTaken); - noResult(node, UseChildrenCalledExplicitly); return; } @@ -1786,39 +2033,55 @@ void SpeculativeJIT::compile(Node* node) switch (op) { case JSConstant: + case DoubleConstant: + case PhantomDirectArguments: + case PhantomClonedArguments: initConstantInfo(node); break; - case PhantomArguments: - initConstantInfo(node); - break; - - case WeakJSConstant: - m_jit.addWeakReference(node->weakConstant()); - initConstantInfo(node); + case LazyJSConstant: + compileLazyJSConstant(node); break; case Identity: { - RELEASE_ASSERT_NOT_REACHED(); + speculate(node, node->child1()); + switch (node->child1().useKind()) { + case DoubleRepUse: + case DoubleRepRealUse: { + SpeculateDoubleOperand op(this, node->child1()); + doubleResult(op.fpr(), node); + break; + } + case Int52RepUse: + case AnyIntUse: + case DoubleRepAnyIntUse: { + RELEASE_ASSERT_NOT_REACHED(); + break; + } + default: { + JSValueOperand op(this, node->child1()); + GPRTemporary resultTag(this, Reuse, op, TagWord); + GPRTemporary resultPayload(this, Reuse, op, PayloadWord); + GPRReg sourceTag = op.tagGPR(); + GPRReg sourcePayload = op.payloadGPR(); + GPRReg resultTagGPR = resultTag.gpr(); + GPRReg resultPayloadGPR = resultPayload.gpr(); + m_jit.move(sourceTag, resultTagGPR); + m_jit.move(sourcePayload, resultPayloadGPR); + jsValueResult(resultTagGPR, resultPayloadGPR, node); + break; + } + } // switch break; } case GetLocal: { - SpeculatedType prediction = node->variableAccessData()->prediction(); AbstractValue& value = m_state.variables().operand(node->local()); - // If we have no prediction for this local, then don't attempt to compile. - if (prediction == SpecNone) { - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); - break; - } - // If the CFA is tracking this variable and it found that the variable // cannot have been assigned, then don't attempt to proceed. if (value.isClear()) { - // FIXME: We should trap instead. - // https://bugs.webkit.org/show_bug.cgi?id=110383 - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); + m_compileOkay = false; break; } @@ -1868,8 +2131,7 @@ void SpeculativeJIT::compile(Node* node) break; } - case FlushedJSValue: - case FlushedArguments: { + case FlushedJSValue: { GPRTemporary result(this); GPRTemporary tag(this); m_jit.load32(JITCompiler::payloadFor(node->machineLocal()), result.gpr()); @@ -1900,13 +2162,23 @@ void SpeculativeJIT::compile(Node* node) break; } - case MovHint: - case ZombieHint: - case Check: { - RELEASE_ASSERT_NOT_REACHED(); + case MovHint: { + compileMovHint(m_currentNode); + noResult(node); + break; + } + + case ZombieHint: { + recordSetLocal(m_currentNode->unlinkedLocal(), VirtualRegister(), DataFormatDead); + noResult(node); break; } + case ExitOK: { + noResult(node); + break; + } + case SetLocal: { switch (node->variableAccessData()->flushFormat()) { case FlushedDouble: { @@ -1945,16 +2217,7 @@ void SpeculativeJIT::compile(Node* node) break; } - case FlushedJSValue: - case FlushedArguments: { - if (generationInfoFromVirtualRegister(node->child1()->virtualRegister()).registerFormat() == DataFormatDouble) { - SpeculateDoubleOperand value(this, node->child1(), ManualOperandSpeculation); - m_jit.storeDouble(value.fpr(), JITCompiler::addressFor(node->machineLocal())); - noResult(node); - recordSetLocal(DataFormatDouble); - break; - } - + case FlushedJSValue: { JSValueOperand value(this, node->child1()); m_jit.store32(value.payloadGPR(), JITCompiler::payloadFor(node->machineLocal())); m_jit.store32(value.tagGPR(), JITCompiler::tagFor(node->machineLocal())); @@ -1975,60 +2238,19 @@ void SpeculativeJIT::compile(Node* node) // But it may be profitable to use this as a hook to run speculation checks // on arguments, thereby allowing us to trivially eliminate such checks if // the argument is not used. + recordSetLocal(dataFormatFor(node->variableAccessData()->flushFormat())); break; case BitAnd: case BitOr: case BitXor: - if (isInt32Constant(node->child1().node())) { - SpeculateInt32Operand op2(this, node->child2()); - GPRTemporary result(this, Reuse, op2); - - bitOp(op, valueOfInt32Constant(node->child1().node()), op2.gpr(), result.gpr()); - - int32Result(result.gpr(), node); - } else if (isInt32Constant(node->child2().node())) { - SpeculateInt32Operand op1(this, node->child1()); - GPRTemporary result(this, Reuse, op1); - - bitOp(op, valueOfInt32Constant(node->child2().node()), op1.gpr(), result.gpr()); - - int32Result(result.gpr(), node); - } else { - SpeculateInt32Operand op1(this, node->child1()); - SpeculateInt32Operand op2(this, node->child2()); - GPRTemporary result(this, Reuse, op1, op2); - - GPRReg reg1 = op1.gpr(); - GPRReg reg2 = op2.gpr(); - bitOp(op, reg1, reg2, result.gpr()); - - int32Result(result.gpr(), node); - } + compileBitwiseOp(node); break; case BitRShift: case BitLShift: case BitURShift: - if (isInt32Constant(node->child2().node())) { - SpeculateInt32Operand op1(this, node->child1()); - GPRTemporary result(this, Reuse, op1); - - shiftOp(op, op1.gpr(), valueOfInt32Constant(node->child2().node()) & 0x1f, result.gpr()); - - int32Result(result.gpr(), node); - } else { - // Do not allow shift amount to be used as the result, MacroAssembler does not permit this. - SpeculateInt32Operand op1(this, node->child1()); - SpeculateInt32Operand op2(this, node->child2()); - GPRTemporary result(this, Reuse, op1); - - GPRReg reg1 = op1.gpr(); - GPRReg reg2 = op2.gpr(); - shiftOp(op, reg1, reg2, result.gpr()); - - int32Result(result.gpr(), node); - } + compileShiftOp(node); break; case UInt32ToNumber: { @@ -2046,35 +2268,51 @@ void SpeculativeJIT::compile(Node* node) break; } - case Int32ToDouble: { - compileInt32ToDouble(node); + case DoubleRep: { + compileDoubleRep(node); break; } - case ValueAdd: { - JSValueOperand op1(this, node->child1()); - JSValueOperand op2(this, node->child2()); + case ValueRep: { + compileValueRep(node); + break; + } + + case ValueAdd: + compileValueAdd(node); + break; + + case StrCat: { + JSValueOperand op1(this, node->child1(), ManualOperandSpeculation); + JSValueOperand op2(this, node->child2(), ManualOperandSpeculation); + JSValueOperand op3(this, node->child3(), ManualOperandSpeculation); - GPRReg op1TagGPR = op1.tagGPR(); - GPRReg op1PayloadGPR = op1.payloadGPR(); - GPRReg op2TagGPR = op2.tagGPR(); - GPRReg op2PayloadGPR = op2.payloadGPR(); + JSValueRegs op1Regs = op1.jsValueRegs(); + JSValueRegs op2Regs = op2.jsValueRegs(); + JSValueRegs op3Regs; + + if (node->child3()) + op3Regs = op3.jsValueRegs(); flushRegisters(); - - GPRResult2 resultTag(this); - GPRResult resultPayload(this); - if (isKnownNotNumber(node->child1().node()) || isKnownNotNumber(node->child2().node())) - callOperation(operationValueAddNotNumber, resultTag.gpr(), resultPayload.gpr(), op1TagGPR, op1PayloadGPR, op2TagGPR, op2PayloadGPR); + + GPRFlushedCallResult result(this); + if (node->child3()) + callOperation(operationStrCat3, result.gpr(), op1Regs, op2Regs, op3Regs); else - callOperation(operationValueAdd, resultTag.gpr(), resultPayload.gpr(), op1TagGPR, op1PayloadGPR, op2TagGPR, op2PayloadGPR); + callOperation(operationStrCat2, result.gpr(), op1Regs, op2Regs); + m_jit.exceptionCheck(); - jsValueResult(resultTag.gpr(), resultPayload.gpr(), node); + cellResult(result.gpr(), node); break; } case ArithAdd: - compileAdd(node); + compileArithAdd(node); + break; + + case ArithClz32: + compileArithClz32(node); break; case MakeRope: @@ -2103,39 +2341,15 @@ void SpeculativeJIT::compile(Node* node) break; } - case ArithAbs: { - switch (node->child1().useKind()) { - case Int32Use: { - SpeculateStrictInt32Operand op1(this, node->child1()); - GPRTemporary result(this, Reuse, op1); - GPRTemporary scratch(this); - - m_jit.move(op1.gpr(), result.gpr()); - m_jit.rshift32(result.gpr(), MacroAssembler::TrustedImm32(31), scratch.gpr()); - m_jit.add32(scratch.gpr(), result.gpr()); - m_jit.xor32(scratch.gpr(), result.gpr()); - speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, result.gpr(), MacroAssembler::TrustedImm32(1 << 31))); - int32Result(result.gpr(), node); - break; - } - - - case NumberUse: { - SpeculateDoubleOperand op1(this, node->child1()); - FPRTemporary result(this); - - m_jit.absDouble(op1.fpr(), result.fpr()); - doubleResult(result.fpr(), node); - break; - } - - default: - RELEASE_ASSERT_NOT_REACHED(); - break; - } + case ArithPow: { + compileArithPow(node); break; } - + + case ArithAbs: + compileArithAbs(node); + break; + case ArithMin: case ArithMax: { switch (node->binaryUseKind()) { @@ -2162,7 +2376,7 @@ void SpeculativeJIT::compile(Node* node) break; } - case NumberUse: { + case DoubleRepUse: { SpeculateDoubleOperand op1(this, node->child1()); SpeculateDoubleOperand op2(this, node->child2()); FPRTemporary result(this, op1); @@ -2206,40 +2420,41 @@ void SpeculativeJIT::compile(Node* node) } break; } - - case ArithSqrt: { - SpeculateDoubleOperand op1(this, node->child1()); - FPRTemporary result(this, op1); - - m_jit.sqrtDouble(op1.fpr(), result.fpr()); - - doubleResult(result.fpr(), node); + + case ArithSqrt: + compileArithSqrt(node); break; - } - case ArithSin: { - SpeculateDoubleOperand op1(this, node->child1()); - FPRReg op1FPR = op1.fpr(); + case ArithFRound: + compileArithFRound(node); + break; - flushRegisters(); - - FPRResult result(this); - callOperation(sin, result.fpr(), op1FPR); - doubleResult(result.fpr(), node); + case ArithRandom: + compileArithRandom(node); break; - } - case ArithCos: { - SpeculateDoubleOperand op1(this, node->child1()); - FPRReg op1FPR = op1.fpr(); + case ArithRound: + case ArithFloor: + case ArithCeil: + case ArithTrunc: + compileArithRounding(node); + break; - flushRegisters(); - - FPRResult result(this); - callOperation(cos, result.fpr(), op1FPR); - doubleResult(result.fpr(), node); + case ArithSin: + compileArithSin(node); + break; + + case ArithCos: + compileArithCos(node); + break; + + case ArithTan: + compileArithTan(node); + break; + + case ArithLog: + compileArithLog(node); break; - } case LogicalNot: compileLogicalNot(node); @@ -2264,27 +2479,20 @@ void SpeculativeJIT::compile(Node* node) if (compare(node, JITCompiler::GreaterThanOrEqual, JITCompiler::DoubleGreaterThanOrEqual, operationCompareGreaterEq)) return; break; - - case CompareEqConstant: - ASSERT(isNullConstant(node->child2().node())); - if (nonSpeculativeCompareNull(node, node->child1())) - return; - break; case CompareEq: if (compare(node, JITCompiler::Equal, JITCompiler::DoubleEqual, operationCompareEq)) return; break; - case CompareStrictEqConstant: - if (compileStrictEqForConstant(node, node->child1(), valueOfJSConstant(node->child2().node()))) - return; - break; - case CompareStrictEq: if (compileStrictEq(node)) return; break; + + case CompareEqPtr: + compileCompareEqPtr(node); + break; case StringCharCodeAt: { compileGetCharCodeAt(node); @@ -2318,19 +2526,41 @@ void SpeculativeJIT::compile(Node* node) case Array::SelectUsingPredictions: case Array::ForceExit: RELEASE_ASSERT_NOT_REACHED(); +#if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE) terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); +#endif + break; + case Array::Undecided: { + SpeculateStrictInt32Operand index(this, node->child2()); + GPRTemporary resultTag(this, Reuse, index); + GPRTemporary resultPayload(this); + + GPRReg indexGPR = index.gpr(); + GPRReg resultTagGPR = resultTag.gpr(); + GPRReg resultPayloadGPR = resultPayload.gpr(); + + speculationCheck(OutOfBounds, JSValueRegs(), node, + m_jit.branch32(MacroAssembler::LessThan, indexGPR, MacroAssembler::TrustedImm32(0))); + + use(node->child1()); + index.use(); + + m_jit.move(MacroAssembler::TrustedImm32(JSValue::UndefinedTag), resultTagGPR); + m_jit.move(MacroAssembler::TrustedImm32(0), resultPayloadGPR); + jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly); break; + } case Array::Generic: { SpeculateCellOperand base(this, node->child1()); // Save a register, speculate cell. We'll probably be right. JSValueOperand property(this, node->child2()); GPRReg baseGPR = base.gpr(); - GPRReg propertyTagGPR = property.tagGPR(); - GPRReg propertyPayloadGPR = property.payloadGPR(); + JSValueRegs propertyRegs = property.jsValueRegs(); flushRegisters(); - GPRResult2 resultTag(this); - GPRResult resultPayload(this); - callOperation(operationGetByValCell, resultTag.gpr(), resultPayload.gpr(), baseGPR, propertyTagGPR, propertyPayloadGPR); + GPRFlushedCallResult2 resultTag(this); + GPRFlushedCallResult resultPayload(this); + callOperation(operationGetByValCell, JSValueRegs(resultTag.gpr(), resultPayload.gpr()), baseGPR, propertyRegs); + m_jit.exceptionCheck(); jsValueResult(resultTag.gpr(), resultPayload.gpr(), node); break; @@ -2351,21 +2581,46 @@ void SpeculativeJIT::compile(Node* node) GPRTemporary resultPayload(this); if (node->arrayMode().type() == Array::Int32) { + ASSERT(!node->arrayMode().isSaneChain()); + speculationCheck( OutOfBounds, JSValueRegs(), 0, m_jit.branch32( MacroAssembler::Equal, - MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)), + MacroAssembler::BaseIndex( + storageReg, propertyReg, MacroAssembler::TimesEight, TagOffset), TrustedImm32(JSValue::EmptyValueTag))); - m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayload.gpr()); + m_jit.load32( + MacroAssembler::BaseIndex( + storageReg, propertyReg, MacroAssembler::TimesEight, PayloadOffset), + resultPayload.gpr()); int32Result(resultPayload.gpr(), node); break; } GPRTemporary resultTag(this); - m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTag.gpr()); - speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, resultTag.gpr(), TrustedImm32(JSValue::EmptyValueTag))); - m_jit.load32(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayload.gpr()); + m_jit.load32( + MacroAssembler::BaseIndex( + storageReg, propertyReg, MacroAssembler::TimesEight, TagOffset), + resultTag.gpr()); + m_jit.load32( + MacroAssembler::BaseIndex( + storageReg, propertyReg, MacroAssembler::TimesEight, PayloadOffset), + resultPayload.gpr()); + if (node->arrayMode().isSaneChain()) { + JITCompiler::Jump notHole = m_jit.branch32( + MacroAssembler::NotEqual, resultTag.gpr(), + TrustedImm32(JSValue::EmptyValueTag)); + m_jit.move(TrustedImm32(JSValue::UndefinedTag), resultTag.gpr()); + m_jit.move(TrustedImm32(0), resultPayload.gpr()); + notHole.link(&m_jit); + } else { + speculationCheck( + LoadFromHole, JSValueRegs(), 0, + m_jit.branch32( + MacroAssembler::Equal, resultTag.gpr(), + TrustedImm32(JSValue::EmptyValueTag))); + } jsValueResult(resultTag.gpr(), resultPayload.gpr(), node); break; } @@ -2520,8 +2775,11 @@ void SpeculativeJIT::compile(Node* node) case Array::String: compileGetByValOnString(node); break; - case Array::Arguments: - compileGetByValOnArguments(node); + case Array::DirectArguments: + compileGetByValOnDirectArguments(node); + break; + case Array::ScopedArguments: + compileGetByValOnScopedArguments(node); break; default: { TypedArrayType type = node->arrayMode().typedArrayType(); @@ -2532,6 +2790,37 @@ void SpeculativeJIT::compile(Node* node) } } break; } + + case ToLowerCase: { + compileToLowerCase(node); + break; + } + + case NumberToStringWithRadix: { + compileNumberToStringWithRadix(node); + break; + } + + case GetByValWithThis: { + JSValueOperand base(this, node->child1()); + JSValueRegs baseRegs = base.jsValueRegs(); + JSValueOperand thisValue(this, node->child2()); + JSValueRegs thisValueRegs = thisValue.jsValueRegs(); + JSValueOperand subscript(this, node->child3()); + JSValueRegs subscriptRegs = subscript.jsValueRegs(); + + GPRFlushedCallResult resultPayload(this); + GPRFlushedCallResult2 resultTag(this); + GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRReg resultTagGPR = resultTag.gpr(); + + flushRegisters(); + callOperation(operationGetByValWithThis, JSValueRegs(resultTagGPR, resultPayloadGPR), baseRegs, thisValueRegs, subscriptRegs); + m_jit.exceptionCheck(); + + jsValueResult(resultTagGPR, resultPayloadGPR, node); + break; + } case PutByValDirect: case PutByVal: @@ -2548,8 +2837,10 @@ void SpeculativeJIT::compile(Node* node) case Array::SelectUsingPredictions: case Array::ForceExit: RELEASE_ASSERT_NOT_REACHED(); +#if COMPILER_QUIRK(CONSIDERS_UNREACHABLE_CODE) terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); alreadyHandled = true; +#endif break; case Array::Generic: { ASSERT(node->op() == PutByVal || node->op() == PutByValDirect); @@ -2558,16 +2849,15 @@ void SpeculativeJIT::compile(Node* node) JSValueOperand property(this, child2); JSValueOperand value(this, child3); GPRReg baseGPR = base.gpr(); - GPRReg propertyTagGPR = property.tagGPR(); - GPRReg propertyPayloadGPR = property.payloadGPR(); - GPRReg valueTagGPR = value.tagGPR(); - GPRReg valuePayloadGPR = value.payloadGPR(); + JSValueRegs propertyRegs = property.jsValueRegs(); + JSValueRegs valueRegs = value.jsValueRegs(); flushRegisters(); if (node->op() == PutByValDirect) - callOperation(m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict, baseGPR, propertyTagGPR, propertyPayloadGPR, valueTagGPR, valuePayloadGPR); + callOperation(m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectCellStrict : operationPutByValDirectCellNonStrict, baseGPR, propertyRegs, valueRegs); else - callOperation(m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict, baseGPR, propertyTagGPR, propertyPayloadGPR, valueTagGPR, valuePayloadGPR); + callOperation(m_jit.codeBlock()->isStrictMode() ? operationPutByValCellStrict : operationPutByValCellNonStrict, baseGPR, propertyRegs, valueRegs); + m_jit.exceptionCheck(); noResult(node); alreadyHandled = true; @@ -2685,12 +2975,12 @@ void SpeculativeJIT::compile(Node* node) addSlowPathGenerator(slowPathCall( slowCases, this, m_jit.codeBlock()->isStrictMode() ? operationPutByValDirectBeyondArrayBoundsStrict : operationPutByValDirectBeyondArrayBoundsNonStrict, - NoResult, baseReg, propertyReg, valueTagReg, valuePayloadReg)); + NoResult, baseReg, propertyReg, JSValueRegs(valueTagReg, valuePayloadReg))); } else { addSlowPathGenerator(slowPathCall( slowCases, this, m_jit.codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict, - NoResult, baseReg, propertyReg, valueTagReg, valuePayloadReg)); + NoResult, baseReg, propertyReg, JSValueRegs(valueTagReg, valuePayloadReg))); } } @@ -2698,12 +2988,6 @@ void SpeculativeJIT::compile(Node* node) break; } - case Array::Arguments: - // FIXME: we could at some point make this work. Right now we're assuming that the register - // pressure would be too great. - RELEASE_ASSERT_NOT_REACHED(); - break; - default: { TypedArrayType type = arrayMode.typedArrayType(); if (isInt(type)) @@ -2714,54 +2998,309 @@ void SpeculativeJIT::compile(Node* node) break; } - case RegExpExec: { - if (compileRegExpExec(node)) - return; + case PutByValWithThis: { +#if CPU(X86) + // We don't have enough registers on X86 to do this + // without setting up the call frame incrementally. + unsigned index = 0; + m_jit.poke(GPRInfo::callFrameRegister, index++); + + { + JSValueOperand base(this, m_jit.graph().varArgChild(node, 0)); + GPRReg baseTag = base.tagGPR(); + GPRReg basePayload = base.payloadGPR(); + + JSValueOperand thisValue(this, m_jit.graph().varArgChild(node, 1)); + GPRReg thisValueTag = thisValue.tagGPR(); + GPRReg thisValuePayload = thisValue.payloadGPR(); + + JSValueOperand property(this, m_jit.graph().varArgChild(node, 2)); + GPRReg propertyTag = property.tagGPR(); + GPRReg propertyPayload = property.payloadGPR(); + + m_jit.poke(basePayload, index++); + m_jit.poke(baseTag, index++); + + m_jit.poke(thisValuePayload, index++); + m_jit.poke(thisValueTag, index++); + + m_jit.poke(propertyPayload, index++); + m_jit.poke(propertyTag, index++); - if (!node->adjustedRefCount()) { - SpeculateCellOperand base(this, node->child1()); - SpeculateCellOperand argument(this, node->child2()); - GPRReg baseGPR = base.gpr(); - GPRReg argumentGPR = argument.gpr(); - flushRegisters(); - GPRResult result(this); - callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR); + } + + JSValueOperand value(this, m_jit.graph().varArgChild(node, 3)); + GPRReg valueTag = value.tagGPR(); + GPRReg valuePayload = value.payloadGPR(); + m_jit.poke(valuePayload, index++); + m_jit.poke(valueTag, index++); + + flushRegisters(); + appendCall(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValWithThisStrict : operationPutByValWithThis); + m_jit.exceptionCheck(); +#elif CPU(MIPS) + // We don't have enough registers on MIPS either but the ABI is a little different. + unsigned index = 4; + m_jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0); + { + JSValueOperand base(this, m_jit.graph().varArgChild(node, 0)); + GPRReg baseTag = base.tagGPR(); + GPRReg basePayload = base.payloadGPR(); + + JSValueOperand thisValue(this, m_jit.graph().varArgChild(node, 1)); + GPRReg thisValueTag = thisValue.tagGPR(); + GPRReg thisValuePayload = thisValue.payloadGPR(); + + JSValueOperand property(this, m_jit.graph().varArgChild(node, 2)); + GPRReg propertyTag = property.tagGPR(); + GPRReg propertyPayload = property.payloadGPR(); + + // for operationPutByValWithThis[Strict](), base is a 64 bits + // argument, so it should be double word aligned on the stack. + // This requirement still applies when it's in argument registers + // instead of on the stack. + m_jit.move(basePayload, GPRInfo::argumentGPR2); + m_jit.move(baseTag, GPRInfo::argumentGPR3); + + m_jit.poke(thisValuePayload, index++); + m_jit.poke(thisValueTag, index++); + + m_jit.poke(propertyPayload, index++); + m_jit.poke(propertyTag, index++); + + flushRegisters(); + } + + JSValueOperand value(this, m_jit.graph().varArgChild(node, 3)); + GPRReg valueTag = value.tagGPR(); + GPRReg valuePayload = value.payloadGPR(); + m_jit.poke(valuePayload, index++); + m_jit.poke(valueTag, index++); + + flushRegisters(); + appendCall(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValWithThisStrict : operationPutByValWithThis); + m_jit.exceptionCheck(); +#else + static_assert(GPRInfo::numberOfRegisters >= 8, "We are assuming we have enough registers to make this call without incrementally setting up the arguments."); + + JSValueOperand base(this, m_jit.graph().varArgChild(node, 0)); + JSValueRegs baseRegs = base.jsValueRegs(); + + JSValueOperand thisValue(this, m_jit.graph().varArgChild(node, 1)); + JSValueRegs thisRegs = thisValue.jsValueRegs(); + + JSValueOperand property(this, m_jit.graph().varArgChild(node, 2)); + JSValueRegs propertyRegs = property.jsValueRegs(); + + JSValueOperand value(this, m_jit.graph().varArgChild(node, 3)); + JSValueRegs valueRegs = value.jsValueRegs(); + + flushRegisters(); + callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValWithThisStrict : operationPutByValWithThis, + NoResult, baseRegs, thisRegs, propertyRegs, valueRegs); + m_jit.exceptionCheck(); +#endif // CPU(X86) + + noResult(node); + break; + } + + case RegExpExec: { + SpeculateCellOperand globalObject(this, node->child1()); + GPRReg globalObjectGPR = globalObject.gpr(); + + if (node->child2().useKind() == RegExpObjectUse) { + if (node->child3().useKind() == StringUse) { + SpeculateCellOperand base(this, node->child2()); + SpeculateCellOperand argument(this, node->child3()); + GPRReg baseGPR = base.gpr(); + GPRReg argumentGPR = argument.gpr(); + speculateRegExpObject(node->child2(), baseGPR); + speculateString(node->child3(), argumentGPR); + + flushRegisters(); + GPRFlushedCallResult2 resultTag(this); + GPRFlushedCallResult resultPayload(this); + callOperation( + operationRegExpExecString, JSValueRegs(resultTag.gpr(), resultPayload.gpr()), + globalObjectGPR, baseGPR, argumentGPR); + m_jit.exceptionCheck(); + + jsValueResult(resultTag.gpr(), resultPayload.gpr(), node); + break; + } - // Must use jsValueResult because otherwise we screw up register - // allocation, which thinks that this node has a result. - booleanResult(result.gpr(), node); + SpeculateCellOperand base(this, node->child2()); + JSValueOperand argument(this, node->child3()); + GPRReg baseGPR = base.gpr(); + JSValueRegs argumentRegs = argument.jsValueRegs(); + speculateRegExpObject(node->child2(), baseGPR); + + flushRegisters(); + GPRFlushedCallResult2 resultTag(this); + GPRFlushedCallResult resultPayload(this); + callOperation( + operationRegExpExec, JSValueRegs(resultTag.gpr(), resultPayload.gpr()), globalObjectGPR, baseGPR, argumentRegs); + m_jit.exceptionCheck(); + + jsValueResult(resultTag.gpr(), resultPayload.gpr(), node); break; } - - SpeculateCellOperand base(this, node->child1()); - SpeculateCellOperand argument(this, node->child2()); - GPRReg baseGPR = base.gpr(); - GPRReg argumentGPR = argument.gpr(); + + JSValueOperand base(this, node->child2()); + JSValueOperand argument(this, node->child3()); + JSValueRegs baseRegs = base.jsValueRegs(); + JSValueRegs argumentRegs = argument.jsValueRegs(); flushRegisters(); - GPRResult2 resultTag(this); - GPRResult resultPayload(this); - callOperation(operationRegExpExec, resultTag.gpr(), resultPayload.gpr(), baseGPR, argumentGPR); + GPRFlushedCallResult2 resultTag(this); + GPRFlushedCallResult resultPayload(this); + callOperation( + operationRegExpExecGeneric, JSValueRegs(resultTag.gpr(), resultPayload.gpr()), globalObjectGPR, baseRegs, argumentRegs); + m_jit.exceptionCheck(); jsValueResult(resultTag.gpr(), resultPayload.gpr(), node); break; } case RegExpTest: { - SpeculateCellOperand base(this, node->child1()); - SpeculateCellOperand argument(this, node->child2()); - GPRReg baseGPR = base.gpr(); - GPRReg argumentGPR = argument.gpr(); + SpeculateCellOperand globalObject(this, node->child1()); + GPRReg globalObjectGPR = globalObject.gpr(); + + if (node->child2().useKind() == RegExpObjectUse) { + if (node->child3().useKind() == StringUse) { + SpeculateCellOperand base(this, node->child2()); + SpeculateCellOperand argument(this, node->child3()); + GPRReg baseGPR = base.gpr(); + GPRReg argumentGPR = argument.gpr(); + speculateRegExpObject(node->child2(), baseGPR); + speculateString(node->child3(), argumentGPR); + + flushRegisters(); + GPRFlushedCallResult result(this); + callOperation( + operationRegExpTestString, result.gpr(), globalObjectGPR, baseGPR, argumentGPR); + m_jit.exceptionCheck(); + + booleanResult(result.gpr(), node); + break; + } + + SpeculateCellOperand base(this, node->child2()); + JSValueOperand argument(this, node->child3()); + GPRReg baseGPR = base.gpr(); + JSValueRegs argumentRegs = argument.jsValueRegs(); + speculateRegExpObject(node->child2(), baseGPR); + flushRegisters(); + GPRFlushedCallResult result(this); + callOperation( + operationRegExpTest, result.gpr(), globalObjectGPR, baseGPR, argumentRegs); + m_jit.exceptionCheck(); + + booleanResult(result.gpr(), node); + break; + } + + JSValueOperand base(this, node->child2()); + JSValueOperand argument(this, node->child3()); + JSValueRegs baseRegs = base.jsValueRegs(); + JSValueRegs argumentRegs = argument.jsValueRegs(); + flushRegisters(); - GPRResult result(this); - callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR); + GPRFlushedCallResult result(this); + callOperation( + operationRegExpTestGeneric, result.gpr(), globalObjectGPR, baseRegs, argumentRegs); + m_jit.exceptionCheck(); - // If we add a DataFormatBool, we should use it here. booleanResult(result.gpr(), node); break; } + + case StringReplace: + case StringReplaceRegExp: { + if (node->child1().useKind() == StringUse + && node->child2().useKind() == RegExpObjectUse + && node->child3().useKind() == StringUse) { + if (JSString* replace = node->child3()->dynamicCastConstant<JSString*>(*m_jit.vm())) { + if (!replace->length()) { + SpeculateCellOperand string(this, node->child1()); + SpeculateCellOperand regExp(this, node->child2()); + GPRReg stringGPR = string.gpr(); + GPRReg regExpGPR = regExp.gpr(); + speculateString(node->child1(), stringGPR); + speculateRegExpObject(node->child2(), regExpGPR); + + flushRegisters(); + GPRFlushedCallResult2 resultTag(this); + GPRFlushedCallResult resultPayload(this); + callOperation( + operationStringProtoFuncReplaceRegExpEmptyStr, JSValueRegs(resultTag.gpr(), resultPayload.gpr()), stringGPR, regExpGPR); + m_jit.exceptionCheck(); + cellResult(resultPayload.gpr(), node); + break; + } + } + + SpeculateCellOperand string(this, node->child1()); + SpeculateCellOperand regExp(this, node->child2()); + SpeculateCellOperand replace(this, node->child3()); + GPRReg stringGPR = string.gpr(); + GPRReg regExpGPR = regExp.gpr(); + GPRReg replaceGPR = replace.gpr(); + speculateString(node->child1(), stringGPR); + speculateRegExpObject(node->child2(), regExpGPR); + speculateString(node->child3(), replaceGPR); + + flushRegisters(); + GPRFlushedCallResult2 resultTag(this); + GPRFlushedCallResult resultPayload(this); + callOperation( + operationStringProtoFuncReplaceRegExpString, JSValueRegs(resultTag.gpr(), resultPayload.gpr()), + stringGPR, regExpGPR, replaceGPR); + m_jit.exceptionCheck(); + cellResult(resultPayload.gpr(), node); + break; + } + + // If we fixed up the edge of child2, we inserted a Check(@child2, String). + OperandSpeculationMode child2SpeculationMode = AutomaticOperandSpeculation; + if (node->child2().useKind() == StringUse) + child2SpeculationMode = ManualOperandSpeculation; + + JSValueOperand string(this, node->child1()); + JSValueOperand search(this, node->child2(), child2SpeculationMode); + JSValueOperand replace(this, node->child3()); + JSValueRegs stringRegs = string.jsValueRegs(); + JSValueRegs searchRegs = search.jsValueRegs(); + JSValueRegs replaceRegs = replace.jsValueRegs(); + + flushRegisters(); + GPRFlushedCallResult2 resultTag(this); + GPRFlushedCallResult resultPayload(this); + callOperation( + operationStringProtoFuncReplaceGeneric, JSValueRegs(resultTag.gpr(), resultPayload.gpr()), + stringRegs, searchRegs, replaceRegs); + m_jit.exceptionCheck(); + cellResult(resultPayload.gpr(), node); + break; + } + + case GetRegExpObjectLastIndex: { + compileGetRegExpObjectLastIndex(node); + break; + } + + case SetRegExpObjectLastIndex: { + compileSetRegExpObjectLastIndex(node); + break; + } + + case RecordRegExpCachedResult: { + compileRecordRegExpCachedResult(node); + break; + } case ArrayPush: { ASSERT(node->arrayMode().isJSArray()); @@ -2815,7 +3354,7 @@ void SpeculativeJIT::compile(Node* node) slowPathCall( slowPath, this, operationArrayPush, JSValueRegs(storageGPR, storageLengthGPR), - valueTagGPR, valuePayloadGPR, baseGPR)); + JSValueRegs(valueTagGPR, valuePayloadGPR), baseGPR)); jsValueResult(storageGPR, storageLengthGPR, node); break; @@ -2826,7 +3365,7 @@ void SpeculativeJIT::compile(Node* node) FPRReg valueFPR = value.fpr(); DFG_TYPE_CHECK( - JSValueRegs(), node->child2(), SpecFullRealNumber, + JSValueRegs(), node->child2(), SpecDoubleReal, m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, valueFPR, valueFPR)); m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR); @@ -2866,7 +3405,8 @@ void SpeculativeJIT::compile(Node* node) m_jit.add32(TrustedImm32(1), MacroAssembler::Address(storageGPR, OBJECT_OFFSETOF(ArrayStorage, m_numValuesInVector))); m_jit.move(TrustedImm32(JSValue::Int32Tag), storageGPR); - addSlowPathGenerator(slowPathCall(slowPath, this, operationArrayPush, JSValueRegs(storageGPR, storageLengthGPR), valueTagGPR, valuePayloadGPR, baseGPR)); + addSlowPathGenerator(slowPathCall(slowPath, this, operationArrayPush, JSValueRegs(storageGPR, storageLengthGPR), + JSValueRegs(valueTagGPR, valuePayloadGPR), baseGPR)); jsValueResult(storageGPR, storageLengthGPR, node); break; @@ -2942,7 +3482,7 @@ void SpeculativeJIT::compile(Node* node) MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight), tempFPR); MacroAssembler::Jump slowCase = m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempFPR, tempFPR); - JSValue nan = JSValue(JSValue::EncodeAsDouble, QNaN); + JSValue nan = JSValue(JSValue::EncodeAsDouble, PNaN); m_jit.store32( MacroAssembler::TrustedImm32(nan.u.asBits.tag), MacroAssembler::BaseIndex(storageGPR, valuePayloadGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag))); @@ -3011,8 +3551,13 @@ void SpeculativeJIT::compile(Node* node) break; } + case ArraySlice: { + compileArraySlice(node); + break; + } + case DFG::Jump: { - jump(node->takenBlock()); + jump(node->targetBlock()); noResult(node); break; } @@ -3047,12 +3592,8 @@ void SpeculativeJIT::compile(Node* node) } } - // Grab the return address. - m_jit.emitGetReturnPCFromCallFrameHeaderPtr(GPRInfo::regT2); - // Restore our caller's "r". - m_jit.emitGetCallerFrameFromCallFrameHeaderPtr(GPRInfo::callFrameRegister); - // Return. - m_jit.restoreReturnAddressBeforeReturn(GPRInfo::regT2); + m_jit.emitRestoreCalleeSaves(); + m_jit.emitFunctionEpilogue(); m_jit.ret(); noResult(node); @@ -3060,79 +3601,141 @@ void SpeculativeJIT::compile(Node* node) } case Throw: - case ThrowReferenceError: { + case ThrowStaticError: { // We expect that throw statements are rare and are intended to exit the code block // anyway, so we just OSR back to the old JIT for now. terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); break; } + case BooleanToNumber: { + switch (node->child1().useKind()) { + case BooleanUse: { + SpeculateBooleanOperand value(this, node->child1()); + GPRTemporary result(this); // FIXME: We could reuse, but on speculation fail would need recovery to restore tag (akin to add). + + m_jit.move(value.gpr(), result.gpr()); + + int32Result(result.gpr(), node); + break; + } + + case UntypedUse: { + JSValueOperand value(this, node->child1()); + + if (!m_interpreter.needsTypeCheck(node->child1(), SpecBoolInt32 | SpecBoolean)) { + GPRTemporary result(this); + + GPRReg valueGPR = value.payloadGPR(); + GPRReg resultGPR = result.gpr(); + + m_jit.move(valueGPR, resultGPR); + int32Result(result.gpr(), node); + break; + } + + GPRTemporary resultTag(this); + GPRTemporary resultPayload(this); + + GPRReg valueTagGPR = value.tagGPR(); + GPRReg valuePayloadGPR = value.payloadGPR(); + GPRReg resultTagGPR = resultTag.gpr(); + GPRReg resultPayloadGPR = resultPayload.gpr(); + + m_jit.move(valuePayloadGPR, resultPayloadGPR); + JITCompiler::Jump isBoolean = m_jit.branch32( + JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::BooleanTag)); + m_jit.move(valueTagGPR, resultTagGPR); + JITCompiler::Jump done = m_jit.jump(); + isBoolean.link(&m_jit); + m_jit.move(TrustedImm32(JSValue::Int32Tag), resultTagGPR); + done.link(&m_jit); + + jsValueResult(resultTagGPR, resultPayloadGPR, node); + break; + } + + default: + RELEASE_ASSERT_NOT_REACHED(); + break; + } + break; + } + case ToPrimitive: { RELEASE_ASSERT(node->child1().useKind() == UntypedUse); - JSValueOperand op1(this, node->child1()); - GPRTemporary resultTag(this, Reuse, op1, TagWord); - GPRTemporary resultPayload(this, Reuse, op1, PayloadWord); + JSValueOperand argument(this, node->child1()); + GPRTemporary resultTag(this, Reuse, argument, TagWord); + GPRTemporary resultPayload(this, Reuse, argument, PayloadWord); - GPRReg op1TagGPR = op1.tagGPR(); - GPRReg op1PayloadGPR = op1.payloadGPR(); + GPRReg argumentTagGPR = argument.tagGPR(); + GPRReg argumentPayloadGPR = argument.payloadGPR(); GPRReg resultTagGPR = resultTag.gpr(); GPRReg resultPayloadGPR = resultPayload.gpr(); - op1.use(); + argument.use(); - if (!(m_state.forNode(node->child1()).m_type & ~(SpecFullNumber | SpecBoolean))) { - m_jit.move(op1TagGPR, resultTagGPR); - m_jit.move(op1PayloadGPR, resultPayloadGPR); - } else { - MacroAssembler::Jump alreadyPrimitive = m_jit.branch32(MacroAssembler::NotEqual, op1TagGPR, TrustedImm32(JSValue::CellTag)); - MacroAssembler::Jump notPrimitive = m_jit.branchPtr(MacroAssembler::NotEqual, MacroAssembler::Address(op1PayloadGPR, JSCell::structureOffset()), MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())); - - alreadyPrimitive.link(&m_jit); - m_jit.move(op1TagGPR, resultTagGPR); - m_jit.move(op1PayloadGPR, resultPayloadGPR); - - addSlowPathGenerator( - slowPathCall( - notPrimitive, this, operationToPrimitive, - JSValueRegs(resultTagGPR, resultPayloadGPR), op1TagGPR, op1PayloadGPR)); - } + MacroAssembler::Jump alreadyPrimitive = m_jit.branchIfNotCell(argument.jsValueRegs()); + MacroAssembler::Jump notPrimitive = m_jit.branchIfObject(argumentPayloadGPR); + + alreadyPrimitive.link(&m_jit); + m_jit.move(argumentTagGPR, resultTagGPR); + m_jit.move(argumentPayloadGPR, resultPayloadGPR); + + addSlowPathGenerator( + slowPathCall( + notPrimitive, this, operationToPrimitive, + JSValueRegs(resultTagGPR, resultPayloadGPR), JSValueRegs(argumentTagGPR, argumentPayloadGPR))); jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly); break; } - - case ToString: { - if (node->child1().useKind() == UntypedUse) { - JSValueOperand op1(this, node->child1()); - GPRReg op1PayloadGPR = op1.payloadGPR(); - GPRReg op1TagGPR = op1.tagGPR(); - - GPRResult result(this); - GPRReg resultGPR = result.gpr(); - + + case ToNumber: { + JSValueOperand argument(this, node->child1()); + GPRTemporary resultTag(this, Reuse, argument, TagWord); + GPRTemporary resultPayload(this, Reuse, argument, PayloadWord); + + GPRReg argumentPayloadGPR = argument.payloadGPR(); + GPRReg argumentTagGPR = argument.tagGPR(); + JSValueRegs argumentRegs = argument.jsValueRegs(); + JSValueRegs resultRegs(resultTag.gpr(), resultPayload.gpr()); + + argument.use(); + + // We have several attempts to remove ToNumber. But ToNumber still exists. + // It means that converting non-numbers to numbers by this ToNumber is not rare. + // Instead of the slow path generator, we emit callOperation here. + if (!(m_state.forNode(node->child1()).m_type & SpecBytecodeNumber)) { flushRegisters(); - - JITCompiler::Jump done; - if (node->child1()->prediction() & SpecString) { - JITCompiler::Jump slowPath1 = m_jit.branch32( - JITCompiler::NotEqual, op1TagGPR, TrustedImm32(JSValue::CellTag)); - JITCompiler::Jump slowPath2 = m_jit.branchPtr( - JITCompiler::NotEqual, - JITCompiler::Address(op1PayloadGPR, JSCell::structureOffset()), - TrustedImmPtr(m_jit.vm()->stringStructure.get())); - m_jit.move(op1PayloadGPR, resultGPR); - done = m_jit.jump(); - slowPath1.link(&m_jit); - slowPath2.link(&m_jit); + callOperation(operationToNumber, resultRegs, argumentRegs); + m_jit.exceptionCheck(); + } else { + MacroAssembler::Jump notNumber; + { + GPRTemporary scratch(this); + notNumber = m_jit.branchIfNotNumber(argument.jsValueRegs(), scratch.gpr()); } - callOperation(operationToString, resultGPR, op1TagGPR, op1PayloadGPR); - if (done.isSet()) - done.link(&m_jit); - cellResult(resultGPR, node); - break; + m_jit.move(argumentTagGPR, resultRegs.tagGPR()); + m_jit.move(argumentPayloadGPR, resultRegs.payloadGPR()); + MacroAssembler::Jump done = m_jit.jump(); + + notNumber.link(&m_jit); + silentSpillAllRegisters(resultRegs); + callOperation(operationToNumber, resultRegs, argumentRegs); + silentFillAllRegisters(resultRegs); + m_jit.exceptionCheck(); + + done.link(&m_jit); } + + jsValueResult(resultRegs.tagGPR(), resultRegs.payloadGPR(), node, UseChildrenCalledExplicitly); + break; + } - compileToStringOnCell(node); + case ToString: + case CallStringConstructor: { + compileToStringOrCallStringConstructor(node); break; } @@ -3142,9 +3745,9 @@ void SpeculativeJIT::compile(Node* node) } case NewArray: { - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); - if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) { - Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) { + RegisteredStructure structure = m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())); ASSERT(structure->indexingType() == node->indexingType()); ASSERT( hasUndecided(structure->indexingType()) @@ -3160,7 +3763,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg resultGPR = result.gpr(); GPRReg storageGPR = storage.gpr(); - emitAllocateJSArray(resultGPR, structure, storageGPR, numElements); + emitAllocateRawObject(resultGPR, structure, storageGPR, numElements, numElements); // At this point, one way or another, resultGPR and storageGPR have pointers to // the JSArray and the Butterfly, respectively. @@ -3178,7 +3781,7 @@ void SpeculativeJIT::compile(Node* node) SpeculateDoubleOperand operand(this, use); FPRReg opFPR = operand.fpr(); DFG_TYPE_CHECK( - JSValueRegs(), use, SpecFullRealNumber, + JSValueRegs(), use, SpecDoubleReal, m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR)); m_jit.storeDouble(opFPR, MacroAssembler::Address(storageGPR, sizeof(double) * operandIdx)); @@ -3217,9 +3820,10 @@ void SpeculativeJIT::compile(Node* node) if (!node->numChildren()) { flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation( - operationNewEmptyArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())); + operationNewEmptyArray, result.gpr(), m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()))); + m_jit.exceptionCheck(); cellResult(result.gpr(), node); break; } @@ -3246,7 +3850,7 @@ void SpeculativeJIT::compile(Node* node) JSValueRegs(), use, SpecFullRealNumber, m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR)); - m_jit.storeDouble(opFPR, reinterpret_cast<char*>(buffer + operandIdx)); + m_jit.storeDouble(opFPR, TrustedImmPtr(reinterpret_cast<char*>(buffer + operandIdx))); break; } case ALL_INT32_INDEXING_TYPES: { @@ -3292,11 +3896,12 @@ void SpeculativeJIT::compile(Node* node) m_jit.storePtr(TrustedImmPtr(scratchSize), scratch.gpr()); } - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation( - operationNewArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), + operationNewArray, result.gpr(), m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), static_cast<void*>(buffer), node->numChildren()); + m_jit.exceptionCheck(); if (scratchSize) { GPRTemporary scratch(this); @@ -3309,57 +3914,26 @@ void SpeculativeJIT::compile(Node* node) break; } + case NewArrayWithSpread: { + compileNewArrayWithSpread(node); + break; + } + + case Spread: { + compileSpread(node); + break; + } + case NewArrayWithSize: { - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); - if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) { + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) { SpeculateStrictInt32Operand size(this, node->child1()); GPRTemporary result(this); - GPRTemporary storage(this); - GPRTemporary scratch(this); - GPRTemporary scratch2(this); - + GPRReg sizeGPR = size.gpr(); GPRReg resultGPR = result.gpr(); - GPRReg storageGPR = storage.gpr(); - GPRReg scratchGPR = scratch.gpr(); - GPRReg scratch2GPR = scratch2.gpr(); - - MacroAssembler::JumpList slowCases; - slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX))); - - ASSERT((1 << 3) == sizeof(JSValue)); - m_jit.move(sizeGPR, scratchGPR); - m_jit.lshift32(TrustedImm32(3), scratchGPR); - m_jit.add32(TrustedImm32(sizeof(IndexingHeader)), scratchGPR, resultGPR); - slowCases.append( - emitAllocateBasicStorage(resultGPR, storageGPR)); - m_jit.subPtr(scratchGPR, storageGPR); - Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()); - emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases); - - m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength())); - m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength())); - - if (hasDouble(node->indexingType())) { - JSValue nan = JSValue(JSValue::EncodeAsDouble, QNaN); - - m_jit.move(sizeGPR, scratchGPR); - MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, scratchGPR); - MacroAssembler::Label loop = m_jit.label(); - m_jit.sub32(TrustedImm32(1), scratchGPR); - m_jit.store32(TrustedImm32(nan.u.asBits.tag), MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag))); - m_jit.store32(TrustedImm32(nan.u.asBits.payload), MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload))); - m_jit.branchTest32(MacroAssembler::NonZero, scratchGPR).linkTo(loop, &m_jit); - done.link(&m_jit); - } - - addSlowPathGenerator(adoptPtr( - new CallArrayAllocatorWithVariableSizeSlowPathGenerator( - slowCases, this, operationNewArrayWithSize, resultGPR, - globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), - globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage), - sizeGPR))); - + + compileAllocateNewArrayWithSize(globalObject, resultGPR, sizeGPR, node->indexingType()); cellResult(resultGPR, node); break; } @@ -3367,25 +3941,25 @@ void SpeculativeJIT::compile(Node* node) SpeculateStrictInt32Operand size(this, node->child1()); GPRReg sizeGPR = size.gpr(); flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); GPRReg structureGPR = selectScratchGPR(sizeGPR); - MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX)); - m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), structureGPR); + MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)); + m_jit.move(TrustedImmPtr(m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()))), structureGPR); MacroAssembler::Jump done = m_jit.jump(); bigLength.link(&m_jit); - m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)), structureGPR); + m_jit.move(TrustedImmPtr(m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage))), structureGPR); done.link(&m_jit); - callOperation( - operationNewArrayWithSize, resultGPR, structureGPR, sizeGPR); + callOperation(operationNewArrayWithSize, resultGPR, structureGPR, sizeGPR, nullptr); + m_jit.exceptionCheck(); cellResult(resultGPR, node); break; } case NewArrayBuffer: { - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); IndexingType indexingType = node->indexingType(); - if (!globalObject->isHavingABadTime() && !hasArrayStorage(indexingType)) { + if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(indexingType)) { unsigned numElements = node->numConstants(); GPRTemporary result(this); @@ -3394,7 +3968,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg resultGPR = result.gpr(); GPRReg storageGPR = storage.gpr(); - emitAllocateJSArray(resultGPR, globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType), storageGPR, numElements); + emitAllocateRawObject(resultGPR, m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType)), storageGPR, numElements, numElements); if (node->indexingType() == ArrayWithDouble) { JSValue* data = m_jit.codeBlock()->constantBuffer(node->startConstant()); @@ -3420,9 +3994,10 @@ void SpeculativeJIT::compile(Node* node) } flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); - callOperation(operationNewArrayBuffer, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), node->startConstant(), node->numConstants()); + callOperation(operationNewArrayBuffer, result.gpr(), m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), node->startConstant(), node->numConstants()); + m_jit.exceptionCheck(); cellResult(result.gpr(), node); break; @@ -3435,19 +4010,18 @@ void SpeculativeJIT::compile(Node* node) break; case UntypedUse: { JSValueOperand argument(this, node->child1()); - GPRReg argumentTagGPR = argument.tagGPR(); - GPRReg argumentPayloadGPR = argument.payloadGPR(); + JSValueRegs argumentRegs = argument.jsValueRegs(); flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); callOperation( operationNewTypedArrayWithOneArgumentForType(node->typedArrayType()), - resultGPR, globalObject->typedArrayStructure(node->typedArrayType()), - argumentTagGPR, argumentPayloadGPR); + resultGPR, m_jit.graph().registerStructure(globalObject->typedArrayStructureConcurrently(node->typedArrayType())), argumentRegs); + m_jit.exceptionCheck(); cellResult(resultGPR, node); break; @@ -3461,15 +4035,22 @@ void SpeculativeJIT::compile(Node* node) case NewRegexp: { flushRegisters(); - GPRResult resultPayload(this); - GPRResult2 resultTag(this); + GPRFlushedCallResult resultPayload(this); + GPRFlushedCallResult2 resultTag(this); - callOperation(operationNewRegexp, resultTag.gpr(), resultPayload.gpr(), m_jit.codeBlock()->regexp(node->regexpIndex())); + RegExp* regexp = node->castOperand<RegExp*>(); + callOperation(operationNewRegexp, JSValueRegs(resultTag.gpr(), resultPayload.gpr()), regexp); + m_jit.exceptionCheck(); // FIXME: make the callOperation above explicitly return a cell result, or jitAssert the tag is a cell tag. cellResult(resultPayload.gpr(), node); break; } + + case CallObjectConstructor: { + compileCallObjectConstructor(node); + break; + } case ToThis: { ASSERT(node->child1().useKind() == UntypedUse); @@ -3482,25 +4063,23 @@ void SpeculativeJIT::compile(Node* node) GPRReg tempTagGPR = tempTag.gpr(); MacroAssembler::JumpList slowCases; - slowCases.append(m_jit.branch32( - MacroAssembler::NotEqual, thisValueTagGPR, TrustedImm32(JSValue::CellTag))); - m_jit.loadPtr( - MacroAssembler::Address(thisValuePayloadGPR, JSCell::structureOffset()), tempGPR); - slowCases.append(m_jit.branch8( - MacroAssembler::NotEqual, - MacroAssembler::Address(tempGPR, Structure::typeInfoTypeOffset()), - TrustedImm32(FinalObjectType))); + slowCases.append(m_jit.branchIfNotCell(thisValue.jsValueRegs())); + slowCases.append( + m_jit.branchTest8( + MacroAssembler::NonZero, + MacroAssembler::Address(thisValuePayloadGPR, JSCell::typeInfoFlagsOffset()), + MacroAssembler::TrustedImm32(OverridesToThis))); m_jit.move(thisValuePayloadGPR, tempGPR); m_jit.move(thisValueTagGPR, tempTagGPR); J_JITOperation_EJ function; - if (m_jit.graph().executableFor(node->codeOrigin)->isStrictMode()) + if (m_jit.graph().executableFor(node->origin.semantic)->isStrictMode()) function = operationToThisStrict; else function = operationToThis; addSlowPathGenerator( slowPathCall( slowCases, this, function, - JSValueRegs(tempTagGPR, tempGPR), thisValueTagGPR, thisValuePayloadGPR)); + JSValueRegs(tempTagGPR, tempGPR), JSValueRegs(thisValueTagGPR, thisValuePayloadGPR))); jsValueResult(tempTagGPR, tempGPR, node); break; @@ -3524,13 +4103,25 @@ void SpeculativeJIT::compile(Node* node) GPRReg allocatorGPR = allocator.gpr(); GPRReg structureGPR = structure.gpr(); GPRReg scratchGPR = scratch.gpr(); + // Rare data is only used to access the allocator & structure + // We can avoid using an additional GPR this way + GPRReg rareDataGPR = structureGPR; + GPRReg inlineCapacityGPR = rareDataGPR; MacroAssembler::JumpList slowPath; - m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR); - m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR); + slowPath.append(m_jit.branch8(JITCompiler::NotEqual, + JITCompiler::Address(calleeGPR, JSCell::typeInfoTypeOffset()), TrustedImm32(JSFunctionType))); + m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfRareData()), rareDataGPR); + slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, rareDataGPR)); + m_jit.loadPtr(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfObjectAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR); + m_jit.loadPtr(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfObjectAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR); slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, allocatorGPR)); - emitAllocateJSObject(resultGPR, allocatorGPR, structureGPR, TrustedImmPtr(0), scratchGPR, slowPath); + emitAllocateJSObject(resultGPR, nullptr, allocatorGPR, structureGPR, TrustedImmPtr(0), scratchGPR, slowPath); + + m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfRareData()), rareDataGPR); + m_jit.load32(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfObjectAllocationProfile() + ObjectAllocationProfile::offsetOfInlineCapacity()), inlineCapacityGPR); + m_jit.emitInitializeInlineStorage(resultGPR, inlineCapacityGPR); addSlowPathGenerator(slowPathCall(slowPath, this, operationCreateThis, resultGPR, calleeGPR, node->inlineCapacity())); @@ -3538,12 +4129,6 @@ void SpeculativeJIT::compile(Node* node) break; } - case AllocationProfileWatchpoint: - case TypedArrayWatchpoint: { - noResult(node); - break; - } - case NewObject: { GPRTemporary result(this); GPRTemporary allocator(this); @@ -3555,12 +4140,13 @@ void SpeculativeJIT::compile(Node* node) MacroAssembler::JumpList slowPath; - Structure* structure = node->structure(); + RegisteredStructure structure = node->structure(); size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity()); - MarkedAllocator* allocatorPtr = &m_jit.vm()->heap.allocatorForObjectWithoutDestructor(allocationSize); + MarkedAllocator* allocatorPtr = subspaceFor<JSFinalObject>(*m_jit.vm())->allocatorFor(allocationSize); m_jit.move(TrustedImmPtr(allocatorPtr), allocatorGPR); - emitAllocateJSObject(resultGPR, allocatorGPR, TrustedImmPtr(structure), TrustedImmPtr(0), scratchGPR, slowPath); + emitAllocateJSObject(resultGPR, allocatorPtr, allocatorGPR, TrustedImmPtr(structure), TrustedImmPtr(0), scratchGPR, slowPath); + m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity()); addSlowPathGenerator(slowPathCall(slowPath, this, operationNewObject, resultGPR, structure)); @@ -3570,107 +4156,70 @@ void SpeculativeJIT::compile(Node* node) case GetCallee: { GPRTemporary result(this); - m_jit.loadPtr(JITCompiler::payloadFor(JSStack::Callee), result.gpr()); + m_jit.loadPtr(JITCompiler::payloadFor(CallFrameSlot::callee), result.gpr()); cellResult(result.gpr(), node); break; } - case GetScope: { - SpeculateCellOperand function(this, node->child1()); - GPRTemporary result(this, Reuse, function); - m_jit.loadPtr(JITCompiler::Address(function.gpr(), JSFunction::offsetOfScopeChain()), result.gpr()); - cellResult(result.gpr(), node); - break; - } - - case GetMyScope: { + case GetArgumentCountIncludingThis: { GPRTemporary result(this); - GPRReg resultGPR = result.gpr(); - - m_jit.loadPtr(JITCompiler::payloadFor(JSStack::ScopeChain), resultGPR); - cellResult(resultGPR, node); + m_jit.load32(JITCompiler::payloadFor(CallFrameSlot::argumentCount), result.gpr()); + int32Result(result.gpr(), node); break; } - case SkipTopScope: { - SpeculateCellOperand scope(this, node->child1()); - GPRTemporary result(this, Reuse, scope); - GPRReg resultGPR = result.gpr(); - m_jit.move(scope.gpr(), resultGPR); - JITCompiler::Jump activationNotCreated = - m_jit.branchTestPtr( - JITCompiler::Zero, - JITCompiler::payloadFor( - static_cast<VirtualRegister>(m_jit.graph().machineActivationRegister()))); - m_jit.loadPtr(JITCompiler::Address(resultGPR, JSScope::offsetOfNext()), resultGPR); - activationNotCreated.link(&m_jit); - cellResult(resultGPR, node); + case GetScope: + compileGetScope(node); break; - } - - case SkipScope: { - SpeculateCellOperand scope(this, node->child1()); - GPRTemporary result(this, Reuse, scope); - m_jit.loadPtr(JITCompiler::Address(scope.gpr(), JSScope::offsetOfNext()), result.gpr()); - cellResult(result.gpr(), node); + + case SkipScope: + compileSkipScope(node); break; - } - - case GetClosureRegisters: { - if (WriteBarrierBase<Unknown>* registers = m_jit.graph().tryGetRegisters(node->child1().node())) { - GPRTemporary result(this); - GPRReg resultGPR = result.gpr(); - m_jit.move(TrustedImmPtr(registers), resultGPR); - storageResult(resultGPR, node); - break; - } - SpeculateCellOperand scope(this, node->child1()); - GPRTemporary result(this); - GPRReg scopeGPR = scope.gpr(); - GPRReg resultGPR = result.gpr(); - - m_jit.loadPtr(JITCompiler::Address(scopeGPR, JSVariableObject::offsetOfRegisters()), resultGPR); - storageResult(resultGPR, node); + case GetGlobalObject: + compileGetGlobalObject(node); break; - } + case GetClosureVar: { - StorageOperand registers(this, node->child1()); + SpeculateCellOperand base(this, node->child1()); GPRTemporary resultTag(this); GPRTemporary resultPayload(this); - GPRReg registersGPR = registers.gpr(); + GPRReg baseGPR = base.gpr(); GPRReg resultTagGPR = resultTag.gpr(); GPRReg resultPayloadGPR = resultPayload.gpr(); - m_jit.load32(JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTagGPR); - m_jit.load32(JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayloadGPR); + m_jit.load32(JITCompiler::Address(baseGPR, JSEnvironmentRecord::offsetOfVariable(node->scopeOffset()) + TagOffset), resultTagGPR); + m_jit.load32(JITCompiler::Address(baseGPR, JSEnvironmentRecord::offsetOfVariable(node->scopeOffset()) + PayloadOffset), resultPayloadGPR); jsValueResult(resultTagGPR, resultPayloadGPR, node); break; } + case PutClosureVar: { - StorageOperand registers(this, node->child2()); - JSValueOperand value(this, node->child3()); - GPRTemporary scratchRegister(this); + SpeculateCellOperand base(this, node->child1()); + JSValueOperand value(this, node->child2()); - GPRReg registersGPR = registers.gpr(); + GPRReg baseGPR = base.gpr(); GPRReg valueTagGPR = value.tagGPR(); GPRReg valuePayloadGPR = value.payloadGPR(); - speculate(node, node->child1()); - - m_jit.store32(valueTagGPR, JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag))); - m_jit.store32(valuePayloadGPR, JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload))); + m_jit.store32(valueTagGPR, JITCompiler::Address(baseGPR, JSEnvironmentRecord::offsetOfVariable(node->scopeOffset()) + TagOffset)); + m_jit.store32(valuePayloadGPR, JITCompiler::Address(baseGPR, JSEnvironmentRecord::offsetOfVariable(node->scopeOffset()) + PayloadOffset)); noResult(node); break; } - + + case TryGetById: { + compileTryGetById(node); + break; + } + case GetById: { - ASSERT(node->prediction()); - + // FIXME https://bugs.webkit.org/show_bug.cgi?id=161158 + // dedup with SpeculativeJIT::compileTryGetById and 64-bit version of this. switch (node->child1().useKind()) { case CellUse: { SpeculateCellOperand base(this, node->child1()); - GPRTemporary resultTag(this, Reuse, base); - GPRTemporary resultPayload(this); + GPRTemporary resultTag(this); + GPRTemporary resultPayload(this, Reuse, base); GPRReg baseGPR = base.gpr(); GPRReg resultTagGPR = resultTag.gpr(); @@ -3678,7 +4227,7 @@ void SpeculativeJIT::compile(Node* node) base.use(); - cachedGetById(node->codeOrigin, InvalidGPRReg, baseGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber()); + cachedGetById(node->origin.semantic, InvalidGPRReg, baseGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber()); jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly); break; @@ -3686,8 +4235,8 @@ void SpeculativeJIT::compile(Node* node) case UntypedUse: { JSValueOperand base(this, node->child1()); - GPRTemporary resultTag(this, Reuse, base, TagWord); - GPRTemporary resultPayload(this); + GPRTemporary resultTag(this); + GPRTemporary resultPayload(this, Reuse, base, TagWord); GPRReg baseTagGPR = base.tagGPR(); GPRReg basePayloadGPR = base.payloadGPR(); @@ -3696,9 +4245,9 @@ void SpeculativeJIT::compile(Node* node) base.use(); - JITCompiler::Jump notCell = m_jit.branch32(JITCompiler::NotEqual, baseTagGPR, TrustedImm32(JSValue::CellTag)); + JITCompiler::Jump notCell = m_jit.branchIfNotCell(base.jsValueRegs()); - cachedGetById(node->codeOrigin, baseTagGPR, basePayloadGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), notCell); + cachedGetById(node->origin.semantic, baseTagGPR, basePayloadGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), notCell); jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly); break; @@ -3711,6 +4260,25 @@ void SpeculativeJIT::compile(Node* node) break; } + case GetByIdWithThis: { + JSValueOperand base(this, node->child1()); + JSValueRegs baseRegs = base.jsValueRegs(); + JSValueOperand thisValue(this, node->child2()); + JSValueRegs thisRegs = thisValue.jsValueRegs(); + + GPRFlushedCallResult resultPayload(this); + GPRFlushedCallResult2 resultTag(this); + GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRReg resultTagGPR = resultTag.gpr(); + + flushRegisters(); + callOperation(operationGetByIdWithThis, JSValueRegs(resultTagGPR, resultPayloadGPR), baseRegs, thisRegs, identifierUID(node->identifierNumber())); + m_jit.exceptionCheck(); + + jsValueResult(resultTagGPR, resultPayloadGPR, node); + break; + } + case GetByIdFlush: { if (!node->prediction()) { terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); @@ -3723,16 +4291,16 @@ void SpeculativeJIT::compile(Node* node) GPRReg baseGPR = base.gpr(); - GPRResult resultTag(this); - GPRResult2 resultPayload(this); - GPRReg resultTagGPR = resultTag.gpr(); + GPRFlushedCallResult resultPayload(this); + GPRFlushedCallResult2 resultTag(this); GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRReg resultTagGPR = resultTag.gpr(); base.use(); flushRegisters(); - cachedGetById(node->codeOrigin, InvalidGPRReg, baseGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), JITCompiler::Jump(), DontSpill); + cachedGetById(node->origin.semantic, InvalidGPRReg, baseGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), JITCompiler::Jump(), DontSpill); jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly); break; @@ -3743,18 +4311,18 @@ void SpeculativeJIT::compile(Node* node) GPRReg baseTagGPR = base.tagGPR(); GPRReg basePayloadGPR = base.payloadGPR(); - GPRResult resultTag(this); - GPRResult2 resultPayload(this); - GPRReg resultTagGPR = resultTag.gpr(); + GPRFlushedCallResult resultPayload(this); + GPRFlushedCallResult2 resultTag(this); GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRReg resultTagGPR = resultTag.gpr(); base.use(); flushRegisters(); - JITCompiler::Jump notCell = m_jit.branch32(JITCompiler::NotEqual, baseTagGPR, TrustedImm32(JSValue::CellTag)); + JITCompiler::Jump notCell = m_jit.branchIfNotCell(base.jsValueRegs()); - cachedGetById(node->codeOrigin, baseTagGPR, basePayloadGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), notCell, DontSpill); + cachedGetById(node->origin.semantic, baseTagGPR, basePayloadGPR, resultTagGPR, resultPayloadGPR, node->identifierNumber(), notCell, DontSpill); jsValueResult(resultTagGPR, resultPayloadGPR, node, UseChildrenCalledExplicitly); break; @@ -3770,92 +4338,65 @@ void SpeculativeJIT::compile(Node* node) case GetArrayLength: compileGetArrayLength(node); break; - - case CheckFunction: { - SpeculateCellOperand function(this, node->child1()); - speculationCheck(BadFunction, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, function.gpr(), node->function())); - noResult(node); + + case DeleteById: { + compileDeleteById(node); break; } - case CheckExecutable: { - SpeculateCellOperand function(this, node->child1()); - speculationCheck(BadExecutable, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, JITCompiler::Address(function.gpr(), JSFunction::offsetOfExecutable()), node->executable())); - noResult(node); + case DeleteByVal: { + compileDeleteByVal(node); break; } - case CheckStructure: { - SpeculateCellOperand base(this, node->child1()); - - ASSERT(node->structureSet().size()); - - if (node->structureSet().size() == 1) { - speculationCheck( - BadCache, JSValueSource::unboxedCell(base.gpr()), 0, - m_jit.branchWeakPtr( - JITCompiler::NotEqual, - JITCompiler::Address(base.gpr(), JSCell::structureOffset()), - node->structureSet()[0])); - } else { - GPRTemporary structure(this); - - m_jit.loadPtr(JITCompiler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr()); - - JITCompiler::JumpList done; - - for (size_t i = 0; i < node->structureSet().size() - 1; ++i) - done.append(m_jit.branchWeakPtr(JITCompiler::Equal, structure.gpr(), node->structureSet()[i])); - - speculationCheck( - BadCache, JSValueSource::unboxedCell(base.gpr()), 0, - m_jit.branchWeakPtr( - JITCompiler::NotEqual, structure.gpr(), node->structureSet().last())); - - done.link(&m_jit); - } - + case CheckCell: { + SpeculateCellOperand cell(this, node->child1()); + speculationCheck(BadCell, JSValueSource::unboxedCell(cell.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, cell.gpr(), node->cellOperand()->cell())); noResult(node); break; } - - case StructureTransitionWatchpoint: { - // There is a fascinating question here of what to do about array profiling. - // We *could* try to tell the OSR exit about where the base of the access is. - // The DFG will have kept it alive, though it may not be in a register, and - // we shouldn't really load it since that could be a waste. For now though, - // we'll just rely on the fact that when a watchpoint fires then that's - // quite a hint already. - - m_jit.addWeakReference(node->structure()); - -#if !ASSERT_DISABLED - SpeculateCellOperand op1(this, node->child1()); - JITCompiler::Jump isOK = m_jit.branchPtr(JITCompiler::Equal, JITCompiler::Address(op1.gpr(), JSCell::structureOffset()), TrustedImmPtr(node->structure())); - m_jit.breakpoint(); - isOK.link(&m_jit); -#else - speculateCell(node->child1()); -#endif + case CheckNotEmpty: { + JSValueOperand operand(this, node->child1()); + GPRReg tagGPR = operand.tagGPR(); + speculationCheck(TDZFailure, JSValueSource(), nullptr, m_jit.branch32(JITCompiler::Equal, tagGPR, TrustedImm32(JSValue::EmptyValueTag))); noResult(node); break; } + + case CheckStringIdent: + compileCheckStringIdent(node); + break; + + case GetExecutable: { + SpeculateCellOperand function(this, node->child1()); + GPRTemporary result(this, Reuse, function); + GPRReg functionGPR = function.gpr(); + GPRReg resultGPR = result.gpr(); + speculateCellType(node->child1(), functionGPR, SpecFunction, JSFunctionType); + m_jit.loadPtr(JITCompiler::Address(functionGPR, JSFunction::offsetOfExecutable()), resultGPR); + cellResult(resultGPR, node); + break; + } - case PhantomPutStructure: { - ASSERT(isKnownCell(node->child1().node())); - m_jit.jitCode()->common.notifyCompilingStructureTransition(m_jit.graph().m_plan, m_jit.codeBlock(), node); - noResult(node); + case CheckStructure: { + compileCheckStructure(node); break; } - + case PutStructure: { + RegisteredStructure oldStructure = node->transition()->previous; + RegisteredStructure newStructure = node->transition()->next; + m_jit.jitCode()->common.notifyCompilingStructureTransition(m_jit.graph().m_plan, m_jit.codeBlock(), node); SpeculateCellOperand base(this, node->child1()); GPRReg baseGPR = base.gpr(); - m_jit.storePtr(MacroAssembler::TrustedImmPtr(node->structureTransitionData().newStructure), MacroAssembler::Address(baseGPR, JSCell::structureOffset())); + ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType()); + ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type()); + ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags()); + m_jit.storePtr(TrustedImmPtr(newStructure), MacroAssembler::Address(baseGPR, JSCell::structureIDOffset())); noResult(node); break; @@ -3869,18 +4410,13 @@ void SpeculativeJIT::compile(Node* node) compileReallocatePropertyStorage(node); break; - case GetButterfly: { - SpeculateCellOperand base(this, node->child1()); - GPRTemporary result(this, Reuse, base); - - GPRReg baseGPR = base.gpr(); - GPRReg resultGPR = result.gpr(); - - m_jit.loadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), resultGPR); + case NukeStructureAndSetButterfly: + compileNukeStructureAndSetButterfly(node); + break; - storageResult(resultGPR, node); + case GetButterfly: + compileGetButterfly(node); break; - } case GetIndexedPropertyStorage: { compileGetIndexedPropertyStorage(node); @@ -3906,7 +4442,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg resultTagGPR = resultTag.gpr(); GPRReg resultPayloadGPR = resultPayload.gpr(); - StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()]; + StorageAccessData& storageAccessData = node->storageAccessData(); m_jit.load32(JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayloadGPR); m_jit.load32(JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTagGPR); @@ -3915,6 +4451,47 @@ void SpeculativeJIT::compile(Node* node) break; } + case GetGetterSetterByOffset: { + StorageOperand storage(this, node->child1()); + GPRTemporary resultPayload(this); + + GPRReg storageGPR = storage.gpr(); + GPRReg resultPayloadGPR = resultPayload.gpr(); + + StorageAccessData& storageAccessData = node->storageAccessData(); + + m_jit.load32(JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayloadGPR); + + cellResult(resultPayloadGPR, node); + break; + } + + case GetGetter: { + SpeculateCellOperand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); + + GPRReg op1GPR = op1.gpr(); + GPRReg resultGPR = result.gpr(); + + m_jit.loadPtr(JITCompiler::Address(op1GPR, GetterSetter::offsetOfGetter()), resultGPR); + + cellResult(resultGPR, node); + break; + } + + case GetSetter: { + SpeculateCellOperand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); + + GPRReg op1GPR = op1.gpr(); + GPRReg resultGPR = result.gpr(); + + m_jit.loadPtr(JITCompiler::Address(op1GPR, GetterSetter::offsetOfSetter()), resultGPR); + + cellResult(resultGPR, node); + break; + } + case PutByOffset: { StorageOperand storage(this, node->child1()); JSValueOperand value(this, node->child3()); @@ -3925,7 +4502,7 @@ void SpeculativeJIT::compile(Node* node) speculate(node, node->child2()); - StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()]; + StorageAccessData& storageAccessData = node->storageAccessData(); m_jit.storePtr(valueTagGPR, JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag))); m_jit.storePtr(valuePayloadGPR, JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload))); @@ -3933,6 +4510,23 @@ void SpeculativeJIT::compile(Node* node) noResult(node); break; } + + case PutByIdFlush: { + SpeculateCellOperand base(this, node->child1()); + JSValueOperand value(this, node->child2()); + GPRTemporary scratch(this); + + GPRReg baseGPR = base.gpr(); + GPRReg valueTagGPR = value.tagGPR(); + GPRReg valuePayloadGPR = value.payloadGPR(); + GPRReg scratchGPR = scratch.gpr(); + flushRegisters(); + + cachedPutById(node->origin.semantic, baseGPR, valueTagGPR, valuePayloadGPR, scratchGPR, node->identifierNumber(), NotDirect, MacroAssembler::Jump(), DontSpill); + + noResult(node); + break; + } case PutById: { SpeculateCellOperand base(this, node->child1()); @@ -3944,7 +4538,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg valuePayloadGPR = value.payloadGPR(); GPRReg scratchGPR = scratch.gpr(); - cachedPutById(node->codeOrigin, baseGPR, valueTagGPR, valuePayloadGPR, scratchGPR, node->identifierNumber(), NotDirect); + cachedPutById(node->origin.semantic, baseGPR, valueTagGPR, valuePayloadGPR, scratchGPR, node->identifierNumber(), NotDirect); noResult(node); break; @@ -3960,17 +4554,62 @@ void SpeculativeJIT::compile(Node* node) GPRReg valuePayloadGPR = value.payloadGPR(); GPRReg scratchGPR = scratch.gpr(); - cachedPutById(node->codeOrigin, baseGPR, valueTagGPR, valuePayloadGPR, scratchGPR, node->identifierNumber(), Direct); + cachedPutById(node->origin.semantic, baseGPR, valueTagGPR, valuePayloadGPR, scratchGPR, node->identifierNumber(), Direct); + + noResult(node); + break; + } + + case PutByIdWithThis: { + JSValueOperand base(this, node->child1()); + JSValueRegs baseRegs = base.jsValueRegs(); + JSValueOperand thisValue(this, node->child2()); + JSValueRegs thisRegs = thisValue.jsValueRegs(); + JSValueOperand value(this, node->child3()); + JSValueRegs valueRegs = value.jsValueRegs(); + + flushRegisters(); + callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByIdWithThisStrict : operationPutByIdWithThis, + NoResult, baseRegs, thisRegs, valueRegs, identifierUID(node->identifierNumber())); + m_jit.exceptionCheck(); noResult(node); break; } + case PutGetterById: + case PutSetterById: { + compilePutAccessorById(node); + break; + } + + case PutGetterSetterById: { + compilePutGetterSetterById(node); + break; + } + + case PutGetterByVal: + case PutSetterByVal: { + compilePutAccessorByVal(node); + break; + } + + case DefineDataProperty: { + compileDefineDataProperty(node); + break; + } + + case DefineAccessorProperty: { + compileDefineAccessorProperty(node); + break; + } + + case GetGlobalLexicalVariable: case GetGlobalVar: { GPRTemporary resultPayload(this); GPRTemporary resultTag(this); - m_jit.move(TrustedImmPtr(node->registerPointer()), resultPayload.gpr()); + m_jit.move(TrustedImmPtr(node->variablePointer()), resultPayload.gpr()); m_jit.load32(JITCompiler::Address(resultPayload.gpr(), OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), resultTag.gpr()); m_jit.load32(JITCompiler::Address(resultPayload.gpr(), OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), resultPayload.gpr()); @@ -3978,91 +4617,68 @@ void SpeculativeJIT::compile(Node* node) break; } - case PutGlobalVar: { - JSValueOperand value(this, node->child1()); + case PutGlobalVariable: { + JSValueOperand value(this, node->child2()); // FIXME: if we happen to have a spare register - and _ONLY_ if we happen to have // a spare register - a good optimization would be to put the register pointer into // a register and then do a zero offset store followed by a four-offset store (or // vice-versa depending on endianness). - m_jit.store32(value.tagGPR(), node->registerPointer()->tagPointer()); - m_jit.store32(value.payloadGPR(), node->registerPointer()->payloadPointer()); + m_jit.store32(value.tagGPR(), node->variablePointer()->tagPointer()); + m_jit.store32(value.payloadGPR(), node->variablePointer()->payloadPointer()); noResult(node); break; } case NotifyWrite: { - VariableWatchpointSet* set = node->variableWatchpointSet(); - - JSValueOperand value(this, node->child1()); - GPRReg valueTagGPR = value.tagGPR(); - GPRReg valuePayloadGPR = value.payloadGPR(); - - GPRTemporary temp(this); - GPRReg tempGPR = temp.gpr(); - - m_jit.load8(set->addressOfState(), tempGPR); - - JITCompiler::JumpList ready; - - ready.append(m_jit.branch32(JITCompiler::Equal, tempGPR, TrustedImm32(IsInvalidated))); - - if (set->state() == ClearWatchpoint) { - JITCompiler::Jump isWatched = - m_jit.branch32(JITCompiler::NotEqual, tempGPR, TrustedImm32(ClearWatchpoint)); - - m_jit.store32(valueTagGPR, &set->addressOfInferredValue()->u.asBits.tag); - m_jit.store32(valuePayloadGPR, &set->addressOfInferredValue()->u.asBits.payload); - m_jit.store8(TrustedImm32(IsWatched), set->addressOfState()); - ready.append(m_jit.jump()); - - isWatched.link(&m_jit); - } + compileNotifyWrite(node); + break; + } - JITCompiler::Jump definitelyNotEqual = m_jit.branch32( - JITCompiler::NotEqual, - JITCompiler::AbsoluteAddress(&set->addressOfInferredValue()->u.asBits.payload), - valuePayloadGPR); - ready.append(m_jit.branch32( - JITCompiler::Equal, - JITCompiler::AbsoluteAddress(&set->addressOfInferredValue()->u.asBits.tag), - valueTagGPR)); - definitelyNotEqual.link(&m_jit); - - JITCompiler::Jump slowCase = m_jit.branchTest8( - JITCompiler::NonZero, JITCompiler::AbsoluteAddress(set->addressOfSetIsNotEmpty())); - m_jit.store8(TrustedImm32(IsInvalidated), set->addressOfState()); - m_jit.store32( - TrustedImm32(JSValue::EmptyValueTag), - &set->addressOfInferredValue()->u.asBits.tag); - m_jit.store32( - TrustedImm32(0), &set->addressOfInferredValue()->u.asBits.payload); - - ready.link(&m_jit); - - addSlowPathGenerator( - slowPathCall(slowCase, this, operationInvalidate, NoResult, set)); - - noResult(node); + case ParseInt: { + compileParseInt(node); break; } - case VarInjectionWatchpoint: - case VariableWatchpoint: { - noResult(node); + case CheckTypeInfoFlags: { + compileCheckTypeInfoFlags(node); break; } - case CheckHasInstance: { + case OverridesHasInstance: { + + Node* hasInstanceValueNode = node->child2().node(); + JSFunction* defaultHasInstanceFunction = jsCast<JSFunction*>(node->cellOperand()->value()); + + MacroAssembler::JumpList notDefaultHasInstanceValue; SpeculateCellOperand base(this, node->child1()); - GPRTemporary structure(this); + JSValueOperand hasInstanceValue(this, node->child2()); + GPRTemporary result(this); + + GPRReg baseGPR = base.gpr(); + GPRReg resultGPR = result.gpr(); + + // It would be great if constant folding handled automatically the case where we knew the hasInstance function + // was a constant. Unfortunately, the folding rule for OverridesHasInstance is in the strength reduction phase + // since it relies on OSR information. https://bugs.webkit.org/show_bug.cgi?id=154832 + if (!hasInstanceValueNode->isCellConstant() || defaultHasInstanceFunction != hasInstanceValueNode->asCell()) { + JSValueRegs hasInstanceValueRegs = hasInstanceValue.jsValueRegs(); + notDefaultHasInstanceValue.append(m_jit.branchIfNotCell(hasInstanceValueRegs)); + notDefaultHasInstanceValue.append(m_jit.branchPtr(MacroAssembler::NotEqual, hasInstanceValueRegs.payloadGPR(), TrustedImmPtr(node->cellOperand()))); + } - // Speculate that base 'ImplementsDefaultHasInstance'. - m_jit.loadPtr(MacroAssembler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr()); - speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(structure.gpr(), Structure::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance))); + // Check that constructor 'ImplementsDefaultHasInstance'. + m_jit.test8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance), resultGPR); + MacroAssembler::Jump done = m_jit.jump(); - noResult(node); + if (!notDefaultHasInstanceValue.empty()) { + notDefaultHasInstanceValue.link(&m_jit); + moveTrueTo(resultGPR); + } + + done.link(&m_jit); + booleanResult(resultGPR, node); break; } @@ -4071,13 +4687,26 @@ void SpeculativeJIT::compile(Node* node) break; } + case InstanceOfCustom: { + compileInstanceOfCustom(node); + break; + } + + case IsEmpty: { + JSValueOperand value(this, node->child1()); + GPRTemporary result(this, Reuse, value, TagWord); + m_jit.comparePtr(JITCompiler::Equal, value.tagGPR(), TrustedImm32(JSValue::EmptyValueTag), result.gpr()); + booleanResult(result.gpr(), node); + break; + } + case IsUndefined: { JSValueOperand value(this, node->child1()); GPRTemporary result(this); GPRTemporary localGlobalObject(this); GPRTemporary remoteGlobalObject(this); - JITCompiler::Jump isCell = m_jit.branch32(JITCompiler::Equal, value.tagGPR(), JITCompiler::TrustedImm32(JSValue::CellTag)); + JITCompiler::Jump isCell = m_jit.branchIfCell(value.jsValueRegs()); m_jit.compare32(JITCompiler::Equal, value.tagGPR(), TrustedImm32(JSValue::UndefinedTag), result.gpr()); JITCompiler::Jump done = m_jit.jump(); @@ -4088,15 +4717,18 @@ void SpeculativeJIT::compile(Node* node) m_jit.move(TrustedImm32(0), result.gpr()); notMasqueradesAsUndefined = m_jit.jump(); } else { - m_jit.loadPtr(JITCompiler::Address(value.payloadGPR(), JSCell::structureOffset()), result.gpr()); - JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(result.gpr(), Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)); + JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8( + JITCompiler::NonZero, + JITCompiler::Address(value.payloadGPR(), JSCell::typeInfoFlagsOffset()), + TrustedImm32(MasqueradesAsUndefined)); m_jit.move(TrustedImm32(0), result.gpr()); notMasqueradesAsUndefined = m_jit.jump(); isMasqueradesAsUndefined.link(&m_jit); GPRReg localGlobalObjectGPR = localGlobalObject.gpr(); GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr(); - m_jit.move(TrustedImmPtr(m_jit.globalObjectFor(node->codeOrigin)), localGlobalObjectGPR); + m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.globalObjectFor(node->origin.semantic)), localGlobalObjectGPR); + m_jit.loadPtr(JITCompiler::Address(value.payloadGPR(), JSCell::structureIDOffset()), result.gpr()); m_jit.loadPtr(JITCompiler::Address(result.gpr(), Structure::globalObjectOffset()), remoteGlobalObjectGPR); m_jit.compare32(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, result.gpr()); } @@ -4126,106 +4758,124 @@ void SpeculativeJIT::compile(Node* node) break; } - case IsString: { + case IsObject: { JSValueOperand value(this, node->child1()); GPRTemporary result(this, Reuse, value, TagWord); - - JITCompiler::Jump isNotCell = m_jit.branch32(JITCompiler::NotEqual, value.tagGPR(), JITCompiler::TrustedImm32(JSValue::CellTag)); - - m_jit.loadPtr(JITCompiler::Address(value.payloadGPR(), JSCell::structureOffset()), result.gpr()); - m_jit.compare8(JITCompiler::Equal, JITCompiler::Address(result.gpr(), Structure::typeInfoTypeOffset()), TrustedImm32(StringType), result.gpr()); + + JITCompiler::Jump isNotCell = m_jit.branchIfNotCell(value.jsValueRegs()); + + m_jit.compare8(JITCompiler::AboveOrEqual, + JITCompiler::Address(value.payloadGPR(), JSCell::typeInfoTypeOffset()), + TrustedImm32(ObjectType), + result.gpr()); JITCompiler::Jump done = m_jit.jump(); - + isNotCell.link(&m_jit); m_jit.move(TrustedImm32(0), result.gpr()); - + done.link(&m_jit); booleanResult(result.gpr(), node); break; } - case IsObject: { - JSValueOperand value(this, node->child1()); - GPRReg valueTagGPR = value.tagGPR(); - GPRReg valuePayloadGPR = value.payloadGPR(); - GPRResult result(this); - GPRReg resultGPR = result.gpr(); - flushRegisters(); - callOperation(operationIsObject, resultGPR, valueTagGPR, valuePayloadGPR); - booleanResult(result.gpr(), node); + case IsObjectOrNull: { + compileIsObjectOrNull(node); break; } case IsFunction: { - JSValueOperand value(this, node->child1()); - GPRReg valueTagGPR = value.tagGPR(); - GPRReg valuePayloadGPR = value.payloadGPR(); - GPRResult result(this); + compileIsFunction(node); + break; + } + + case IsCellWithType: { + compileIsCellWithType(node); + break; + } + + case IsTypedArrayView: { + compileIsTypedArrayView(node); + break; + } + + case TypeOf: { + compileTypeOf(node); + break; + } + + case MapHash: { + JSValueOperand input(this, node->child1()); + + JSValueRegs inputRegs = input.jsValueRegs(); + + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); + flushRegisters(); - callOperation(operationIsFunction, resultGPR, valueTagGPR, valuePayloadGPR); - booleanResult(result.gpr(), node); + callOperation(operationMapHash, resultGPR, inputRegs); + m_jit.exceptionCheck(); + int32Result(resultGPR, node); break; } - case TypeOf: { - JSValueOperand value(this, node->child1(), ManualOperandSpeculation); - GPRReg tagGPR = value.tagGPR(); - GPRReg payloadGPR = value.payloadGPR(); - GPRTemporary temp(this); - GPRReg tempGPR = temp.gpr(); - GPRResult result(this); + + case GetMapBucket: { + SpeculateCellOperand map(this, node->child1()); + JSValueOperand key(this, node->child2()); + SpeculateInt32Operand hash(this, node->child3()); + GPRFlushedCallResult result(this); + + GPRReg mapGPR = map.gpr(); + JSValueRegs keyRegs = key.jsValueRegs(); + GPRReg hashGPR = hash.gpr(); GPRReg resultGPR = result.gpr(); - JITCompiler::JumpList doneJumps; + + if (node->child1().useKind() == MapObjectUse) + speculateMapObject(node->child1(), mapGPR); + else if (node->child1().useKind() == SetObjectUse) + speculateSetObject(node->child1(), mapGPR); + else + RELEASE_ASSERT_NOT_REACHED(); flushRegisters(); + if (node->child1().useKind() == MapObjectUse) + callOperation(operationJSMapFindBucket, resultGPR, mapGPR, keyRegs, hashGPR); + else + callOperation(operationJSSetFindBucket, resultGPR, mapGPR, keyRegs, hashGPR); + m_jit.exceptionCheck(); + cellResult(resultGPR, node); + break; + } - ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == CellUse || node->child1().useKind() == StringUse); - - JITCompiler::Jump isNotCell = m_jit.branch32(JITCompiler::NotEqual, tagGPR, JITCompiler::TrustedImm32(JSValue::CellTag)); - if (node->child1().useKind() != UntypedUse) - DFG_TYPE_CHECK(JSValueRegs(tagGPR, payloadGPR), node->child1(), SpecCell, isNotCell); - - if (!node->child1()->shouldSpeculateObject() || node->child1().useKind() == StringUse) { - m_jit.loadPtr(JITCompiler::Address(payloadGPR, JSCell::structureOffset()), tempGPR); - JITCompiler::Jump notString = m_jit.branch8(JITCompiler::NotEqual, JITCompiler::Address(tempGPR, Structure::typeInfoTypeOffset()), TrustedImm32(StringType)); - if (node->child1().useKind() == StringUse) - DFG_TYPE_CHECK(JSValueRegs(tagGPR, payloadGPR), node->child1(), SpecString, notString); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.stringString()), resultGPR); - doneJumps.append(m_jit.jump()); - if (node->child1().useKind() != StringUse) { - notString.link(&m_jit); - callOperation(operationTypeOf, resultGPR, payloadGPR); - doneJumps.append(m_jit.jump()); - } - } else { - callOperation(operationTypeOf, resultGPR, payloadGPR); - doneJumps.append(m_jit.jump()); - } + case LoadFromJSMapBucket: { + SpeculateCellOperand bucket(this, node->child1()); + GPRTemporary resultPayload(this); + GPRTemporary resultTag(this); - if (node->child1().useKind() == UntypedUse) { - isNotCell.link(&m_jit); + GPRReg bucketGPR = bucket.gpr(); + GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRReg resultTagGPR = resultTag.gpr(); - m_jit.add32(TrustedImm32(1), tagGPR, tempGPR); - JITCompiler::Jump notNumber = m_jit.branch32(JITCompiler::AboveOrEqual, tempGPR, JITCompiler::TrustedImm32(JSValue::LowestTag + 1)); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.numberString()), resultGPR); - doneJumps.append(m_jit.jump()); - notNumber.link(&m_jit); + auto notBucket = m_jit.branchTestPtr(MacroAssembler::Zero, bucketGPR); + m_jit.loadValue(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfValue()), JSValueRegs(resultTagGPR, resultPayloadGPR)); + auto done = m_jit.jump(); + + notBucket.link(&m_jit); + m_jit.move(TrustedImm32(JSValue::UndefinedTag), resultTagGPR); + m_jit.move(TrustedImm32(0), resultPayloadGPR); + done.link(&m_jit); + jsValueResult(resultTagGPR, resultPayloadGPR, node); + break; + } - JITCompiler::Jump notUndefined = m_jit.branch32(JITCompiler::NotEqual, tagGPR, TrustedImm32(JSValue::UndefinedTag)); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.undefinedString()), resultGPR); - doneJumps.append(m_jit.jump()); - notUndefined.link(&m_jit); + case IsNonEmptyMapBucket: { + SpeculateCellOperand bucket(this, node->child1()); + GPRTemporary result(this); - JITCompiler::Jump notNull = m_jit.branch32(JITCompiler::NotEqual, tagGPR, TrustedImm32(JSValue::NullTag)); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.objectString()), resultGPR); - doneJumps.append(m_jit.jump()); - notNull.link(&m_jit); + GPRReg bucketGPR = bucket.gpr(); + GPRReg resultGPR = result.gpr(); - // Only boolean left - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.booleanString()), resultGPR); - } - doneJumps.link(&m_jit); - cellResult(resultGPR, node); + m_jit.comparePtr(MacroAssembler::NotEqual, bucketGPR, TrustedImm32(0), resultGPR); + booleanResult(resultGPR, node); break; } @@ -4233,414 +4883,624 @@ void SpeculativeJIT::compile(Node* node) break; case Call: + case TailCall: + case TailCallInlinedCaller: case Construct: + case CallVarargs: + case TailCallVarargs: + case TailCallVarargsInlinedCaller: + case ConstructVarargs: + case CallForwardVarargs: + case TailCallForwardVarargs: + case TailCallForwardVarargsInlinedCaller: + case ConstructForwardVarargs: + case CallEval: + case DirectCall: + case DirectConstruct: + case DirectTailCall: + case DirectTailCallInlinedCaller: emitCall(node); break; - case CreateActivation: { - JSValueOperand value(this, node->child1()); - GPRTemporary result(this, Reuse, value, PayloadWord); + case LoadVarargs: { + LoadVarargsData* data = node->loadVarargsData(); - GPRReg valueTagGPR = value.tagGPR(); - GPRReg valuePayloadGPR = value.payloadGPR(); - GPRReg resultGPR = result.gpr(); + GPRReg argumentsTagGPR; + GPRReg argumentsPayloadGPR; + JSValueRegs argumentsRegs; + { + JSValueOperand arguments(this, node->child1()); + argumentsRegs = arguments.jsValueRegs(); + flushRegisters(); + } - m_jit.move(valuePayloadGPR, resultGPR); + callOperation(operationSizeOfVarargs, GPRInfo::returnValueGPR, argumentsRegs, data->offset); + m_jit.exceptionCheck(); - JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag)); + lock(GPRInfo::returnValueGPR); + { + JSValueOperand arguments(this, node->child1()); + argumentsTagGPR = arguments.tagGPR(); + argumentsPayloadGPR = arguments.payloadGPR(); + argumentsRegs = arguments.jsValueRegs(); + flushRegisters(); + } + unlock(GPRInfo::returnValueGPR); - addSlowPathGenerator( - slowPathCall( - notCreated, this, operationCreateActivation, resultGPR, - framePointerOffsetToGetActivationRegisters())); + // FIXME: There is a chance that we will call an effectful length property twice. This is safe + // from the standpoint of the VM's integrity, but it's subtly wrong from a spec compliance + // standpoint. The best solution would be one where we can exit *into* the op_call_varargs right + // past the sizing. + // https://bugs.webkit.org/show_bug.cgi?id=141448 + + GPRReg argCountIncludingThisGPR = + JITCompiler::selectScratchGPR(GPRInfo::returnValueGPR, argumentsTagGPR, argumentsPayloadGPR); - cellResult(resultGPR, node); - break; - } + m_jit.add32(TrustedImm32(1), GPRInfo::returnValueGPR, argCountIncludingThisGPR); + speculationCheck( + VarargsOverflow, JSValueSource(), Edge(), m_jit.branch32( + MacroAssembler::Above, + argCountIncludingThisGPR, + TrustedImm32(data->limit))); + + m_jit.store32(argCountIncludingThisGPR, JITCompiler::payloadFor(data->machineCount)); + + callOperation(operationLoadVarargs, data->machineStart.offset(), argumentsRegs, data->offset, GPRInfo::returnValueGPR, data->mandatoryMinimum); + m_jit.exceptionCheck(); - case FunctionReentryWatchpoint: { noResult(node); break; } - case CreateArguments: { - JSValueOperand value(this, node->child1()); - GPRTemporary result(this, Reuse, value, PayloadWord); - - GPRReg valueTagGPR = value.tagGPR(); - GPRReg valuePayloadGPR = value.payloadGPR(); - GPRReg resultGPR = result.gpr(); - - m_jit.move(valuePayloadGPR, resultGPR); - - JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag)); + case ForwardVarargs: { + compileForwardVarargs(node); + break; + } - if (node->codeOrigin.inlineCallFrame) { - addSlowPathGenerator( - slowPathCall( - notCreated, this, operationCreateInlinedArguments, resultGPR, - node->codeOrigin.inlineCallFrame)); - } else { - addSlowPathGenerator( - slowPathCall(notCreated, this, operationCreateArguments, resultGPR)); - } + case CreateActivation: { + compileCreateActivation(node); + break; + } - cellResult(resultGPR, node); + case CreateDirectArguments: { + compileCreateDirectArguments(node); break; } - case TearOffActivation: { - JSValueOperand activationValue(this, node->child1()); - GPRTemporary scratch(this); + case GetFromArguments: { + compileGetFromArguments(node); + break; + } - GPRReg activationValueTagGPR = activationValue.tagGPR(); - GPRReg activationValuePayloadGPR = activationValue.payloadGPR(); - GPRReg scratchGPR = scratch.gpr(); + case PutToArguments: { + compilePutToArguments(node); + break; + } - JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, activationValueTagGPR, TrustedImm32(JSValue::EmptyValueTag)); - - SymbolTable* symbolTable = m_jit.symbolTableFor(node->codeOrigin); - int registersOffset = JSActivation::registersOffset(symbolTable); - - int bytecodeCaptureStart = symbolTable->captureStart(); - int machineCaptureStart = m_jit.graph().m_machineCaptureStart; - for (int i = symbolTable->captureCount(); i--;) { - m_jit.loadPtr( - JITCompiler::Address( - GPRInfo::callFrameRegister, (machineCaptureStart - i) * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), - scratchGPR); - m_jit.storePtr( - scratchGPR, JITCompiler::Address( - activationValuePayloadGPR, registersOffset + (bytecodeCaptureStart - i) * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag))); - m_jit.loadPtr( - JITCompiler::Address( - GPRInfo::callFrameRegister, (machineCaptureStart - i) * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), - scratchGPR); - m_jit.storePtr( - scratchGPR, JITCompiler::Address( - activationValuePayloadGPR, registersOffset + (bytecodeCaptureStart - i) * sizeof(Register) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload))); - } - m_jit.addPtr(TrustedImm32(registersOffset), activationValuePayloadGPR, scratchGPR); - m_jit.storePtr(scratchGPR, JITCompiler::Address(activationValuePayloadGPR, JSActivation::offsetOfRegisters())); - - notCreated.link(&m_jit); - noResult(node); + case GetArgument: { + compileGetArgument(node); break; } - case TearOffArguments: { - JSValueOperand unmodifiedArgumentsValue(this, node->child1()); - JSValueOperand activationValue(this, node->child2()); - GPRReg unmodifiedArgumentsValuePayloadGPR = unmodifiedArgumentsValue.payloadGPR(); - GPRReg activationValuePayloadGPR = activationValue.payloadGPR(); - - JITCompiler::Jump created = m_jit.branchTest32( - JITCompiler::NonZero, unmodifiedArgumentsValuePayloadGPR); - - if (node->codeOrigin.inlineCallFrame) { - addSlowPathGenerator( - slowPathCall( - created, this, operationTearOffInlinedArguments, NoResult, - unmodifiedArgumentsValuePayloadGPR, activationValuePayloadGPR, node->codeOrigin.inlineCallFrame)); - } else { - addSlowPathGenerator( - slowPathCall( - created, this, operationTearOffArguments, NoResult, - unmodifiedArgumentsValuePayloadGPR, activationValuePayloadGPR)); - } - - noResult(node); + case CreateScopedArguments: { + compileCreateScopedArguments(node); break; } - case CheckArgumentsNotCreated: { - ASSERT(!isEmptySpeculation( - m_state.variables().operand( - m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)); - speculationCheck( - Uncountable, JSValueRegs(), 0, - m_jit.branch32( - JITCompiler::NotEqual, - JITCompiler::tagFor(m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin)), - TrustedImm32(JSValue::EmptyValueTag))); - noResult(node); + case CreateClonedArguments: { + compileCreateClonedArguments(node); break; } - - case GetMyArgumentsLength: { + + case CreateRest: { + compileCreateRest(node); + break; + } + + case GetRestLength: { + compileGetRestLength(node); + break; + } + + case NewFunction: + case NewGeneratorFunction: + case NewAsyncFunction: + compileNewFunction(node); + break; + + case SetFunctionName: + compileSetFunctionName(node); + break; + + case In: + compileIn(node); + break; + + case HasOwnProperty: { +#if CPU(X86) + ASSERT(node->child2().useKind() == UntypedUse); + SpeculateCellOperand object(this, node->child1()); + JSValueOperand key(this, node->child2()); + GPRTemporary result(this, Reuse, object); + + JSValueRegs keyRegs = key.jsValueRegs(); + GPRReg objectGPR = object.gpr(); + GPRReg resultGPR = result.gpr(); + + speculateObject(node->child1()); + + flushRegisters(); + callOperation(operationHasOwnProperty, resultGPR, objectGPR, keyRegs); + booleanResult(resultGPR, node); +#else + SpeculateCellOperand object(this, node->child1()); + GPRTemporary uniquedStringImpl(this); + GPRTemporary temp(this); + GPRTemporary hash(this); + GPRTemporary structureID(this); GPRTemporary result(this); + + std::optional<SpeculateCellOperand> keyAsCell; + std::optional<JSValueOperand> keyAsValue; + JSValueRegs keyRegs; + if (node->child2().useKind() == UntypedUse) { + keyAsValue.emplace(this, node->child2()); + keyRegs = keyAsValue->jsValueRegs(); + } else { + ASSERT(node->child2().useKind() == StringUse || node->child2().useKind() == SymbolUse); + keyAsCell.emplace(this, node->child2()); + keyRegs = JSValueRegs::payloadOnly(keyAsCell->gpr()); + } + + GPRReg objectGPR = object.gpr(); + GPRReg implGPR = uniquedStringImpl.gpr(); + GPRReg tempGPR = temp.gpr(); + GPRReg hashGPR = hash.gpr(); + GPRReg structureIDGPR = structureID.gpr(); GPRReg resultGPR = result.gpr(); - - if (!isEmptySpeculation( - m_state.variables().operand( - m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) { - speculationCheck( - ArgumentsEscaped, JSValueRegs(), 0, - m_jit.branch32( - JITCompiler::NotEqual, - JITCompiler::tagFor(m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin)), - TrustedImm32(JSValue::EmptyValueTag))); + + speculateObject(node->child1()); + + MacroAssembler::JumpList slowPath; + switch (node->child2().useKind()) { + case SymbolUse: { + speculateSymbol(node->child2(), keyRegs.payloadGPR()); + m_jit.loadPtr(MacroAssembler::Address(keyRegs.payloadGPR(), Symbol::offsetOfSymbolImpl()), implGPR); + break; } - - ASSERT(!node->codeOrigin.inlineCallFrame); - m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultGPR); - m_jit.sub32(TrustedImm32(1), resultGPR); + case StringUse: { + speculateString(node->child2(), keyRegs.payloadGPR()); + m_jit.loadPtr(MacroAssembler::Address(keyRegs.payloadGPR(), JSString::offsetOfValue()), implGPR); + slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, implGPR)); + slowPath.append(m_jit.branchTest32( + MacroAssembler::Zero, MacroAssembler::Address(implGPR, StringImpl::flagsOffset()), + MacroAssembler::TrustedImm32(StringImpl::flagIsAtomic()))); + break; + } + case UntypedUse: { + slowPath.append(m_jit.branchIfNotCell(keyRegs)); + auto isNotString = m_jit.branchIfNotString(keyRegs.payloadGPR()); + m_jit.loadPtr(MacroAssembler::Address(keyRegs.payloadGPR(), JSString::offsetOfValue()), implGPR); + slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, implGPR)); + slowPath.append(m_jit.branchTest32( + MacroAssembler::Zero, MacroAssembler::Address(implGPR, StringImpl::flagsOffset()), + MacroAssembler::TrustedImm32(StringImpl::flagIsAtomic()))); + auto hasUniquedImpl = m_jit.jump(); + + isNotString.link(&m_jit); + slowPath.append(m_jit.branchIfNotSymbol(keyRegs.payloadGPR())); + m_jit.loadPtr(MacroAssembler::Address(keyRegs.payloadGPR(), Symbol::offsetOfSymbolImpl()), implGPR); + + hasUniquedImpl.link(&m_jit); + break; + } + default: + RELEASE_ASSERT_NOT_REACHED(); + } + + // Note that we don't test if the hash is zero here. AtomicStringImpl's can't have a zero + // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only + // ever load the result from the cache if the cache entry matches what we are querying for. + // So we either get super lucky and use zero for the hash and somehow collide with the entity + // we're looking for, or we realize we're comparing against another entity, and go to the + // slow path anyways. + m_jit.load32(MacroAssembler::Address(implGPR, UniquedStringImpl::flagsOffset()), hashGPR); + m_jit.urshift32(MacroAssembler::TrustedImm32(StringImpl::s_flagCount), hashGPR); + m_jit.load32(MacroAssembler::Address(objectGPR, JSCell::structureIDOffset()), structureIDGPR); + m_jit.add32(structureIDGPR, hashGPR); + m_jit.and32(TrustedImm32(HasOwnPropertyCache::mask), hashGPR); + m_jit.mul32(TrustedImm32(sizeof(HasOwnPropertyCache::Entry)), hashGPR, hashGPR); + ASSERT(m_jit.vm()->hasOwnPropertyCache()); + m_jit.move(TrustedImmPtr(m_jit.vm()->hasOwnPropertyCache()), tempGPR); + slowPath.append(m_jit.branchPtr(MacroAssembler::NotEqual, + MacroAssembler::BaseIndex(tempGPR, hashGPR, MacroAssembler::TimesOne, HasOwnPropertyCache::Entry::offsetOfImpl()), implGPR)); + m_jit.load8(MacroAssembler::BaseIndex(tempGPR, hashGPR, MacroAssembler::TimesOne, HasOwnPropertyCache::Entry::offsetOfResult()), resultGPR); + m_jit.load32(MacroAssembler::BaseIndex(tempGPR, hashGPR, MacroAssembler::TimesOne, HasOwnPropertyCache::Entry::offsetOfStructureID()), tempGPR); + slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, structureIDGPR)); + auto done = m_jit.jump(); + + slowPath.link(&m_jit); + silentSpillAllRegisters(resultGPR); + if (node->child2().useKind() != UntypedUse) { + m_jit.move(MacroAssembler::TrustedImm32(JSValue::CellTag), tempGPR); + keyRegs = JSValueRegs(tempGPR, keyRegs.payloadGPR()); + } + callOperation(operationHasOwnProperty, resultGPR, objectGPR, keyRegs); + silentFillAllRegisters(resultGPR); + m_jit.exceptionCheck(); + + done.link(&m_jit); + booleanResult(resultGPR, node); +#endif // CPU(X86) + break; + } + + case StoreBarrier: + case FencedStoreBarrier: { + compileStoreBarrier(node); + break; + } + + case GetEnumerableLength: { + SpeculateCellOperand enumerator(this, node->child1()); + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + m_jit.load32(MacroAssembler::Address(enumerator.gpr(), JSPropertyNameEnumerator::indexedLengthOffset()), resultGPR); int32Result(resultGPR, node); break; } - - case GetMyArgumentsLengthSafe: { + case HasGenericProperty: { + JSValueOperand base(this, node->child1()); + SpeculateCellOperand property(this, node->child2()); + GPRFlushedCallResult resultPayload(this); + GPRFlushedCallResult2 resultTag(this); + JSValueRegs baseRegs = base.jsValueRegs(); + GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRReg resultTagGPR = resultTag.gpr(); + + flushRegisters(); + callOperation(operationHasGenericProperty, JSValueRegs(resultTagGPR, resultPayloadGPR), baseRegs, property.gpr()); + m_jit.exceptionCheck(); + booleanResult(resultPayloadGPR, node); + break; + } + case HasStructureProperty: { + JSValueOperand base(this, node->child1()); + SpeculateCellOperand property(this, node->child2()); + SpeculateCellOperand enumerator(this, node->child3()); GPRTemporary resultPayload(this); GPRTemporary resultTag(this); + + GPRReg baseTagGPR = base.tagGPR(); + GPRReg basePayloadGPR = base.payloadGPR(); + GPRReg propertyGPR = property.gpr(); GPRReg resultPayloadGPR = resultPayload.gpr(); GPRReg resultTagGPR = resultTag.gpr(); - - JITCompiler::Jump created = m_jit.branch32( - JITCompiler::NotEqual, - JITCompiler::tagFor(m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin)), - TrustedImm32(JSValue::EmptyValueTag)); - - if (node->codeOrigin.inlineCallFrame) { - m_jit.move( - Imm32(node->codeOrigin.inlineCallFrame->arguments.size() - 1), - resultPayloadGPR); - } else { - m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultPayloadGPR); - m_jit.sub32(TrustedImm32(1), resultPayloadGPR); - } - m_jit.move(TrustedImm32(JSValue::Int32Tag), resultTagGPR); - - // FIXME: the slow path generator should perform a forward speculation that the - // result is an integer. For now we postpone the speculation by having this return - // a JSValue. - - addSlowPathGenerator( - slowPathCall( - created, this, operationGetArgumentsLength, - JSValueRegs(resultTagGPR, resultPayloadGPR), - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin).offset())); - - jsValueResult(resultTagGPR, resultPayloadGPR, node); + + m_jit.load32(MacroAssembler::Address(basePayloadGPR, JSCell::structureIDOffset()), resultTagGPR); + MacroAssembler::Jump wrongStructure = m_jit.branch32(MacroAssembler::NotEqual, + resultTagGPR, + MacroAssembler::Address(enumerator.gpr(), JSPropertyNameEnumerator::cachedStructureIDOffset())); + + moveTrueTo(resultPayloadGPR); + MacroAssembler::Jump done = m_jit.jump(); + + done.link(&m_jit); + + addSlowPathGenerator(slowPathCall(wrongStructure, this, operationHasGenericProperty, + JSValueRegs(resultTagGPR, resultPayloadGPR), JSValueRegs(baseTagGPR, basePayloadGPR), propertyGPR)); + booleanResult(resultPayloadGPR, node); break; } - - case GetMyArgumentByVal: { - SpeculateStrictInt32Operand index(this, node->child1()); + case HasIndexedProperty: { + SpeculateCellOperand base(this, node->child1()); + SpeculateInt32Operand index(this, node->child2()); GPRTemporary resultPayload(this); GPRTemporary resultTag(this); + + GPRReg baseGPR = base.gpr(); GPRReg indexGPR = index.gpr(); GPRReg resultPayloadGPR = resultPayload.gpr(); GPRReg resultTagGPR = resultTag.gpr(); - - if (!isEmptySpeculation( - m_state.variables().operand( - m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) { - speculationCheck( - ArgumentsEscaped, JSValueRegs(), 0, - m_jit.branch32( - JITCompiler::NotEqual, - JITCompiler::tagFor(m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin)), - TrustedImm32(JSValue::EmptyValueTag))); - } - - m_jit.add32(TrustedImm32(1), indexGPR, resultPayloadGPR); + + MacroAssembler::JumpList slowCases; + ArrayMode mode = node->arrayMode(); + switch (mode.type()) { + case Array::Int32: + case Array::Contiguous: { + ASSERT(!!node->child3()); + StorageOperand storage(this, node->child3()); + GPRTemporary scratch(this); - if (node->codeOrigin.inlineCallFrame) { - speculationCheck( - Uncountable, JSValueRegs(), 0, - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultPayloadGPR, - Imm32(node->codeOrigin.inlineCallFrame->arguments.size()))); - } else { - speculationCheck( - Uncountable, JSValueRegs(), 0, - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultPayloadGPR, - JITCompiler::payloadFor(JSStack::ArgumentCount))); + GPRReg storageGPR = storage.gpr(); + GPRReg scratchGPR = scratch.gpr(); + + slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()))); + m_jit.load32(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag)), scratchGPR); + slowCases.append(m_jit.branch32(MacroAssembler::Equal, scratchGPR, TrustedImm32(JSValue::EmptyValueTag))); + break; } - - JITCompiler::JumpList slowArgument; - JITCompiler::JumpList slowArgumentOutOfBounds; - if (m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) { - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); - const SlowArgument* slowArguments = m_jit.graph().m_slowArguments.get(); - slowArgumentOutOfBounds.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, indexGPR, - Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount()))); + case Array::Double: { + ASSERT(!!node->child3()); + StorageOperand storage(this, node->child3()); + FPRTemporary scratch(this); + FPRReg scratchFPR = scratch.fpr(); + GPRReg storageGPR = storage.gpr(); - COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes); - m_jit.move(ImmPtr(slowArguments), resultPayloadGPR); - m_jit.load32( - JITCompiler::BaseIndex( - resultPayloadGPR, indexGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(SlowArgument, index)), - resultPayloadGPR); + slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()))); + m_jit.loadDouble(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), scratchFPR); + slowCases.append(m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, scratchFPR, scratchFPR)); + break; + } + case Array::ArrayStorage: { + ASSERT(!!node->child3()); + StorageOperand storage(this, node->child3()); + GPRTemporary scratch(this); - m_jit.load32( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), - resultTagGPR); - m_jit.load32( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), - resultPayloadGPR); - slowArgument.append(m_jit.jump()); + GPRReg storageGPR = storage.gpr(); + GPRReg scratchGPR = scratch.gpr(); + + slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset()))); + m_jit.load32(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, ArrayStorage::vectorOffset() + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), scratchGPR); + slowCases.append(m_jit.branch32(MacroAssembler::Equal, scratchGPR, TrustedImm32(JSValue::EmptyValueTag))); + break; } - slowArgumentOutOfBounds.link(&m_jit); - - m_jit.load32( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight, - m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), - resultTagGPR); - m_jit.load32( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight, - m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), - resultPayloadGPR); - - slowArgument.link(&m_jit); - jsValueResult(resultTagGPR, resultPayloadGPR, node); + default: { + slowCases.append(m_jit.jump()); + break; + } + } + + moveTrueTo(resultPayloadGPR); + MacroAssembler::Jump done = m_jit.jump(); + + addSlowPathGenerator(slowPathCall(slowCases, this, operationHasIndexedProperty, JSValueRegs(resultTagGPR, resultPayloadGPR), baseGPR, indexGPR, static_cast<int32_t>(node->internalMethodType()))); + + done.link(&m_jit); + booleanResult(resultPayloadGPR, node); break; } - case GetMyArgumentByValSafe: { - SpeculateStrictInt32Operand index(this, node->child1()); + case GetDirectPname: { + Edge& baseEdge = m_jit.graph().varArgChild(node, 0); + Edge& propertyEdge = m_jit.graph().varArgChild(node, 1); + + SpeculateCellOperand base(this, baseEdge); + SpeculateCellOperand property(this, propertyEdge); + GPRReg baseGPR = base.gpr(); + GPRReg propertyGPR = property.gpr(); + +#if CPU(X86) + GPRFlushedCallResult resultPayload(this); + GPRFlushedCallResult2 resultTag(this); + GPRTemporary scratch(this); + + GPRReg resultTagGPR = resultTag.gpr(); + GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRReg scratchGPR = scratch.gpr(); + + // Not enough registers on X86 for this code, so always use the slow path. + flushRegisters(); + m_jit.move(MacroAssembler::TrustedImm32(JSValue::CellTag), scratchGPR); + callOperation(operationGetByValCell, JSValueRegs(resultTagGPR, resultPayloadGPR), baseGPR, JSValueRegs(scratchGPR, propertyGPR)); + m_jit.exceptionCheck(); +#else GPRTemporary resultPayload(this); GPRTemporary resultTag(this); - GPRReg indexGPR = index.gpr(); - GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRTemporary scratch(this); + GPRReg resultTagGPR = resultTag.gpr(); - - JITCompiler::JumpList slowPath; + GPRReg resultPayloadGPR = resultPayload.gpr(); + GPRReg scratchGPR = scratch.gpr(); + + Edge& indexEdge = m_jit.graph().varArgChild(node, 2); + Edge& enumeratorEdge = m_jit.graph().varArgChild(node, 3); + + SpeculateInt32Operand index(this, indexEdge); + SpeculateCellOperand enumerator(this, enumeratorEdge); + + GPRReg indexGPR = index.gpr(); + GPRReg enumeratorGPR = enumerator.gpr(); + + MacroAssembler::JumpList slowPath; + + // Check the structure + m_jit.load32(MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()), scratchGPR); slowPath.append( m_jit.branch32( - JITCompiler::NotEqual, - JITCompiler::tagFor(m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin)), - TrustedImm32(JSValue::EmptyValueTag))); - - m_jit.add32(TrustedImm32(1), indexGPR, resultPayloadGPR); - if (node->codeOrigin.inlineCallFrame) { - slowPath.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultPayloadGPR, - Imm32(node->codeOrigin.inlineCallFrame->arguments.size()))); - } else { - slowPath.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultPayloadGPR, - JITCompiler::payloadFor(JSStack::ArgumentCount))); - } + MacroAssembler::NotEqual, + scratchGPR, + MacroAssembler::Address( + enumeratorGPR, JSPropertyNameEnumerator::cachedStructureIDOffset()))); - JITCompiler::JumpList slowArgument; - JITCompiler::JumpList slowArgumentOutOfBounds; - if (m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) { - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); - const SlowArgument* slowArguments = m_jit.graph().m_slowArguments.get(); - slowArgumentOutOfBounds.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, indexGPR, - Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount()))); + // Compute the offset + // If index is less than the enumerator's cached inline storage, then it's an inline access + MacroAssembler::Jump outOfLineAccess = m_jit.branch32(MacroAssembler::AboveOrEqual, + indexGPR, MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedInlineCapacityOffset())); - COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes); - m_jit.move(ImmPtr(slowArguments), resultPayloadGPR); - m_jit.load32( - JITCompiler::BaseIndex( - resultPayloadGPR, indexGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(SlowArgument, index)), - resultPayloadGPR); - m_jit.load32( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), - resultTagGPR); - m_jit.load32( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), - resultPayloadGPR); - slowArgument.append(m_jit.jump()); - } - slowArgumentOutOfBounds.link(&m_jit); - - m_jit.load32( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight, - m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), - resultTagGPR); - m_jit.load32( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultPayloadGPR, JITCompiler::TimesEight, - m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), - resultPayloadGPR); - - if (node->codeOrigin.inlineCallFrame) { - addSlowPathGenerator( - slowPathCall( - slowPath, this, operationGetInlinedArgumentByVal, - JSValueRegs(resultTagGPR, resultPayloadGPR), - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin).offset(), - node->codeOrigin.inlineCallFrame, indexGPR)); - } else { - addSlowPathGenerator( - slowPathCall( - slowPath, this, operationGetArgumentByVal, - JSValueRegs(resultTagGPR, resultPayloadGPR), - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin).offset(), - indexGPR)); - } + m_jit.move(indexGPR, scratchGPR); + m_jit.signExtend32ToPtr(scratchGPR, scratchGPR); + m_jit.load32(MacroAssembler::BaseIndex(baseGPR, scratchGPR, MacroAssembler::TimesEight, JSObject::offsetOfInlineStorage() + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTagGPR); + m_jit.load32(MacroAssembler::BaseIndex(baseGPR, scratchGPR, MacroAssembler::TimesEight, JSObject::offsetOfInlineStorage() + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayloadGPR); + + MacroAssembler::Jump done = m_jit.jump(); - slowArgument.link(&m_jit); + // Otherwise it's out of line + outOfLineAccess.link(&m_jit); + m_jit.move(indexGPR, scratchGPR); + m_jit.sub32(MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedInlineCapacityOffset()), scratchGPR); + m_jit.neg32(scratchGPR); + m_jit.signExtend32ToPtr(scratchGPR, scratchGPR); + // We use resultPayloadGPR as a temporary here. We have to make sure clobber it after getting the + // value out of indexGPR and enumeratorGPR because resultPayloadGPR could reuse either of those registers. + m_jit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), resultPayloadGPR); + int32_t offsetOfFirstProperty = static_cast<int32_t>(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue); + m_jit.load32(MacroAssembler::BaseIndex(resultPayloadGPR, scratchGPR, MacroAssembler::TimesEight, offsetOfFirstProperty + OBJECT_OFFSETOF(JSValue, u.asBits.tag)), resultTagGPR); + m_jit.load32(MacroAssembler::BaseIndex(resultPayloadGPR, scratchGPR, MacroAssembler::TimesEight, offsetOfFirstProperty + OBJECT_OFFSETOF(JSValue, u.asBits.payload)), resultPayloadGPR); + + done.link(&m_jit); + + addSlowPathGenerator(slowPathCall(slowPath, this, operationGetByValCell, JSValueRegs(resultTagGPR, resultPayloadGPR), baseGPR, propertyGPR)); +#endif + jsValueResult(resultTagGPR, resultPayloadGPR, node); break; } - - case NewFunctionNoCheck: - compileNewFunctionNoCheck(node); + case GetPropertyEnumerator: { + SpeculateCellOperand base(this, node->child1()); + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + callOperation(operationGetPropertyEnumerator, resultGPR, base.gpr()); + m_jit.exceptionCheck(); + cellResult(resultGPR, node); break; - - case NewFunction: { - JSValueOperand value(this, node->child1()); - GPRTemporary resultTag(this, Reuse, value, TagWord); - GPRTemporary resultPayload(this, Reuse, value, PayloadWord); - - GPRReg valueTagGPR = value.tagGPR(); - GPRReg valuePayloadGPR = value.payloadGPR(); + } + case GetEnumeratorStructurePname: + case GetEnumeratorGenericPname: { + SpeculateCellOperand enumerator(this, node->child1()); + SpeculateInt32Operand index(this, node->child2()); + GPRTemporary scratch(this); + GPRTemporary resultPayload(this); + GPRTemporary resultTag(this); + + GPRReg enumeratorGPR = enumerator.gpr(); + GPRReg indexGPR = index.gpr(); + GPRReg scratchGPR = scratch.gpr(); GPRReg resultTagGPR = resultTag.gpr(); GPRReg resultPayloadGPR = resultPayload.gpr(); - - m_jit.move(valuePayloadGPR, resultPayloadGPR); - m_jit.move(valueTagGPR, resultTagGPR); - - JITCompiler::Jump notCreated = m_jit.branch32(JITCompiler::Equal, valueTagGPR, TrustedImm32(JSValue::EmptyValueTag)); - - addSlowPathGenerator( - slowPathCall( - notCreated, this, operationNewFunction, JSValueRegs(resultTagGPR, resultPayloadGPR), - m_jit.codeBlock()->functionDecl(node->functionDeclIndex()))); - + + MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, indexGPR, + MacroAssembler::Address(enumeratorGPR, (op == GetEnumeratorStructurePname) + ? JSPropertyNameEnumerator::endStructurePropertyIndexOffset() + : JSPropertyNameEnumerator::endGenericPropertyIndexOffset())); + + m_jit.move(MacroAssembler::TrustedImm32(JSValue::NullTag), resultTagGPR); + m_jit.move(MacroAssembler::TrustedImm32(0), resultPayloadGPR); + + MacroAssembler::Jump done = m_jit.jump(); + inBounds.link(&m_jit); + + m_jit.loadPtr(MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedPropertyNamesVectorOffset()), scratchGPR); + m_jit.loadPtr(MacroAssembler::BaseIndex(scratchGPR, indexGPR, MacroAssembler::ScalePtr), resultPayloadGPR); + m_jit.move(MacroAssembler::TrustedImm32(JSValue::CellTag), resultTagGPR); + + done.link(&m_jit); jsValueResult(resultTagGPR, resultPayloadGPR, node); break; } - - case NewFunctionExpression: - compileNewFunctionExpression(node); + case ToIndexString: { + SpeculateInt32Operand index(this, node->child1()); + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + callOperation(operationToIndexString, resultGPR, index.gpr()); + m_jit.exceptionCheck(); + cellResult(resultGPR, node); break; - - case In: - compileIn(node); + } + case ProfileType: { + JSValueOperand value(this, node->child1()); + GPRTemporary scratch1(this); + GPRTemporary scratch2(this); + GPRTemporary scratch3(this); + + GPRReg scratch1GPR = scratch1.gpr(); + GPRReg scratch2GPR = scratch2.gpr(); + GPRReg scratch3GPR = scratch3.gpr(); + + JITCompiler::Jump isTDZValue = m_jit.branch32(JITCompiler::Equal, value.tagGPR(), TrustedImm32(JSValue::EmptyValueTag)); + + // Load the TypeProfilerLog into Scratch2. + TypeProfilerLog* cachedTypeProfilerLog = m_jit.vm()->typeProfilerLog(); + m_jit.move(TrustedImmPtr(cachedTypeProfilerLog), scratch2GPR); + + // Load the next LogEntry into Scratch1. + m_jit.loadPtr(MacroAssembler::Address(scratch2GPR, TypeProfilerLog::currentLogEntryOffset()), scratch1GPR); + + // Store the JSValue onto the log entry. + m_jit.store32(value.tagGPR(), MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::valueOffset() + OBJECT_OFFSETOF(JSValue, u.asBits.tag))); + m_jit.store32(value.payloadGPR(), MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::valueOffset() + OBJECT_OFFSETOF(JSValue, u.asBits.payload))); + + // Store the structureID of the cell if valueGPR is a cell, otherwise, store 0 on the log entry. + MacroAssembler::Jump isNotCell = m_jit.branchIfNotCell(value.jsValueRegs()); + m_jit.load32(MacroAssembler::Address(value.payloadGPR(), JSCell::structureIDOffset()), scratch3GPR); + m_jit.store32(scratch3GPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::structureIDOffset())); + MacroAssembler::Jump skipIsCell = m_jit.jump(); + isNotCell.link(&m_jit); + m_jit.store32(TrustedImm32(0), MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::structureIDOffset())); + skipIsCell.link(&m_jit); + + // Store the typeLocation on the log entry. + TypeLocation* cachedTypeLocation = node->typeLocation(); + m_jit.move(TrustedImmPtr(cachedTypeLocation), scratch3GPR); + m_jit.storePtr(scratch3GPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::locationOffset())); + + // Increment the current log entry. + m_jit.addPtr(TrustedImm32(sizeof(TypeProfilerLog::LogEntry)), scratch1GPR); + m_jit.storePtr(scratch1GPR, MacroAssembler::Address(scratch2GPR, TypeProfilerLog::currentLogEntryOffset())); + MacroAssembler::Jump clearLog = m_jit.branchPtr(MacroAssembler::Equal, scratch1GPR, TrustedImmPtr(cachedTypeProfilerLog->logEndPtr())); + addSlowPathGenerator( + slowPathCall(clearLog, this, operationProcessTypeProfilerLogDFG, NoResult)); + + isTDZValue.link(&m_jit); + + noResult(node); + break; + } + case ProfileControlFlow: { + GPRTemporary scratch1(this); + BasicBlockLocation* basicBlockLocation = node->basicBlockLocation(); + basicBlockLocation->emitExecuteCode(m_jit, scratch1.gpr()); + noResult(node); break; + } - case StoreBarrier: - case ConditionalStoreBarrier: - case StoreBarrierWithNullCheck: { - compileStoreBarrier(node); + case LogShadowChickenPrologue: { + flushRegisters(); + prepareForExternalCall(); + m_jit.emitStoreCodeOrigin(node->origin.semantic); + + GPRTemporary scratch1(this, GPRInfo::nonArgGPR0); // This must be a non-argument GPR. + GPRReg scratch1Reg = scratch1.gpr(); + GPRTemporary shadowPacket(this); + GPRReg shadowPacketReg = shadowPacket.gpr(); + GPRTemporary scratch2(this); + GPRReg scratch2Reg = scratch2.gpr(); + + m_jit.ensureShadowChickenPacket(shadowPacketReg, scratch1Reg, scratch2Reg); + + SpeculateCellOperand scope(this, node->child1()); + GPRReg scopeReg = scope.gpr(); + + m_jit.logShadowChickenProloguePacket(shadowPacketReg, scratch1Reg, scopeReg); + noResult(node); + break; + } + + case LogShadowChickenTail: { + flushRegisters(); + prepareForExternalCall(); + CallSiteIndex callSiteIndex = m_jit.emitStoreCodeOrigin(node->origin.semantic); + + GPRTemporary scratch1(this, GPRInfo::nonArgGPR0); // This must be a non-argument GPR. + GPRReg scratch1Reg = scratch1.gpr(); + GPRTemporary shadowPacket(this); + GPRReg shadowPacketReg = shadowPacket.gpr(); + GPRTemporary scratch2(this); + GPRReg scratch2Reg = scratch2.gpr(); + + m_jit.ensureShadowChickenPacket(shadowPacketReg, scratch1Reg, scratch2Reg); + + JSValueOperand thisValue(this, node->child1()); + JSValueRegs thisRegs = thisValue.jsValueRegs(); + SpeculateCellOperand scope(this, node->child2()); + GPRReg scopeReg = scope.gpr(); + + m_jit.logShadowChickenTailPacket(shadowPacketReg, thisRegs, scopeReg, m_jit.codeBlock(), callSiteIndex); + noResult(node); break; } @@ -4653,49 +5513,102 @@ void SpeculativeJIT::compile(Node* node) emitInvalidationPoint(node); break; - case CheckWatchdogTimer: - speculationCheck( - WatchdogTimerFired, JSValueRegs(), 0, - m_jit.branchTest8( - JITCompiler::NonZero, - JITCompiler::AbsoluteAddress(m_jit.vm()->watchdog.timerDidFireAddress()))); + case CheckWatchdogTimer: { + ASSERT(m_jit.vm()->watchdog()); + GPRTemporary unused(this); + GPRReg unusedGPR = unused.gpr(); + + JITCompiler::Jump timerDidFire = m_jit.branchTest8(JITCompiler::NonZero, + JITCompiler::AbsoluteAddress(m_jit.vm()->watchdog()->timerDidFireAddress())); + + addSlowPathGenerator(slowPathCall(timerDidFire, this, operationHandleWatchdogTimer, unusedGPR)); break; + } case CountExecution: m_jit.add64(TrustedImm32(1), MacroAssembler::AbsoluteAddress(node->executionCounter()->address())); break; case Phantom: + case Check: DFG_NODE_DO_TO_CHILDREN(m_jit.graph(), node, speculate); noResult(node); break; - case Breakpoint: - case ProfileWillCall: - case ProfileDidCall: case PhantomLocal: case LoopHint: // This is a no-op. noResult(node); break; + + case MaterializeNewObject: + compileMaterializeNewObject(node); + break; + + case PutDynamicVar: { + compilePutDynamicVar(node); + break; + } + + case GetDynamicVar: { + compileGetDynamicVar(node); + break; + } + + case ResolveScope: { + compileResolveScope(node); + break; + } + + case CallDOM: + compileCallDOM(node); + break; + + case CallDOMGetter: + compileCallDOMGetter(node); + break; + + case CheckDOM: + compileCheckDOM(node); + break; case Unreachable: - RELEASE_ASSERT_NOT_REACHED(); + unreachable(node); break; case LastNodeType: case Phi: case Upsilon: - case GetArgument: case ExtractOSREntryLocal: case CheckTierUpInLoop: case CheckTierUpAtReturn: case CheckTierUpAndOSREnter: - case Int52ToDouble: - case Int52ToValue: + case Int52Rep: + case FiatInt52: + case Int52Constant: case CheckInBounds: case ArithIMul: - RELEASE_ASSERT_NOT_REACHED(); + case MultiGetByOffset: + case MultiPutByOffset: + case CheckBadCell: + case BottomValue: + case PhantomNewObject: + case PhantomNewFunction: + case PhantomNewGeneratorFunction: + case PhantomNewAsyncFunction: + case PhantomCreateActivation: + case PutHint: + case CheckStructureImmediate: + case MaterializeCreateActivation: + case PutStack: + case KillStack: + case GetStack: + case GetMyArgumentByVal: + case GetMyArgumentByValOutOfBounds: + case PhantomCreateRest: + case PhantomSpread: + case PhantomNewArrayWithSpread: + DFG_CRASH(m_jit.graph(), node, "unexpected node in DFG backend"); break; } @@ -4706,35 +5619,83 @@ void SpeculativeJIT::compile(Node* node) use(node); } -#if ENABLE(GGC) -void SpeculativeJIT::writeBarrier(GPRReg ownerGPR, GPRReg valueTagGPR, Edge valueUse, GPRReg scratch1, GPRReg scratch2) +void SpeculativeJIT::moveTrueTo(GPRReg gpr) { - JITCompiler::Jump isNotCell; - if (!isKnownCell(valueUse.node())) - isNotCell = m_jit.branch32(JITCompiler::NotEqual, valueTagGPR, JITCompiler::TrustedImm32(JSValue::CellTag)); + m_jit.move(TrustedImm32(1), gpr); +} - JITCompiler::Jump definitelyNotMarked = genericWriteBarrier(m_jit, ownerGPR, scratch1, scratch2); - storeToWriteBarrierBuffer(ownerGPR, scratch1, scratch2); - definitelyNotMarked.link(&m_jit); +void SpeculativeJIT::moveFalseTo(GPRReg gpr) +{ + m_jit.move(TrustedImm32(0), gpr); +} - if (!isKnownCell(valueUse.node())) - isNotCell.link(&m_jit); +void SpeculativeJIT::blessBoolean(GPRReg) +{ } -void SpeculativeJIT::writeBarrier(JSCell* owner, GPRReg valueTagGPR, Edge valueUse, GPRReg scratch1, GPRReg scratch2) +void SpeculativeJIT::compileArithRandom(Node* node) { - JITCompiler::Jump isNotCell; - if (!isKnownCell(valueUse.node())) - isNotCell = m_jit.branch32(JITCompiler::NotEqual, valueTagGPR, JITCompiler::TrustedImm32(JSValue::CellTag)); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); - JITCompiler::Jump definitelyNotMarked = genericWriteBarrier(m_jit, owner); - storeToWriteBarrierBuffer(owner, scratch1, scratch2); - definitelyNotMarked.link(&m_jit); + flushRegisters(); - if (!isKnownCell(valueUse.node())) - isNotCell.link(&m_jit); + FPRResult result(this); + callOperation(operationRandom, result.fpr(), globalObject); + // operationRandom does not raise any exception. + doubleResult(result.fpr(), node); +} + +void SpeculativeJIT::emitInitializeButterfly(GPRReg storageGPR, GPRReg sizeGPR, JSValueRegs emptyValueRegs, GPRReg scratchGPR) +{ + m_jit.move(sizeGPR, scratchGPR); + MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, scratchGPR); + MacroAssembler::Label loop = m_jit.label(); + m_jit.sub32(TrustedImm32(1), scratchGPR); + m_jit.store32(emptyValueRegs.tagGPR(), MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag))); + m_jit.store32(emptyValueRegs.payloadGPR(), MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload))); + m_jit.branchTest32(MacroAssembler::NonZero, scratchGPR).linkTo(loop, &m_jit); + done.link(&m_jit); +} + +void SpeculativeJIT::compileAllocateNewArrayWithSize(JSGlobalObject* globalObject, GPRReg resultGPR, GPRReg sizeGPR, IndexingType indexingType, bool shouldConvertLargeSizeToArrayStorage) +{ + GPRTemporary storage(this); + GPRTemporary scratch(this); + GPRTemporary scratch2(this); + + GPRReg storageGPR = storage.gpr(); + GPRReg scratchGPR = scratch.gpr(); + GPRReg scratch2GPR = scratch2.gpr(); + + m_jit.move(TrustedImmPtr(0), storageGPR); + + MacroAssembler::JumpList slowCases; + if (shouldConvertLargeSizeToArrayStorage) + slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH))); + + // We can use result as a scratch for this. + emitAllocateButterfly(storageGPR, sizeGPR, scratchGPR, scratch2GPR, resultGPR, slowCases); + + JSValue hole; + if (hasDouble(indexingType)) + hole = JSValue(JSValue::EncodeAsDouble, PNaN); + else + hole = JSValue(); + JSValueRegs emptyValueRegs(scratchGPR, scratch2GPR); + m_jit.move(TrustedImm32(hole.tag()), emptyValueRegs.tagGPR()); + m_jit.move(TrustedImm32(hole.payload()), emptyValueRegs.payloadGPR()); + // We can use result as a scratch for this. + emitInitializeButterfly(storageGPR, sizeGPR, emptyValueRegs, resultGPR); + + RegisteredStructure structure = m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType)); + emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases); + + addSlowPathGenerator(std::make_unique<CallArrayAllocatorWithVariableSizeSlowPathGenerator>( + slowCases, this, operationNewArrayWithSize, resultGPR, + structure, + shouldConvertLargeSizeToArrayStorage ? m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)) : structure, + sizeGPR, storageGPR)); } -#endif // ENABLE(GGC) #endif |