diff options
Diffstat (limited to 'Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp')
| -rw-r--r-- | Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp | 4740 |
1 files changed, 2830 insertions, 1910 deletions
diff --git a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp index ea9f88613..3393cab39 100644 --- a/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp +++ b/Source/JavaScriptCore/dfg/DFGSpeculativeJIT64.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011, 2012, 2013, 2014 Apple Inc. All rights reserved. + * Copyright (C) 2011-2017 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -28,15 +28,27 @@ #if ENABLE(DFG_JIT) -#include "Arguments.h" #include "ArrayPrototype.h" +#include "CallFrameShuffler.h" #include "DFGAbstractInterpreterInlines.h" #include "DFGCallArrayAllocatorSlowPathGenerator.h" #include "DFGOperations.h" #include "DFGSlowPathGenerator.h" -#include "Debugger.h" -#include "JSCJSValueInlines.h" +#include "DirectArguments.h" +#include "GetterSetter.h" +#include "HasOwnPropertyCache.h" +#include "JSCInlines.h" +#include "JSEnvironmentRecord.h" +#include "JSLexicalEnvironment.h" +#include "JSMap.h" +#include "JSPropertyNameEnumerator.h" +#include "JSSet.h" #include "ObjectPrototype.h" +#include "SetupVarargsFrame.h" +#include "SpillRegistersMode.h" +#include "StringPrototype.h" +#include "TypeProfilerLog.h" +#include "Watchdog.h" namespace JSC { namespace DFG { @@ -78,21 +90,9 @@ GPRReg SpeculativeJIT::fillJSValue(Edge edge) GPRReg gpr = allocate(); if (edge->hasConstant()) { - if (isInt32Constant(edge.node())) { - info.fillJSValue(*m_stream, gpr, DataFormatJSInt32); - JSValue jsValue = jsNumber(valueOfInt32Constant(edge.node())); - m_jit.move(MacroAssembler::Imm64(JSValue::encode(jsValue)), gpr); - } else if (isNumberConstant(edge.node())) { - info.fillJSValue(*m_stream, gpr, DataFormatJSDouble); - JSValue jsValue(JSValue::EncodeAsDouble, valueOfNumberConstant(edge.node())); - m_jit.move(MacroAssembler::Imm64(JSValue::encode(jsValue)), gpr); - } else { - ASSERT(isJSConstant(edge.node())); - JSValue jsValue = valueOfJSConstant(edge.node()); - m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); - info.fillJSValue(*m_stream, gpr, DataFormatJS); - } - + JSValue jsValue = edge->asJSValue(); + m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); + info.fillJSValue(*m_stream, gpr, DataFormatJS); m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); } else { DataFormat spillFormat = info.spillFormat(); @@ -105,21 +105,9 @@ GPRReg SpeculativeJIT::fillJSValue(Edge edge) break; } - case DataFormatInt52: - case DataFormatStrictInt52: { - m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - boxInt52(gpr, gpr, spillFormat); - return gpr; - } - default: m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - if (spillFormat == DataFormatDouble) { - // Need to box the double, since we want a JSValue. - m_jit.sub64(GPRInfo::tagTypeNumberRegister, gpr); - spillFormat = DataFormatJSDouble; - } else - RELEASE_ASSERT(spillFormat & DataFormatJS); + DFG_ASSERT(m_jit.graph(), m_currentNode, spillFormat & DataFormatJS); break; } info.fillJSValue(*m_stream, gpr, spillFormat); @@ -142,28 +130,6 @@ GPRReg SpeculativeJIT::fillJSValue(Edge edge) return gpr; } - case DataFormatDouble: { - FPRReg fpr = info.fpr(); - GPRReg gpr = boxDouble(fpr); - - // Update all info - info.fillJSValue(*m_stream, gpr, DataFormatJSDouble); - m_fprs.release(fpr); - m_gprs.retain(gpr, virtualRegister, SpillOrderJS); - - return gpr; - } - - case DataFormatInt52: - case DataFormatStrictInt52: { - GPRReg gpr = info.gpr(); - lock(gpr); - GPRReg resultGPR = allocate(); - boxInt52(gpr, resultGPR, info.registerFormat()); - unlock(gpr); - return resultGPR; - } - case DataFormatCell: // No retag required on JSVALUE64! case DataFormatJS: @@ -178,20 +144,34 @@ GPRReg SpeculativeJIT::fillJSValue(Edge edge) case DataFormatBoolean: case DataFormatStorage: + case DataFormatDouble: + case DataFormatInt52: // this type currently never occurs - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Corrupt data format"); return InvalidGPRReg; } } -void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg resultGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode) +void SpeculativeJIT::cachedGetById(CodeOrigin origin, JSValueRegs base, JSValueRegs result, unsigned identifierNumber, JITCompiler::Jump slowPathTarget , SpillRegistersMode mode, AccessType type) { + cachedGetById(origin, base.gpr(), result.gpr(), identifierNumber, slowPathTarget, mode, type); +} + +void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg resultGPR, unsigned identifierNumber, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode, AccessType type) +{ + CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(codeOrigin, m_stream->size()); + RegisterSet usedRegisters = this->usedRegisters(); + if (spillMode == DontSpill) { + // We've already flushed registers to the stack, we don't need to spill these. + usedRegisters.set(baseGPR, false); + usedRegisters.set(resultGPR, false); + } JITGetByIdGenerator gen( - m_jit.codeBlock(), codeOrigin, usedRegisters(), GPRInfo::callFrameRegister, - JSValueRegs(baseGPR), JSValueRegs(resultGPR), spillMode != NeedToSpill); + m_jit.codeBlock(), codeOrigin, callSite, usedRegisters, identifierUID(identifierNumber), + JSValueRegs(baseGPR), JSValueRegs(resultGPR), type); gen.generateFastPath(m_jit); JITCompiler::JumpList slowCases; @@ -199,20 +179,29 @@ void SpeculativeJIT::cachedGetById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg slowCases.append(slowPathTarget); slowCases.append(gen.slowPathJump()); - OwnPtr<SlowPathGenerator> slowPath = slowPathCall( - slowCases, this, operationGetByIdOptimize, resultGPR, gen.stubInfo(), baseGPR, - identifierUID(identifierNumber), spillMode); + auto slowPath = slowPathCall( + slowCases, this, type == AccessType::Get ? operationGetByIdOptimize : operationTryGetByIdOptimize, + spillMode, ExceptionCheckRequirement::CheckNeeded, + resultGPR, gen.stubInfo(), baseGPR, identifierUID(identifierNumber)); m_jit.addGetById(gen, slowPath.get()); - addSlowPathGenerator(slowPath.release()); + addSlowPathGenerator(WTFMove(slowPath)); } -void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget) +void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg valueGPR, GPRReg scratchGPR, unsigned identifierNumber, PutKind putKind, JITCompiler::Jump slowPathTarget, SpillRegistersMode spillMode) { + CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(codeOrigin, m_stream->size()); + RegisterSet usedRegisters = this->usedRegisters(); + if (spillMode == DontSpill) { + // We've already flushed registers to the stack, we don't need to spill these. + usedRegisters.set(baseGPR, false); + usedRegisters.set(valueGPR, false); + } + JITPutByIdGenerator gen( - m_jit.codeBlock(), codeOrigin, usedRegisters(), GPRInfo::callFrameRegister, - JSValueRegs(baseGPR), JSValueRegs(valueGPR), scratchGPR, false, - m_jit.ecmaModeFor(codeOrigin), putKind); + m_jit.codeBlock(), codeOrigin, callSite, usedRegisters, JSValueRegs(baseGPR), + JSValueRegs(valueGPR), scratchGPR, m_jit.ecmaModeFor(codeOrigin), putKind); + gen.generateFastPath(m_jit); JITCompiler::JumpList slowCases; @@ -220,152 +209,132 @@ void SpeculativeJIT::cachedPutById(CodeOrigin codeOrigin, GPRReg baseGPR, GPRReg slowCases.append(slowPathTarget); slowCases.append(gen.slowPathJump()); - OwnPtr<SlowPathGenerator> slowPath = slowPathCall( + auto slowPath = slowPathCall( slowCases, this, gen.slowPathFunction(), NoResult, gen.stubInfo(), valueGPR, baseGPR, identifierUID(identifierNumber)); m_jit.addPutById(gen, slowPath.get()); - addSlowPathGenerator(slowPath.release()); + addSlowPathGenerator(WTFMove(slowPath)); } -void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNull(Edge operand, bool invert) +void SpeculativeJIT::nonSpeculativeNonPeepholeCompareNullOrUndefined(Edge operand) { - JSValueOperand arg(this, operand); + ASSERT_WITH_MESSAGE(!masqueradesAsUndefinedWatchpointIsStillValid() || !isKnownCell(operand.node()), "The Compare should have been eliminated, it is known to be always false."); + + JSValueOperand arg(this, operand, ManualOperandSpeculation); GPRReg argGPR = arg.gpr(); - GPRTemporary result(this, Reuse, arg); + GPRTemporary result(this); GPRReg resultGPR = result.gpr(); - - JITCompiler::Jump notCell; - - JITCompiler::Jump notMasqueradesAsUndefined; - if (masqueradesAsUndefinedWatchpointIsStillValid()) { - if (!isKnownCell(operand.node())) - notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister); - m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultGPR); - notMasqueradesAsUndefined = m_jit.jump(); + m_jit.move(TrustedImm32(0), resultGPR); + + JITCompiler::JumpList done; + if (masqueradesAsUndefinedWatchpointIsStillValid()) { + if (!isKnownNotCell(operand.node())) + done.append(m_jit.branchIfCell(JSValueRegs(argGPR))); } else { GPRTemporary localGlobalObject(this); GPRTemporary remoteGlobalObject(this); + GPRTemporary scratch(this); + JITCompiler::Jump notCell; if (!isKnownCell(operand.node())) - notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister); - - m_jit.loadPtr(JITCompiler::Address(argGPR, JSCell::structureOffset()), resultGPR); - JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(resultGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined)); - - m_jit.move(invert ? TrustedImm32(1) : TrustedImm32(0), resultGPR); - notMasqueradesAsUndefined = m_jit.jump(); + notCell = m_jit.branchIfNotCell(JSValueRegs(argGPR)); + + JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8( + JITCompiler::Zero, + JITCompiler::Address(argGPR, JSCell::typeInfoFlagsOffset()), + JITCompiler::TrustedImm32(MasqueradesAsUndefined)); + done.append(isNotMasqueradesAsUndefined); - isMasqueradesAsUndefined.link(&m_jit); GPRReg localGlobalObjectGPR = localGlobalObject.gpr(); GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr(); - m_jit.move(JITCompiler::TrustedImmPtr(m_jit.graph().globalObjectFor(operand->codeOrigin)), localGlobalObjectGPR); + m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)), localGlobalObjectGPR); + m_jit.emitLoadStructure(argGPR, resultGPR, scratch.gpr()); m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR); - m_jit.comparePtr(invert ? JITCompiler::NotEqual : JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, resultGPR); + m_jit.comparePtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, resultGPR); + done.append(m_jit.jump()); + if (!isKnownCell(operand.node())) + notCell.link(&m_jit); } - if (!isKnownCell(operand.node())) { - JITCompiler::Jump done = m_jit.jump(); - - notCell.link(&m_jit); - + if (!isKnownNotOther(operand.node())) { m_jit.move(argGPR, resultGPR); m_jit.and64(JITCompiler::TrustedImm32(~TagBitUndefined), resultGPR); - m_jit.compare64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm32(ValueNull), resultGPR); - - done.link(&m_jit); + m_jit.compare64(JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm32(ValueNull), resultGPR); } - - notMasqueradesAsUndefined.link(&m_jit); + + done.link(&m_jit); m_jit.or32(TrustedImm32(ValueFalse), resultGPR); jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean); } -void SpeculativeJIT::nonSpeculativePeepholeBranchNull(Edge operand, Node* branchNode, bool invert) +void SpeculativeJIT::nonSpeculativePeepholeBranchNullOrUndefined(Edge operand, Node* branchNode) { - BasicBlock* taken = branchNode->takenBlock(); - BasicBlock* notTaken = branchNode->notTakenBlock(); - - if (taken == nextBlock()) { - invert = !invert; - BasicBlock* tmp = taken; - taken = notTaken; - notTaken = tmp; - } + ASSERT_WITH_MESSAGE(!masqueradesAsUndefinedWatchpointIsStillValid() || !isKnownCell(operand.node()), "The Compare should have been eliminated, it is known to be always false."); + + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; - JSValueOperand arg(this, operand); + JSValueOperand arg(this, operand, ManualOperandSpeculation); GPRReg argGPR = arg.gpr(); GPRTemporary result(this, Reuse, arg); GPRReg resultGPR = result.gpr(); - - JITCompiler::Jump notCell; - - if (masqueradesAsUndefinedWatchpointIsStillValid()) { - if (!isKnownCell(operand.node())) - notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister); - jump(invert ? taken : notTaken, ForceJump); + // First, handle the case where "operand" is a cell. + if (masqueradesAsUndefinedWatchpointIsStillValid()) { + if (!isKnownNotCell(operand.node())) { + JITCompiler::Jump isCell = m_jit.branchIfCell(JSValueRegs(argGPR)); + addBranch(isCell, notTaken); + } } else { GPRTemporary localGlobalObject(this); GPRTemporary remoteGlobalObject(this); + GPRTemporary scratch(this); + JITCompiler::Jump notCell; if (!isKnownCell(operand.node())) - notCell = m_jit.branchTest64(MacroAssembler::NonZero, argGPR, GPRInfo::tagMaskRegister); - - m_jit.loadPtr(JITCompiler::Address(argGPR, JSCell::structureOffset()), resultGPR); - branchTest8(JITCompiler::Zero, JITCompiler::Address(resultGPR, Structure::typeInfoFlagsOffset()), JITCompiler::TrustedImm32(MasqueradesAsUndefined), invert ? taken : notTaken); + notCell = m_jit.branchIfNotCell(JSValueRegs(argGPR)); + + branchTest8(JITCompiler::Zero, + JITCompiler::Address(argGPR, JSCell::typeInfoFlagsOffset()), + JITCompiler::TrustedImm32(MasqueradesAsUndefined), notTaken); GPRReg localGlobalObjectGPR = localGlobalObject.gpr(); GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr(); - m_jit.move(TrustedImmPtr(m_jit.graph().globalObjectFor(operand->codeOrigin)), localGlobalObjectGPR); + m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)), localGlobalObjectGPR); + m_jit.emitLoadStructure(argGPR, resultGPR, scratch.gpr()); m_jit.loadPtr(JITCompiler::Address(resultGPR, Structure::globalObjectOffset()), remoteGlobalObjectGPR); - branchPtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, invert ? notTaken : taken); + branchPtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, taken); + + if (!isKnownCell(operand.node())) { + jump(notTaken, ForceJump); + notCell.link(&m_jit); + } } - - if (!isKnownCell(operand.node())) { - jump(notTaken, ForceJump); - - notCell.link(&m_jit); - + + if (isKnownNotOther(operand.node())) + jump(notTaken); + else { + JITCompiler::RelationalCondition condition = JITCompiler::Equal; + if (taken == nextBlock()) { + condition = JITCompiler::NotEqual; + std::swap(taken, notTaken); + } m_jit.move(argGPR, resultGPR); m_jit.and64(JITCompiler::TrustedImm32(~TagBitUndefined), resultGPR); - branch64(invert ? JITCompiler::NotEqual : JITCompiler::Equal, resultGPR, JITCompiler::TrustedImm64(ValueNull), taken); - } - - jump(notTaken); -} - -bool SpeculativeJIT::nonSpeculativeCompareNull(Node* node, Edge operand, bool invert) -{ - unsigned branchIndexInBlock = detectPeepHoleBranch(); - if (branchIndexInBlock != UINT_MAX) { - Node* branchNode = m_block->at(branchIndexInBlock); - - RELEASE_ASSERT(node->adjustedRefCount() == 1); - - nonSpeculativePeepholeBranchNull(operand, branchNode, invert); - - use(node->child1()); - use(node->child2()); - m_indexInBlock = branchIndexInBlock; - m_currentNode = branchNode; - - return true; + branch64(condition, resultGPR, JITCompiler::TrustedImm64(ValueNull), taken); + jump(notTaken); } - - nonSpeculativeNonPeepholeCompareNull(operand, invert); - - return false; } void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, MacroAssembler::RelationalCondition cond, S_JITOperation_EJJ helperFunction) { - BasicBlock* taken = branchNode->takenBlock(); - BasicBlock* notTaken = branchNode->notTakenBlock(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; JITCompiler::ResultCondition callResultCondition = JITCompiler::NonZero; @@ -387,7 +356,7 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, JITCompiler::JumpList slowPath; if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) { - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); arg1.use(); @@ -395,6 +364,7 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, flushRegisters(); callOperation(helperFunction, resultGPR, arg1GPR, arg2GPR); + m_jit.exceptionCheck(); branchTest32(callResultCondition, resultGPR, taken); } else { @@ -419,6 +389,7 @@ void SpeculativeJIT::nonSpeculativePeepholeBranch(Node* node, Node* branchNode, silentSpillAllRegisters(resultGPR); callOperation(helperFunction, resultGPR, arg1GPR, arg2GPR); silentFillAllRegisters(resultGPR); + m_jit.exceptionCheck(); branchTest32(callResultCondition, resultGPR, taken); } @@ -438,14 +409,14 @@ public: JumpType from, SpeculativeJIT* jit, S_JITOperation_EJJ function, GPRReg result, GPRReg arg1, GPRReg arg2) : CallSlowPathGenerator<JumpType, S_JITOperation_EJJ, GPRReg>( - from, jit, function, NeedToSpill, result) + from, jit, function, NeedToSpill, ExceptionCheckRequirement::CheckNeeded, result) , m_arg1(arg1) , m_arg2(arg2) { } protected: - virtual void generateInternal(SpeculativeJIT* jit) override + void generateInternal(SpeculativeJIT* jit) override { this->setUp(jit); this->recordCall(jit->callOperation(this->m_function, this->m_result, m_arg1, m_arg2)); @@ -470,7 +441,7 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler JITCompiler::JumpList slowPath; if (isKnownNotInteger(node->child1().node()) || isKnownNotInteger(node->child2().node())) { - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); arg1.use(); @@ -478,6 +449,7 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler flushRegisters(); callOperation(helperFunction, resultGPR, arg1GPR, arg2GPR); + m_jit.exceptionCheck(); m_jit.or32(TrustedImm32(ValueFalse), resultGPR); jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly); @@ -497,9 +469,8 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler m_jit.or32(TrustedImm32(ValueFalse), resultGPR); if (!isKnownInteger(node->child1().node()) || !isKnownInteger(node->child2().node())) { - addSlowPathGenerator(adoptPtr( - new CompareAndBoxBooleanSlowPathGenerator<JITCompiler::JumpList>( - slowPath, this, helperFunction, resultGPR, arg1GPR, arg2GPR))); + addSlowPathGenerator(std::make_unique<CompareAndBoxBooleanSlowPathGenerator<JITCompiler::JumpList>>( + slowPath, this, helperFunction, resultGPR, arg1GPR, arg2GPR)); } jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly); @@ -508,8 +479,8 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeCompare(Node* node, MacroAssembler void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode, bool invert) { - BasicBlock* taken = branchNode->takenBlock(); - BasicBlock* notTaken = branchNode->notTakenBlock(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; // The branch instruction will branch to the taken block. // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through. @@ -539,6 +510,7 @@ void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode silentSpillAllRegisters(resultGPR); callOperation(operationCompareStrictEqCell, resultGPR, arg1GPR, arg2GPR); silentFillAllRegisters(resultGPR); + m_jit.exceptionCheck(); branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultGPR, taken); } else { @@ -565,6 +537,7 @@ void SpeculativeJIT::nonSpeculativePeepholeStrictEq(Node* node, Node* branchNode silentSpillAllRegisters(resultGPR); callOperation(operationCompareStrictEq, resultGPR, arg1GPR, arg2GPR); silentFillAllRegisters(resultGPR); + m_jit.exceptionCheck(); branchTest32(invert ? JITCompiler::Zero : JITCompiler::NonZero, resultGPR, taken); } @@ -600,6 +573,7 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert) silentSpillAllRegisters(resultGPR); callOperation(operationCompareStrictEqCell, resultGPR, arg1GPR, arg2GPR); silentFillAllRegisters(resultGPR); + m_jit.exceptionCheck(); m_jit.and64(JITCompiler::TrustedImm32(1), resultGPR); m_jit.or32(JITCompiler::TrustedImm32(ValueFalse), resultGPR); @@ -629,11 +603,9 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert) m_jit.move(JITCompiler::TrustedImm64(JSValue::encode(jsBoolean(!invert))), resultGPR); - addSlowPathGenerator( - adoptPtr( - new CompareAndBoxBooleanSlowPathGenerator<MacroAssembler::JumpList>( + addSlowPathGenerator(std::make_unique<CompareAndBoxBooleanSlowPathGenerator<MacroAssembler::JumpList>>( slowPathCases, this, operationCompareStrictEq, resultGPR, arg1GPR, - arg2GPR))); + arg2GPR)); done.link(&m_jit); } @@ -641,74 +613,436 @@ void SpeculativeJIT::nonSpeculativeNonPeepholeStrictEq(Node* node, bool invert) jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean, UseChildrenCalledExplicitly); } -void SpeculativeJIT::emitCall(Node* node) +void SpeculativeJIT::compileMiscStrictEq(Node* node) { - if (node->op() != Call) - RELEASE_ASSERT(node->op() == Construct); - - // For constructors, the this argument is not passed but we have to make space - // for it. - int dummyThisArgument = node->op() == Call ? 0 : 1; - - CallLinkInfo::CallType callType = node->op() == Call ? CallLinkInfo::Call : CallLinkInfo::Construct; + JSValueOperand op1(this, node->child1(), ManualOperandSpeculation); + JSValueOperand op2(this, node->child2(), ManualOperandSpeculation); + GPRTemporary result(this); - Edge calleeEdge = m_jit.graph().m_varArgChildren[node->firstChild()]; - JSValueOperand callee(this, calleeEdge); - GPRReg calleeGPR = callee.gpr(); - use(calleeEdge); + if (node->child1().useKind() == MiscUse) + speculateMisc(node->child1(), op1.jsValueRegs()); + if (node->child2().useKind() == MiscUse) + speculateMisc(node->child2(), op2.jsValueRegs()); - // The call instruction's first child is the function; the subsequent children are the - // arguments. - int numPassedArgs = node->numChildren() - 1; + m_jit.compare64(JITCompiler::Equal, op1.gpr(), op2.gpr(), result.gpr()); + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + jsValueResult(result.gpr(), node, DataFormatJSBoolean); +} + +void SpeculativeJIT::emitCall(Node* node) +{ + CallLinkInfo::CallType callType; + bool isVarargs = false; + bool isForwardVarargs = false; + bool isTail = false; + bool isEmulatedTail = false; + bool isDirect = false; + switch (node->op()) { + case Call: + case CallEval: + callType = CallLinkInfo::Call; + break; + case TailCall: + callType = CallLinkInfo::TailCall; + isTail = true; + break; + case TailCallInlinedCaller: + callType = CallLinkInfo::Call; + isEmulatedTail = true; + break; + case Construct: + callType = CallLinkInfo::Construct; + break; + case CallVarargs: + callType = CallLinkInfo::CallVarargs; + isVarargs = true; + break; + case TailCallVarargs: + callType = CallLinkInfo::TailCallVarargs; + isVarargs = true; + isTail = true; + break; + case TailCallVarargsInlinedCaller: + callType = CallLinkInfo::CallVarargs; + isVarargs = true; + isEmulatedTail = true; + break; + case ConstructVarargs: + callType = CallLinkInfo::ConstructVarargs; + isVarargs = true; + break; + case CallForwardVarargs: + callType = CallLinkInfo::CallVarargs; + isForwardVarargs = true; + break; + case ConstructForwardVarargs: + callType = CallLinkInfo::ConstructVarargs; + isForwardVarargs = true; + break; + case TailCallForwardVarargs: + callType = CallLinkInfo::TailCallVarargs; + isTail = true; + isForwardVarargs = true; + break; + case TailCallForwardVarargsInlinedCaller: + callType = CallLinkInfo::CallVarargs; + isEmulatedTail = true; + isForwardVarargs = true; + break; + case DirectCall: + callType = CallLinkInfo::DirectCall; + isDirect = true; + break; + case DirectConstruct: + callType = CallLinkInfo::DirectConstruct; + isDirect = true; + break; + case DirectTailCall: + callType = CallLinkInfo::DirectTailCall; + isTail = true; + isDirect = true; + break; + case DirectTailCallInlinedCaller: + callType = CallLinkInfo::DirectCall; + isEmulatedTail = true; + isDirect = true; + break; + default: + DFG_CRASH(m_jit.graph(), node, "bad node type"); + break; + } + + GPRReg calleeGPR = InvalidGPRReg; + CallFrameShuffleData shuffleData; - int numArgs = numPassedArgs + dummyThisArgument; + ExecutableBase* executable = nullptr; + FunctionExecutable* functionExecutable = nullptr; + if (isDirect) { + executable = node->castOperand<ExecutableBase*>(); + functionExecutable = jsDynamicCast<FunctionExecutable*>(*m_jit.vm(), executable); + } - m_jit.store32(MacroAssembler::TrustedImm32(numArgs), calleeFramePayloadSlot(numArgs, JSStack::ArgumentCount)); - m_jit.store64(GPRInfo::callFrameRegister, calleeFrameCallerFrame(numArgs)); - m_jit.store64(calleeGPR, calleeFrameSlot(numArgs, JSStack::Callee)); + unsigned numPassedArgs = 0; + unsigned numAllocatedArgs = 0; - for (int i = 0; i < numPassedArgs; i++) { - Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i]; - JSValueOperand arg(this, argEdge); - GPRReg argGPR = arg.gpr(); - use(argEdge); + // Gotta load the arguments somehow. Varargs is trickier. + if (isVarargs || isForwardVarargs) { + RELEASE_ASSERT(!isDirect); + CallVarargsData* data = node->callVarargsData(); + + unsigned numUsedStackSlots = m_jit.graph().m_nextMachineLocal; + + if (isForwardVarargs) { + flushRegisters(); + if (node->child3()) + use(node->child3()); + + GPRReg scratchGPR1; + GPRReg scratchGPR2; + GPRReg scratchGPR3; + + scratchGPR1 = JITCompiler::selectScratchGPR(); + scratchGPR2 = JITCompiler::selectScratchGPR(scratchGPR1); + scratchGPR3 = JITCompiler::selectScratchGPR(scratchGPR1, scratchGPR2); + + m_jit.move(TrustedImm32(numUsedStackSlots), scratchGPR2); + JITCompiler::JumpList slowCase; + InlineCallFrame* inlineCallFrame; + if (node->child3()) + inlineCallFrame = node->child3()->origin.semantic.inlineCallFrame; + else + inlineCallFrame = node->origin.semantic.inlineCallFrame; + // emitSetupVarargsFrameFastCase modifies the stack pointer if it succeeds. + emitSetupVarargsFrameFastCase(m_jit, scratchGPR2, scratchGPR1, scratchGPR2, scratchGPR3, inlineCallFrame, data->firstVarArgOffset, slowCase); + JITCompiler::Jump done = m_jit.jump(); + slowCase.link(&m_jit); + callOperation(operationThrowStackOverflowForVarargs); + m_jit.exceptionCheck(); + m_jit.abortWithReason(DFGVarargsThrowingPathDidNotThrow); + done.link(&m_jit); + } else { + GPRReg argumentsGPR; + GPRReg scratchGPR1; + GPRReg scratchGPR2; + GPRReg scratchGPR3; + + auto loadArgumentsGPR = [&] (GPRReg reservedGPR) { + if (reservedGPR != InvalidGPRReg) + lock(reservedGPR); + JSValueOperand arguments(this, node->child3()); + argumentsGPR = arguments.gpr(); + if (reservedGPR != InvalidGPRReg) + unlock(reservedGPR); + flushRegisters(); + + scratchGPR1 = JITCompiler::selectScratchGPR(argumentsGPR, reservedGPR); + scratchGPR2 = JITCompiler::selectScratchGPR(argumentsGPR, scratchGPR1, reservedGPR); + scratchGPR3 = JITCompiler::selectScratchGPR(argumentsGPR, scratchGPR1, scratchGPR2, reservedGPR); + }; + + loadArgumentsGPR(InvalidGPRReg); + + DFG_ASSERT(m_jit.graph(), node, isFlushed()); + + // Right now, arguments is in argumentsGPR and the register file is flushed. + callOperation(operationSizeFrameForVarargs, GPRInfo::returnValueGPR, argumentsGPR, numUsedStackSlots, data->firstVarArgOffset); + m_jit.exceptionCheck(); + + // Now we have the argument count of the callee frame, but we've lost the arguments operand. + // Reconstruct the arguments operand while preserving the callee frame. + loadArgumentsGPR(GPRInfo::returnValueGPR); + m_jit.move(TrustedImm32(numUsedStackSlots), scratchGPR1); + emitSetVarargsFrame(m_jit, GPRInfo::returnValueGPR, false, scratchGPR1, scratchGPR1); + m_jit.addPtr(TrustedImm32(-(sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(void*)))), scratchGPR1, JITCompiler::stackPointerRegister); + + callOperation(operationSetupVarargsFrame, GPRInfo::returnValueGPR, scratchGPR1, argumentsGPR, data->firstVarArgOffset, GPRInfo::returnValueGPR); + m_jit.exceptionCheck(); + m_jit.addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), GPRInfo::returnValueGPR, JITCompiler::stackPointerRegister); + } + + DFG_ASSERT(m_jit.graph(), node, isFlushed()); + + // We don't need the arguments array anymore. + if (isVarargs) + use(node->child3()); + + // Now set up the "this" argument. + JSValueOperand thisArgument(this, node->child2()); + GPRReg thisArgumentGPR = thisArgument.gpr(); + thisArgument.use(); - m_jit.store64(argGPR, calleeArgumentSlot(numArgs, i + dummyThisArgument)); + m_jit.store64(thisArgumentGPR, JITCompiler::calleeArgumentSlot(0)); + } else { + // The call instruction's first child is the function; the subsequent children are the + // arguments. + numPassedArgs = node->numChildren() - 1; + numAllocatedArgs = numPassedArgs; + + if (functionExecutable) { + // Allocate more args if this would let us avoid arity checks. This is throttled by + // CallLinkInfo's limit. It's probably good to throttle it - if the callee wants a + // ginormous amount of argument space then it's better for them to do it so that when we + // make calls to other things, we don't waste space. + unsigned desiredNumAllocatedArgs = static_cast<unsigned>(functionExecutable->parameterCount()) + 1; + if (desiredNumAllocatedArgs <= Options::maximumDirectCallStackSize()) { + numAllocatedArgs = std::max(numAllocatedArgs, desiredNumAllocatedArgs); + + // Whoever converts to DirectCall should do this adjustment. It's too late for us to + // do this adjustment now since we will have already emitted code that relied on the + // value of m_parameterSlots. + DFG_ASSERT( + m_jit.graph(), node, + Graph::parameterSlotsForArgCount(numAllocatedArgs) + <= m_jit.graph().m_parameterSlots); + } + } + + if (isTail) { + Edge calleeEdge = m_jit.graph().child(node, 0); + JSValueOperand callee(this, calleeEdge); + calleeGPR = callee.gpr(); + if (!isDirect) + callee.use(); + + shuffleData.tagTypeNumber = GPRInfo::tagTypeNumberRegister; + shuffleData.numLocals = m_jit.graph().frameRegisterCount(); + shuffleData.callee = ValueRecovery::inGPR(calleeGPR, DataFormatJS); + shuffleData.args.resize(numAllocatedArgs); + shuffleData.numPassedArgs = numPassedArgs; + + for (unsigned i = 0; i < numPassedArgs; ++i) { + Edge argEdge = m_jit.graph().varArgChild(node, i + 1); + GenerationInfo& info = generationInfo(argEdge.node()); + if (!isDirect) + use(argEdge); + shuffleData.args[i] = info.recovery(argEdge->virtualRegister()); + } + + for (unsigned i = numPassedArgs; i < numAllocatedArgs; ++i) + shuffleData.args[i] = ValueRecovery::constant(jsUndefined()); + + shuffleData.setupCalleeSaveRegisters(m_jit.codeBlock()); + } else { + m_jit.store32(MacroAssembler::TrustedImm32(numPassedArgs), JITCompiler::calleeFramePayloadSlot(CallFrameSlot::argumentCount)); + + for (unsigned i = 0; i < numPassedArgs; i++) { + Edge argEdge = m_jit.graph().m_varArgChildren[node->firstChild() + 1 + i]; + JSValueOperand arg(this, argEdge); + GPRReg argGPR = arg.gpr(); + use(argEdge); + + m_jit.store64(argGPR, JITCompiler::calleeArgumentSlot(i)); + } + + for (unsigned i = numPassedArgs; i < numAllocatedArgs; ++i) + m_jit.storeTrustedValue(jsUndefined(), JITCompiler::calleeArgumentSlot(i)); + } } + + if (!isTail || isVarargs || isForwardVarargs) { + Edge calleeEdge = m_jit.graph().child(node, 0); + JSValueOperand callee(this, calleeEdge); + calleeGPR = callee.gpr(); + callee.use(); + m_jit.store64(calleeGPR, JITCompiler::calleeFrameSlot(CallFrameSlot::callee)); - flushRegisters(); + flushRegisters(); + } - GPRResult result(this); - GPRReg resultGPR = result.gpr(); + CodeOrigin staticOrigin = node->origin.semantic; + ASSERT(!isTail || !staticOrigin.inlineCallFrame || !staticOrigin.inlineCallFrame->getCallerSkippingTailCalls()); + ASSERT(!isEmulatedTail || (staticOrigin.inlineCallFrame && staticOrigin.inlineCallFrame->getCallerSkippingTailCalls())); + CodeOrigin dynamicOrigin = + isEmulatedTail ? *staticOrigin.inlineCallFrame->getCallerSkippingTailCalls() : staticOrigin; - JITCompiler::DataLabelPtr targetToCheck; - JITCompiler::JumpList slowPath; + CallSiteIndex callSite = m_jit.recordCallSiteAndGenerateExceptionHandlingOSRExitIfNeeded(dynamicOrigin, m_stream->size()); + + auto setResultAndResetStack = [&] () { + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + m_jit.move(GPRInfo::returnValueGPR, resultGPR); + + jsValueResult(resultGPR, m_currentNode, DataFormatJS, UseChildrenCalledExplicitly); - m_jit.emitStoreCodeOrigin(node->codeOrigin); + // After the calls are done, we need to reestablish our stack + // pointer. We rely on this for varargs calls, calls with arity + // mismatch (the callframe is slided) and tail calls. + m_jit.addPtr(TrustedImm32(m_jit.graph().stackPointerOffset() * sizeof(Register)), GPRInfo::callFrameRegister, JITCompiler::stackPointerRegister); + }; - m_jit.addPtr(TrustedImm32(calleeFrameOffset(numArgs)), GPRInfo::callFrameRegister); + CallLinkInfo* callLinkInfo = m_jit.codeBlock()->addCallLinkInfo(); + callLinkInfo->setUpCall(callType, m_currentNode->origin.semantic, calleeGPR); + + if (node->op() == CallEval) { + // We want to call operationCallEval but we don't want to overwrite the parameter area in + // which we have created a prototypical eval call frame. This means that we have to + // subtract stack to make room for the call. Lucky for us, at this point we have the whole + // register file to ourselves. + + m_jit.emitStoreCallSiteIndex(callSite); + m_jit.addPtr(TrustedImm32(-static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC))), JITCompiler::stackPointerRegister, GPRInfo::regT0); + m_jit.storePtr(GPRInfo::callFrameRegister, JITCompiler::Address(GPRInfo::regT0, CallFrame::callerFrameOffset())); + + // Now we need to make room for: + // - The caller frame and PC of a call to operationCallEval. + // - Potentially two arguments on the stack. + unsigned requiredBytes = sizeof(CallerFrameAndPC) + sizeof(ExecState*) * 2; + requiredBytes = WTF::roundUpToMultipleOf(stackAlignmentBytes(), requiredBytes); + m_jit.subPtr(TrustedImm32(requiredBytes), JITCompiler::stackPointerRegister); + m_jit.setupArgumentsWithExecState(GPRInfo::regT0); + prepareForExternalCall(); + m_jit.appendCall(operationCallEval); + m_jit.exceptionCheck(); + JITCompiler::Jump done = m_jit.branchTest64(JITCompiler::NonZero, GPRInfo::returnValueGPR); + + // This is the part where we meant to make a normal call. Oops. + m_jit.addPtr(TrustedImm32(requiredBytes), JITCompiler::stackPointerRegister); + m_jit.load64(JITCompiler::calleeFrameSlot(CallFrameSlot::callee), GPRInfo::regT0); + m_jit.emitDumbVirtualCall(callLinkInfo); + + done.link(&m_jit); + setResultAndResetStack(); + return; + } - slowPath.append(m_jit.branchPtrWithPatch(MacroAssembler::NotEqual, calleeGPR, targetToCheck, MacroAssembler::TrustedImmPtr(0))); + if (isDirect) { + callLinkInfo->setExecutableDuringCompilation(executable); + callLinkInfo->setMaxNumArguments(numAllocatedArgs); - m_jit.loadPtr(MacroAssembler::Address(calleeGPR, OBJECT_OFFSETOF(JSFunction, m_scope)), resultGPR); - m_jit.store64(resultGPR, MacroAssembler::Address(GPRInfo::callFrameRegister, static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain)); + if (isTail) { + RELEASE_ASSERT(node->op() == DirectTailCall); + + JITCompiler::PatchableJump patchableJump = m_jit.patchableJump(); + JITCompiler::Label mainPath = m_jit.label(); + + m_jit.emitStoreCallSiteIndex(callSite); + + callLinkInfo->setFrameShuffleData(shuffleData); + CallFrameShuffler(m_jit, shuffleData).prepareForTailCall(); + + JITCompiler::Call call = m_jit.nearTailCall(); + + JITCompiler::Label slowPath = m_jit.label(); + patchableJump.m_jump.linkTo(slowPath, &m_jit); + + silentSpillAllRegisters(InvalidGPRReg); + callOperation(operationLinkDirectCall, callLinkInfo, calleeGPR); + silentFillAllRegisters(InvalidGPRReg); + m_jit.exceptionCheck(); + m_jit.jump().linkTo(mainPath, &m_jit); + + useChildren(node); + + m_jit.addJSDirectTailCall(patchableJump, call, slowPath, callLinkInfo); + return; + } + + JITCompiler::Label mainPath = m_jit.label(); + + m_jit.emitStoreCallSiteIndex(callSite); + + JITCompiler::Call call = m_jit.nearCall(); + JITCompiler::Jump done = m_jit.jump(); + + JITCompiler::Label slowPath = m_jit.label(); + if (isX86()) + m_jit.pop(JITCompiler::selectScratchGPR(calleeGPR)); - JITCompiler::Call fastCall = m_jit.nearCall(); + callOperation(operationLinkDirectCall, callLinkInfo, calleeGPR); + m_jit.exceptionCheck(); + m_jit.jump().linkTo(mainPath, &m_jit); + + done.link(&m_jit); + + setResultAndResetStack(); + + m_jit.addJSDirectCall(call, slowPath, callLinkInfo); + return; + } - JITCompiler::Jump done = m_jit.jump(); + m_jit.emitStoreCallSiteIndex(callSite); + JITCompiler::DataLabelPtr targetToCheck; + JITCompiler::Jump slowPath = m_jit.branchPtrWithPatch(MacroAssembler::NotEqual, calleeGPR, targetToCheck, TrustedImmPtr(0)); + + if (isTail) { + if (node->op() == TailCall) { + callLinkInfo->setFrameShuffleData(shuffleData); + CallFrameShuffler(m_jit, shuffleData).prepareForTailCall(); + } else { + m_jit.emitRestoreCalleeSaves(); + m_jit.prepareForTailCallSlow(); + } + } + + JITCompiler::Call fastCall = isTail ? m_jit.nearTailCall() : m_jit.nearCall(); + + JITCompiler::Jump done = m_jit.jump(); + slowPath.link(&m_jit); - - m_jit.move(calleeGPR, GPRInfo::regT0); // Callee needs to be in regT0 + + if (node->op() == TailCall) { + CallFrameShuffler callFrameShuffler(m_jit, shuffleData); + callFrameShuffler.setCalleeJSValueRegs(JSValueRegs(GPRInfo::regT0)); + callFrameShuffler.prepareForSlowPath(); + } else { + m_jit.move(calleeGPR, GPRInfo::regT0); // Callee needs to be in regT0 + + if (isTail) + m_jit.emitRestoreCalleeSaves(); // This needs to happen after we moved calleeGPR to regT0 + } + + m_jit.move(TrustedImmPtr(callLinkInfo), GPRInfo::regT2); // Link info needs to be in regT2 JITCompiler::Call slowCall = m_jit.nearCall(); - + done.link(&m_jit); - - m_jit.move(GPRInfo::returnValueGPR, resultGPR); - - jsValueResult(resultGPR, m_currentNode, DataFormatJS, UseChildrenCalledExplicitly); - - m_jit.addJSCall(fastCall, slowCall, targetToCheck, callType, calleeGPR, m_currentNode->codeOrigin); + + if (isTail) + m_jit.abortWithReason(JITDidReturnFromTailCall); + else + setResultAndResetStack(); + + m_jit.addJSCall(fastCall, slowCall, targetToCheck, callLinkInfo); } // Clang should allow unreachable [[clang::fallthrough]] in template functions if any template expansion uses it @@ -724,25 +1058,26 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnF { AbstractValue& value = m_state.forNode(edge); SpeculatedType type = value.m_type; - ASSERT(edge.useKind() != KnownInt32Use || !(value.m_type & ~SpecInt32)); - m_interpreter.filter(value, SpecInt32); + ASSERT(edge.useKind() != KnownInt32Use || !(value.m_type & ~SpecInt32Only)); + + m_interpreter.filter(value, SpecInt32Only); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + returnFormat = DataFormatInt32; + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if ((edge->hasConstant() && !isInt32Constant(edge.node())) || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - returnFormat = DataFormatInt32; - return allocate(); - } - GPRReg gpr = allocate(); if (edge->hasConstant()) { m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - ASSERT(isInt32Constant(edge.node())); - m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr); + ASSERT(edge->isInt32Constant()); + m_jit.move(MacroAssembler::Imm32(edge->asInt32()), gpr); info.fillInt32(*m_stream, gpr); returnFormat = DataFormatInt32; return gpr; @@ -750,7 +1085,7 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnF DataFormat spillFormat = info.spillFormat(); - RELEASE_ASSERT((spillFormat & DataFormatJS) || spillFormat == DataFormatInt32 || spillFormat == DataFormatInt52 || spillFormat == DataFormatStrictInt52); + DFG_ASSERT(m_jit.graph(), m_currentNode, (spillFormat & DataFormatJS) || spillFormat == DataFormatInt32); m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); @@ -764,41 +1099,13 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnF } if (spillFormat == DataFormatInt32) { m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr); - m_jit.or64(GPRInfo::tagTypeNumberRegister, gpr); - } else - m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - info.fillJSValue(*m_stream, gpr, DataFormatJSInt32); - returnFormat = DataFormatJSInt32; - return gpr; - } - if (spillFormat == DataFormatInt52 || spillFormat == DataFormatStrictInt52) { - // Generally, this can only happen if we've already proved that the - // value is an int32. That's because if a value originated as a JSValue - // then we would speculate that it's an int32 before representing it as - // an int52. Otherwise, if we knowingly produced an int52, then we would - // be boxing it into a value using Int52ToValue. This assertion is valid - // only because Int52 is something that we introduce at prediction time. - // However: we may have an int32-producing node replaced by an - // int52-producing node due to CSE. So we must do a check. - RELEASE_ASSERT(!(type & ~SpecMachineInt)); - if (type & SpecInt52) { - GPRReg temp = allocate(); - m_jit.signExtend32ToPtr(gpr, temp); - // Currently, we can't supply value profiling information here. :-/ - speculationCheck( - BadType, JSValueRegs(), 0, - m_jit.branch64(MacroAssembler::NotEqual, gpr, temp)); - unlock(temp); - } - if (spillFormat == DataFormatStrictInt52) - m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr); - else { + info.fillInt32(*m_stream, gpr); + returnFormat = DataFormatInt32; + } else { m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); - m_jit.zeroExtend32ToPtr(gpr, gpr); + info.fillJSValue(*m_stream, gpr, DataFormatJSInt32); + returnFormat = DataFormatJSInt32; } - info.fillInt32(*m_stream, gpr); - returnFormat = DataFormatInt32; return gpr; } m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); @@ -810,11 +1117,11 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnF } case DataFormatJS: { - RELEASE_ASSERT(!(type & SpecInt52)); + DFG_ASSERT(m_jit.graph(), m_currentNode, !(type & SpecInt52Only)); // Check the value is an integer. GPRReg gpr = info.gpr(); m_gprs.lock(gpr); - if (type & ~SpecInt32) + if (type & ~SpecInt32Only) speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branch64(MacroAssembler::Below, gpr, GPRInfo::tagTypeNumberRegister)); info.fillJSValue(*m_stream, gpr, DataFormatJSInt32); // If !strict we're done, return. @@ -859,61 +1166,19 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Internal(Edge edge, DataFormat& returnF return gpr; } - case DataFormatStrictInt52: - case DataFormatInt52: { - GPRReg gpr = info.gpr(); - GPRReg result; - DataFormat oldFormat = info.registerFormat(); - if (m_gprs.isLocked(gpr)) { - result = allocate(); - m_jit.move(gpr, result); - } else { - lock(gpr); - info.fillInt32(*m_stream, gpr); - result = gpr; - } - RELEASE_ASSERT(!(type & ~SpecMachineInt)); - if (oldFormat == DataFormatInt52) - m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), result); - if (type & SpecInt52) { - GPRReg temp = allocate(); - m_jit.signExtend32ToPtr(result, temp); - // Currently, we can't supply value profiling information here. :-/ - speculationCheck( - BadType, JSValueRegs(), 0, - m_jit.branch64(MacroAssembler::NotEqual, result, temp)); - unlock(temp); - } - m_jit.zeroExtend32ToPtr(result, result); - returnFormat = DataFormatInt32; - return gpr; - } - - case DataFormatDouble: - case DataFormatJSDouble: { - if (edge->hasConstant() && isInt32Constant(edge.node())) { - GPRReg gpr = allocate(); - ASSERT(isInt32Constant(edge.node())); - m_jit.move(MacroAssembler::Imm32(valueOfInt32Constant(edge.node())), gpr); - returnFormat = DataFormatInt32; - return gpr; - } - FALLTHROUGH; - } + case DataFormatJSDouble: case DataFormatCell: case DataFormatBoolean: case DataFormatJSCell: - case DataFormatJSBoolean: { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - returnFormat = DataFormatInt32; - return allocate(); - } - + case DataFormatJSBoolean: + case DataFormatDouble: case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); + case DataFormatInt52: + case DataFormatStrictInt52: + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Corrupt data format"); return InvalidGPRReg; } } @@ -930,7 +1195,7 @@ GPRReg SpeculativeJIT::fillSpeculateInt32Strict(Edge edge) { DataFormat mustBeDataFormatInt32; GPRReg result = fillSpeculateInt32Internal<true>(edge, mustBeDataFormatInt32); - RELEASE_ASSERT(mustBeDataFormatInt32 == DataFormatInt32); + DFG_ASSERT(m_jit.graph(), m_currentNode, mustBeDataFormatInt32 == DataFormatInt32); return result; } @@ -938,25 +1203,25 @@ GPRReg SpeculativeJIT::fillSpeculateInt52(Edge edge, DataFormat desiredFormat) { ASSERT(desiredFormat == DataFormatInt52 || desiredFormat == DataFormatStrictInt52); AbstractValue& value = m_state.forNode(edge); - SpeculatedType type = value.m_type; - m_interpreter.filter(value, SpecMachineInt); + + m_interpreter.filter(value, SpecAnyInt); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if ((edge->hasConstant() && !valueOfJSConstant(edge.node()).isMachineInt()) || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - GPRReg gpr = allocate(); if (edge->hasConstant()) { - JSValue jsValue = valueOfJSConstant(edge.node()); - ASSERT(jsValue.isMachineInt()); + JSValue jsValue = edge->asJSValue(); + ASSERT(jsValue.isAnyInt()); m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - int64_t value = jsValue.asMachineInt(); + int64_t value = jsValue.asAnyInt(); if (desiredFormat == DataFormatInt52) value = value << JSValue::int52ShiftAmount; m_jit.move(MacroAssembler::Imm64(value), gpr); @@ -966,82 +1231,21 @@ GPRReg SpeculativeJIT::fillSpeculateInt52(Edge edge, DataFormat desiredFormat) DataFormat spillFormat = info.spillFormat(); - RELEASE_ASSERT((spillFormat & DataFormatJS) || spillFormat == DataFormatInt32 || spillFormat == DataFormatInt52 || spillFormat == DataFormatStrictInt52); + DFG_ASSERT(m_jit.graph(), m_currentNode, spillFormat == DataFormatInt52 || spillFormat == DataFormatStrictInt52); m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - if (spillFormat == DataFormatJSInt32 || spillFormat == DataFormatInt32) { - // If we know this was spilled as an integer we can fill without checking. - m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr); - m_jit.signExtend32ToPtr(gpr, gpr); - if (desiredFormat == DataFormatStrictInt52) { - info.fillStrictInt52(*m_stream, gpr); - return gpr; - } - m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); - info.fillInt52(*m_stream, gpr); - return gpr; - } - if (spillFormat == DataFormatInt52 || spillFormat == DataFormatStrictInt52) { - m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - if (desiredFormat == DataFormatStrictInt52) { - if (spillFormat == DataFormatInt52) - m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); - info.fillStrictInt52(*m_stream, gpr); - return gpr; - } - if (spillFormat == DataFormatStrictInt52) - m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); - info.fillInt52(*m_stream, gpr); - return gpr; - } m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - - // Fill as JSValue, and fall through. - info.fillJSValue(*m_stream, gpr, DataFormatJSInt32); - m_gprs.unlock(gpr); - FALLTHROUGH; - } - - case DataFormatJS: { - // Check the value is an integer. Note that we would *like* to unbox an Int52 - // at this point but this is too costly. We only *prove* that this is an Int52 - // even though we check if it's an int32. - GPRReg gpr = info.gpr(); - GPRReg result; - if (m_gprs.isLocked(gpr)) { - result = allocate(); - m_jit.move(gpr, result); - } else { - m_gprs.lock(gpr); - result = gpr; - } - if (type & ~SpecInt32) - speculationCheck(BadType, JSValueRegs(result), edge, m_jit.branch64(MacroAssembler::Below, result, GPRInfo::tagTypeNumberRegister)); - if (result == gpr) // The not-already-locked, so fill in-place, case. - info.fillInt52(*m_stream, gpr, desiredFormat); - m_jit.signExtend32ToPtr(result, result); - if (desiredFormat == DataFormatInt52) - m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), result); - return result; - } - - case DataFormatInt32: - case DataFormatJSInt32: { - GPRReg gpr = info.gpr(); - GPRReg result; - if (m_gprs.isLocked(gpr)) { - result = allocate(); - m_jit.move(gpr, result); - } else { - m_gprs.lock(gpr); - info.fillInt52(*m_stream, gpr, desiredFormat); - result = gpr; + if (desiredFormat == DataFormatStrictInt52) { + if (spillFormat == DataFormatInt52) + m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); + info.fillStrictInt52(*m_stream, gpr); + return gpr; } - m_jit.signExtend32ToPtr(result, result); - if (desiredFormat == DataFormatInt52) - m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), result); - return result; + if (spillFormat == DataFormatStrictInt52) + m_jit.lshift64(TrustedImm32(JSValue::int52ShiftAmount), gpr); + info.fillInt52(*m_stream, gpr); + return gpr; } case DataFormatStrictInt52: { @@ -1078,66 +1282,32 @@ GPRReg SpeculativeJIT::fillSpeculateInt52(Edge edge, DataFormat desiredFormat) return gpr; } - case DataFormatDouble: - case DataFormatJSDouble: - if (edge->hasConstant()) { - JSValue jsValue = valueOfJSConstant(edge.node()); - if (jsValue.isMachineInt()) { - int64_t value = jsValue.asMachineInt(); - if (desiredFormat == DataFormatInt52) - value = value << JSValue::int52ShiftAmount; - GPRReg gpr = allocate(); - m_jit.move(MacroAssembler::Imm64(value), gpr); - return gpr; - } - } - FALLTHROUGH; - case DataFormatCell: - case DataFormatBoolean: - case DataFormatJSCell: - case DataFormatJSBoolean: { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - - case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); - default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); return InvalidGPRReg; } } FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge) { - AbstractValue& value = m_state.forNode(edge); - SpeculatedType type = value.m_type; - ASSERT(edge.useKind() != KnownNumberUse || !(value.m_type & ~SpecFullNumber)); - m_interpreter.filter(value, SpecFullNumber); + ASSERT(edge.useKind() == DoubleRepUse || edge.useKind() == DoubleRepRealUse || edge.useKind() == DoubleRepAnyIntUse); + ASSERT(edge->hasDoubleResult()); VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); if (info.registerFormat() == DataFormatNone) { if (edge->hasConstant()) { - GPRReg gpr = allocate(); - - if (isInt32Constant(edge.node())) { + if (edge->isNumberConstant()) { FPRReg fpr = fprAllocate(); - m_jit.move(MacroAssembler::Imm64(reinterpretDoubleToInt64(static_cast<double>(valueOfInt32Constant(edge.node())))), gpr); - m_jit.move64ToDouble(gpr, fpr); - unlock(gpr); - - // Don't fill double here since that will lead to confusion: the - // register allocator will now think that this is a double while - // everyone else thinks it's an integer. - return fpr; - } - if (isNumberConstant(edge.node())) { - FPRReg fpr = fprAllocate(); - m_jit.move(MacroAssembler::Imm64(reinterpretDoubleToInt64(valueOfNumberConstant(edge.node()))), gpr); - m_jit.move64ToDouble(gpr, fpr); - unlock(gpr); + int64_t doubleAsInt = reinterpretDoubleToInt64(edge->asNumber()); + if (!doubleAsInt) + m_jit.moveZeroToDouble(fpr); + else { + GPRReg gpr = allocate(); + m_jit.move(MacroAssembler::Imm64(doubleAsInt), gpr); + m_jit.move64ToDouble(gpr, fpr); + unlock(gpr); + } m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); info.fillDouble(*m_stream, fpr); @@ -1148,159 +1318,24 @@ FPRReg SpeculativeJIT::fillSpeculateDouble(Edge edge) } DataFormat spillFormat = info.spillFormat(); - switch (spillFormat) { - case DataFormatDouble: { - FPRReg fpr = fprAllocate(); - m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr); - m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); - info.fillDouble(*m_stream, fpr); - return fpr; - } - - case DataFormatInt32: { - GPRReg gpr = allocate(); - - m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - m_jit.load32(JITCompiler::addressFor(virtualRegister), gpr); - info.fillInt32(*m_stream, gpr); - unlock(gpr); - break; - } - - case DataFormatInt52: { - GPRReg gpr = allocate(); - m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - info.fillInt52(*m_stream, gpr); - unlock(gpr); - break; - } - - case DataFormatStrictInt52: { - GPRReg gpr = allocate(); - m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - info.fillStrictInt52(*m_stream, gpr); - unlock(gpr); - break; + if (spillFormat != DataFormatDouble) { + DFG_CRASH( + m_jit.graph(), m_currentNode, toCString( + "Expected ", edge, " to have double format but instead it is spilled as ", + dataFormatToString(spillFormat)).data()); } - - default: - GPRReg gpr = allocate(); - - RELEASE_ASSERT(spillFormat & DataFormatJS); - m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); - m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); - info.fillJSValue(*m_stream, gpr, spillFormat); - unlock(gpr); - break; - } - } - - switch (info.registerFormat()) { - case DataFormatNone: // Should have filled, above. - case DataFormatBoolean: // This type never occurs. - case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); - - case DataFormatCell: - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return fprAllocate(); - - case DataFormatJSCell: - case DataFormatJS: - case DataFormatJSBoolean: { - GPRReg jsValueGpr = info.gpr(); - m_gprs.lock(jsValueGpr); - FPRReg fpr = fprAllocate(); - GPRReg tempGpr = allocate(); - - JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, jsValueGpr, GPRInfo::tagTypeNumberRegister); - - if (type & ~SpecFullNumber) - speculationCheck(BadType, JSValueRegs(jsValueGpr), edge, m_jit.branchTest64(MacroAssembler::Zero, jsValueGpr, GPRInfo::tagTypeNumberRegister)); - - // First, if we get here we have a double encoded as a JSValue - m_jit.move(jsValueGpr, tempGpr); - unboxDouble(tempGpr, fpr); - JITCompiler::Jump hasUnboxedDouble = m_jit.jump(); - - // Finally, handle integers. - isInteger.link(&m_jit); - m_jit.convertInt32ToDouble(jsValueGpr, fpr); - hasUnboxedDouble.link(&m_jit); - - m_gprs.release(jsValueGpr); - m_gprs.unlock(jsValueGpr); - m_gprs.unlock(tempGpr); - m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); - info.fillDouble(*m_stream, fpr); - info.killSpilled(); - return fpr; - } - - case DataFormatJSInt32: - case DataFormatInt32: { - FPRReg fpr = fprAllocate(); - GPRReg gpr = info.gpr(); - m_gprs.lock(gpr); - m_jit.convertInt32ToDouble(gpr, fpr); - m_gprs.unlock(gpr); - return fpr; - } - - case DataFormatInt52: { - FPRReg fpr = fprAllocate(); - GPRReg gpr = info.gpr(); - m_gprs.lock(gpr); - GPRReg temp = allocate(); - m_jit.move(gpr, temp); - m_jit.rshift64(TrustedImm32(JSValue::int52ShiftAmount), temp); - m_jit.convertInt64ToDouble(temp, fpr); - unlock(temp); - m_gprs.unlock(gpr); - return fpr; - } - - case DataFormatStrictInt52: { + DFG_ASSERT(m_jit.graph(), m_currentNode, spillFormat == DataFormatDouble); FPRReg fpr = fprAllocate(); - GPRReg gpr = info.gpr(); - m_gprs.lock(gpr); - m_jit.convertInt64ToDouble(gpr, fpr); - m_gprs.unlock(gpr); - return fpr; - } - - // Unbox the double - case DataFormatJSDouble: { - GPRReg gpr = info.gpr(); - FPRReg fpr = fprAllocate(); - if (m_gprs.isLocked(gpr)) { - // Make sure we don't trample gpr if it is in use. - GPRReg temp = allocate(); - m_jit.move(gpr, temp); - unboxDouble(temp, fpr); - unlock(temp); - } else - unboxDouble(gpr, fpr); - - m_gprs.release(gpr); + m_jit.loadDouble(JITCompiler::addressFor(virtualRegister), fpr); m_fprs.retain(fpr, virtualRegister, SpillOrderDouble); - info.fillDouble(*m_stream, fpr); return fpr; } - case DataFormatDouble: { - FPRReg fpr = info.fpr(); - m_fprs.lock(fpr); - return fpr; - } - - default: - RELEASE_ASSERT_NOT_REACHED(); - return InvalidFPRReg; - } + DFG_ASSERT(m_jit.graph(), m_currentNode, info.registerFormat() == DataFormatDouble); + FPRReg fpr = info.fpr(); + m_fprs.lock(fpr); + return fpr; } GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) @@ -1308,37 +1343,34 @@ GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) AbstractValue& value = m_state.forNode(edge); SpeculatedType type = value.m_type; ASSERT((edge.useKind() != KnownCellUse && edge.useKind() != KnownStringUse) || !(value.m_type & ~SpecCell)); + m_interpreter.filter(value, SpecCell); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if (info.spillFormat() == DataFormatInt32 || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - GPRReg gpr = allocate(); if (edge->hasConstant()) { - JSValue jsValue = valueOfJSConstant(edge.node()); - if (jsValue.isCell()) { - m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); - info.fillJSValue(*m_stream, gpr, DataFormatJSCell); - return gpr; - } - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + JSValue jsValue = edge->asJSValue(); + m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); + m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); + info.fillJSValue(*m_stream, gpr, DataFormatJSCell); return gpr; } - RELEASE_ASSERT(info.spillFormat() & DataFormatJS); + m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); info.fillJSValue(*m_stream, gpr, DataFormatJS); if (type & ~SpecCell) - speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagMaskRegister)); + speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchIfNotCell(JSValueRegs(gpr))); info.fillJSValue(*m_stream, gpr, DataFormatJSCell); return gpr; } @@ -1348,8 +1380,8 @@ GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) GPRReg gpr = info.gpr(); m_gprs.lock(gpr); if (!ASSERT_DISABLED) { - MacroAssembler::Jump checkCell = m_jit.branchTest64(MacroAssembler::Zero, gpr, GPRInfo::tagMaskRegister); - m_jit.breakpoint(); + MacroAssembler::Jump checkCell = m_jit.branchIfCell(JSValueRegs(gpr)); + m_jit.abortWithReason(DFGIsNotCell); checkCell.link(&m_jit); } return gpr; @@ -1359,7 +1391,7 @@ GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) GPRReg gpr = info.gpr(); m_gprs.lock(gpr); if (type & ~SpecCell) - speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchTest64(MacroAssembler::NonZero, gpr, GPRInfo::tagMaskRegister)); + speculationCheck(BadType, JSValueRegs(gpr), edge, m_jit.branchIfNotCell(JSValueRegs(gpr))); info.fillJSValue(*m_stream, gpr, DataFormatJSCell); return gpr; } @@ -1367,20 +1399,16 @@ GPRReg SpeculativeJIT::fillSpeculateCell(Edge edge) case DataFormatJSInt32: case DataFormatInt32: case DataFormatJSDouble: - case DataFormatDouble: case DataFormatJSBoolean: case DataFormatBoolean: - case DataFormatInt52: - case DataFormatStrictInt52: { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - + case DataFormatDouble: case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); + case DataFormatInt52: + case DataFormatStrictInt52: + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Corrupt data format"); return InvalidGPRReg; } } @@ -1389,31 +1417,29 @@ GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge) { AbstractValue& value = m_state.forNode(edge); SpeculatedType type = value.m_type; + ASSERT(edge.useKind() != KnownBooleanUse || !(value.m_type & ~SpecBoolean)); + m_interpreter.filter(value, SpecBoolean); + if (value.isClear()) { + terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + return allocate(); + } + VirtualRegister virtualRegister = edge->virtualRegister(); GenerationInfo& info = generationInfoFromVirtualRegister(virtualRegister); switch (info.registerFormat()) { case DataFormatNone: { - if (info.spillFormat() == DataFormatInt32 || info.spillFormat() == DataFormatDouble) { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - GPRReg gpr = allocate(); if (edge->hasConstant()) { - JSValue jsValue = valueOfJSConstant(edge.node()); - if (jsValue.isBoolean()) { - m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); - m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); - info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean); - return gpr; - } - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); + JSValue jsValue = edge->asJSValue(); + m_gprs.retain(gpr, virtualRegister, SpillOrderConstant); + m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsValue)), gpr); + info.fillJSValue(*m_stream, gpr, DataFormatJSBoolean); return gpr; } - RELEASE_ASSERT(info.spillFormat() & DataFormatJS); + DFG_ASSERT(m_jit.graph(), m_currentNode, info.spillFormat() & DataFormatJS); m_gprs.retain(gpr, virtualRegister, SpillOrderSpilled); m_jit.load64(JITCompiler::addressFor(virtualRegister), gpr); @@ -1449,61 +1475,20 @@ GPRReg SpeculativeJIT::fillSpeculateBoolean(Edge edge) case DataFormatJSInt32: case DataFormatInt32: case DataFormatJSDouble: - case DataFormatDouble: case DataFormatJSCell: case DataFormatCell: - case DataFormatInt52: - case DataFormatStrictInt52: { - terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); - return allocate(); - } - + case DataFormatDouble: case DataFormatStorage: - RELEASE_ASSERT_NOT_REACHED(); + case DataFormatInt52: + case DataFormatStrictInt52: + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad data format"); default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Corrupt data format"); return InvalidGPRReg; } } -JITCompiler::Jump SpeculativeJIT::convertToDouble(GPRReg value, FPRReg result, GPRReg tmp) -{ - JITCompiler::Jump isInteger = m_jit.branch64(MacroAssembler::AboveOrEqual, value, GPRInfo::tagTypeNumberRegister); - - JITCompiler::Jump notNumber = m_jit.branchTest64(MacroAssembler::Zero, value, GPRInfo::tagTypeNumberRegister); - - m_jit.move(value, tmp); - unboxDouble(tmp, result); - - JITCompiler::Jump done = m_jit.jump(); - - isInteger.link(&m_jit); - - m_jit.convertInt32ToDouble(value, result); - - done.link(&m_jit); - - return notNumber; -} - -void SpeculativeJIT::compileBaseValueStoreBarrier(Edge& baseEdge, Edge& valueEdge) -{ -#if ENABLE(GGC) - ASSERT(!isKnownNotCell(valueEdge.node())); - - SpeculateCellOperand base(this, baseEdge); - JSValueOperand value(this, valueEdge); - GPRTemporary scratch1(this); - GPRTemporary scratch2(this); - - writeBarrier(base.gpr(), value.gpr(), valueEdge, scratch1.gpr(), scratch2.gpr()); -#else - UNUSED_PARAM(baseEdge); - UNUSED_PARAM(valueEdge); -#endif -} - void SpeculativeJIT::compileObjectEquality(Node* node) { SpeculateCellOperand op1(this, node->child1()); @@ -1516,54 +1501,73 @@ void SpeculativeJIT::compileObjectEquality(Node* node) if (masqueradesAsUndefinedWatchpointIsStillValid()) { DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchIfNotObject(op1GPR)); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchIfNotObject(op2GPR)); } else { - GPRTemporary structure(this); - GPRReg structureGPR = structure.gpr(); - - m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), node->child1(), SpecObject, m_jit.branchIfNotObject(op1GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), node->child1(), m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); - m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op2GPR), node->child2(), SpecObject, m_jit.branchIfNotObject(op2GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op2GPR), node->child2(), m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } - - MacroAssembler::Jump falseCase = m_jit.branch64(MacroAssembler::NotEqual, op1GPR, op2GPR); - m_jit.move(TrustedImm32(ValueTrue), resultGPR); - MacroAssembler::Jump done = m_jit.jump(); - falseCase.link(&m_jit); - m_jit.move(TrustedImm32(ValueFalse), resultGPR); - done.link(&m_jit); + m_jit.compare64(MacroAssembler::Equal, op1GPR, op2GPR, resultGPR); + m_jit.or32(TrustedImm32(ValueFalse), resultGPR); jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean); } +void SpeculativeJIT::compileObjectStrictEquality(Edge objectChild, Edge otherChild) +{ + SpeculateCellOperand op1(this, objectChild); + JSValueOperand op2(this, otherChild); + GPRTemporary result(this); + + GPRReg op1GPR = op1.gpr(); + GPRReg op2GPR = op2.gpr(); + GPRReg resultGPR = result.gpr(); + + DFG_TYPE_CHECK(JSValueSource::unboxedCell(op1GPR), objectChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); + + // At this point we know that we can perform a straight-forward equality comparison on pointer + // values because we are doing strict equality. + m_jit.compare64(MacroAssembler::Equal, op1GPR, op2GPR, resultGPR); + m_jit.or32(TrustedImm32(ValueFalse), resultGPR); + jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean); +} + +void SpeculativeJIT::compilePeepHoleObjectStrictEquality(Edge objectChild, Edge otherChild, Node* branchNode) +{ + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; + + SpeculateCellOperand op1(this, objectChild); + JSValueOperand op2(this, otherChild); + + GPRReg op1GPR = op1.gpr(); + GPRReg op2GPR = op2.gpr(); + + DFG_TYPE_CHECK(JSValueSource::unboxedCell(op1GPR), objectChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); + + if (taken == nextBlock()) { + branchPtr(MacroAssembler::NotEqual, op1GPR, op2GPR, notTaken); + jump(taken); + } else { + branchPtr(MacroAssembler::Equal, op1GPR, op2GPR, taken); + jump(notTaken); + } +} + void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild) { SpeculateCellOperand op1(this, leftChild); @@ -1573,71 +1577,46 @@ void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge r GPRReg op1GPR = op1.gpr(); GPRReg op2GPR = op2.gpr(); GPRReg resultGPR = result.gpr(); - GPRTemporary structure; - GPRReg structureGPR = InvalidGPRReg; bool masqueradesAsUndefinedWatchpointValid = masqueradesAsUndefinedWatchpointIsStillValid(); - if (!masqueradesAsUndefinedWatchpointValid) { - // The masquerades as undefined case will use the structure register, so allocate it here. - // Do this at the top of the function to avoid branching around a register allocation. - GPRTemporary realStructure(this); - structure.adopt(realStructure); - structureGPR = structure.gpr(); - } - if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } // It seems that most of the time when programs do a == b where b may be either null/undefined // or an object, b is usually an object. Balance the branches to make that case fast. - MacroAssembler::Jump rightNotCell = - m_jit.branchTest64(MacroAssembler::NonZero, op2GPR, GPRInfo::tagMaskRegister); + MacroAssembler::Jump rightNotCell = m_jit.branchIfNotCell(JSValueRegs(op2GPR)); // We know that within this branch, rightChild must be a cell. if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( - JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2GPR)); speculationCheck(BadType, JSValueRegs(op2GPR), rightChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } // At this point we know that we can perform a straight-forward equality comparison on pointer // values because both left and right are pointers to objects that have no special equality // protocols. - MacroAssembler::Jump falseCase = m_jit.branch64(MacroAssembler::NotEqual, op1GPR, op2GPR); - MacroAssembler::Jump trueCase = m_jit.jump(); + m_jit.compare64(MacroAssembler::Equal, op1GPR, op2GPR, resultGPR); + MacroAssembler::Jump done = m_jit.jump(); rightNotCell.link(&m_jit); @@ -1653,21 +1632,17 @@ void SpeculativeJIT::compileObjectToObjectOrOtherEquality(Edge leftChild, Edge r MacroAssembler::NotEqual, resultGPR, MacroAssembler::TrustedImm64(ValueNull))); } - - falseCase.link(&m_jit); - m_jit.move(TrustedImm32(ValueFalse), resultGPR); - MacroAssembler::Jump done = m_jit.jump(); - trueCase.link(&m_jit); - m_jit.move(TrustedImm32(ValueTrue), resultGPR); + m_jit.move(TrustedImm32(0), result.gpr()); + done.link(&m_jit); - + m_jit.or32(TrustedImm32(ValueFalse), resultGPR); jsValueResult(resultGPR, m_currentNode, DataFormatJSBoolean); } void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild, Edge rightChild, Node* branchNode) { - BasicBlock* taken = branchNode->takenBlock(); - BasicBlock* notTaken = branchNode->notTakenBlock(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; SpeculateCellOperand op1(this, leftChild); JSValueOperand op2(this, rightChild, ManualOperandSpeculation); @@ -1676,63 +1651,38 @@ void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild GPRReg op1GPR = op1.gpr(); GPRReg op2GPR = op2.gpr(); GPRReg resultGPR = result.gpr(); - GPRTemporary structure; - GPRReg structureGPR = InvalidGPRReg; - bool masqueradesAsUndefinedWatchpointValid = + bool masqueradesAsUndefinedWatchpointValid = masqueradesAsUndefinedWatchpointIsStillValid(); - if (!masqueradesAsUndefinedWatchpointValid) { - // The masquerades as undefined case will use the structure register, so allocate it here. - // Do this at the top of the function to avoid branching around a register allocation. - GPRTemporary realStructure(this); - structure.adopt(realStructure); - structureGPR = structure.gpr(); - } - if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op1GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op1GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueSource::unboxedCell(op1GPR), leftChild, SpecObject, m_jit.branchIfNotObject(op1GPR)); speculationCheck(BadType, JSValueSource::unboxedCell(op1GPR), leftChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op1GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } // It seems that most of the time when programs do a == b where b may be either null/undefined // or an object, b is usually an object. Balance the branches to make that case fast. - MacroAssembler::Jump rightNotCell = - m_jit.branchTest64(MacroAssembler::NonZero, op2GPR, GPRInfo::tagMaskRegister); + MacroAssembler::Jump rightNotCell = m_jit.branchIfNotCell(JSValueRegs(op2GPR)); // We know that within this branch, rightChild must be a cell. if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( - JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(op2GPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2GPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(op2GPR, JSCell::structureOffset()), structureGPR); DFG_TYPE_CHECK( - JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(op2GPR), rightChild, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(op2GPR)); speculationCheck(BadType, JSValueRegs(op2GPR), rightChild, m_jit.branchTest8( MacroAssembler::NonZero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(op2GPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined))); } @@ -1761,17 +1711,54 @@ void SpeculativeJIT::compilePeepHoleObjectToObjectOrOtherEquality(Edge leftChild jump(notTaken); } +void SpeculativeJIT::compileSymbolUntypedEquality(Node* node, Edge symbolEdge, Edge untypedEdge) +{ + SpeculateCellOperand symbol(this, symbolEdge); + JSValueOperand untyped(this, untypedEdge); + GPRTemporary result(this, Reuse, symbol, untyped); + + GPRReg symbolGPR = symbol.gpr(); + GPRReg untypedGPR = untyped.gpr(); + GPRReg resultGPR = result.gpr(); + + speculateSymbol(symbolEdge, symbolGPR); + + // At this point we know that we can perform a straight-forward equality comparison on pointer + // values because we are doing strict equality. + m_jit.compare64(MacroAssembler::Equal, symbolGPR, untypedGPR, resultGPR); + unblessedBooleanResult(resultGPR, node); +} + void SpeculativeJIT::compileInt32Compare(Node* node, MacroAssembler::RelationalCondition condition) { - SpeculateInt32Operand op1(this, node->child1()); - SpeculateInt32Operand op2(this, node->child2()); - GPRTemporary result(this, Reuse, op1, op2); - - m_jit.compare32(condition, op1.gpr(), op2.gpr(), result.gpr()); - - // If we add a DataFormatBool, we should use it here. - m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); - jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean); + if (node->child1()->isInt32Constant()) { + SpeculateInt32Operand op2(this, node->child2()); + GPRTemporary result(this, Reuse, op2); + int32_t imm = node->child1()->asInt32(); + m_jit.compare32(condition, JITCompiler::Imm32(imm), op2.gpr(), result.gpr()); + + // If we add a DataFormatBool, we should use it here. + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean); + } else if (node->child2()->isInt32Constant()) { + SpeculateInt32Operand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); + int32_t imm = node->child2()->asInt32(); + m_jit.compare32(condition, op1.gpr(), JITCompiler::Imm32(imm), result.gpr()); + + // If we add a DataFormatBool, we should use it here. + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean); + } else { + SpeculateInt32Operand op1(this, node->child1()); + SpeculateInt32Operand op2(this, node->child2()); + GPRTemporary result(this, Reuse, op1, op2); + m_jit.compare32(condition, op1.gpr(), op2.gpr(), result.gpr()); + + // If we add a DataFormatBool, we should use it here. + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + jsValueResult(result.gpr(), m_currentNode, DataFormatJSBoolean); + } } void SpeculativeJIT::compileInt52Compare(Node* node, MacroAssembler::RelationalCondition condition) @@ -1789,8 +1776,8 @@ void SpeculativeJIT::compileInt52Compare(Node* node, MacroAssembler::RelationalC void SpeculativeJIT::compilePeepHoleInt52Branch(Node* node, Node* branchNode, JITCompiler::RelationalCondition condition) { - BasicBlock* taken = branchNode->takenBlock(); - BasicBlock* notTaken = branchNode->notTakenBlock(); + BasicBlock* taken = branchNode->branchData()->taken.block; + BasicBlock* notTaken = branchNode->branchData()->notTaken.block; // The branch instruction will branch to the taken block. // If taken is next, switch taken with notTaken & invert the branch condition so we can fall through. @@ -1830,6 +1817,8 @@ void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse) GPRReg resultGPR = result.gpr(); GPRTemporary structure; GPRReg structureGPR = InvalidGPRReg; + GPRTemporary scratch; + GPRReg scratchGPR = InvalidGPRReg; bool masqueradesAsUndefinedWatchpointValid = masqueradesAsUndefinedWatchpointIsStillValid(); @@ -1838,37 +1827,33 @@ void SpeculativeJIT::compileObjectOrOtherLogicalNot(Edge nodeUse) // The masquerades as undefined case will use the structure register, so allocate it here. // Do this at the top of the function to avoid branching around a register allocation. GPRTemporary realStructure(this); + GPRTemporary realScratch(this); structure.adopt(realStructure); + scratch.adopt(realScratch); structureGPR = structure.gpr(); + scratchGPR = scratch.gpr(); } - MacroAssembler::Jump notCell = m_jit.branchTest64(MacroAssembler::NonZero, valueGPR, GPRInfo::tagMaskRegister); + MacroAssembler::Jump notCell = m_jit.branchIfNotCell(JSValueRegs(valueGPR)); if (masqueradesAsUndefinedWatchpointValid) { DFG_TYPE_CHECK( - JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(valueGPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(valueGPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(valueGPR, JSCell::structureOffset()), structureGPR); - DFG_TYPE_CHECK( - JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - structureGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(valueGPR)); MacroAssembler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8( MacroAssembler::Zero, - MacroAssembler::Address(structureGPR, Structure::typeInfoFlagsOffset()), + MacroAssembler::Address(valueGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(MasqueradesAsUndefined)); + m_jit.emitLoadStructure(valueGPR, structureGPR, scratchGPR); speculationCheck(BadType, JSValueRegs(valueGPR), nodeUse, m_jit.branchPtr( MacroAssembler::Equal, MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()), - MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)))); + TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)))); isNotMasqueradesAsUndefined.link(&m_jit); } @@ -1910,7 +1895,7 @@ void SpeculativeJIT::compileLogicalNot(Node* node) return; } - case NumberUse: { + case DoubleRepUse: { SpeculateDoubleOperand value(this, node->child1()); FPRTemporary scratch(this); GPRTemporary result(this); @@ -1922,7 +1907,8 @@ void SpeculativeJIT::compileLogicalNot(Node* node) return; } - case BooleanUse: { + case BooleanUse: + case KnownBooleanUse: { if (!needsTypeCheck(node->child1(), SpecBoolean)) { SpeculateBooleanOperand value(this, node->child1()); GPRTemporary result(this, Reuse, value); @@ -1955,25 +1941,32 @@ void SpeculativeJIT::compileLogicalNot(Node* node) GPRReg arg1GPR = arg1.gpr(); GPRReg resultGPR = result.gpr(); - - arg1.use(); - - m_jit.move(arg1GPR, resultGPR); - m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), resultGPR); - JITCompiler::Jump slowCase = m_jit.branchTest64(JITCompiler::NonZero, resultGPR, TrustedImm32(static_cast<int32_t>(~1))); - - addSlowPathGenerator( - slowPathCall(slowCase, this, operationConvertJSValueToBoolean, resultGPR, arg1GPR)); - - m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueTrue)), resultGPR); - jsValueResult(resultGPR, node, DataFormatJSBoolean, UseChildrenCalledExplicitly); + + FPRTemporary valueFPR(this); + FPRTemporary tempFPR(this); + + bool shouldCheckMasqueradesAsUndefined = !masqueradesAsUndefinedWatchpointIsStillValid(); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + std::optional<GPRTemporary> scratch; + GPRReg scratchGPR = InvalidGPRReg; + if (shouldCheckMasqueradesAsUndefined) { + scratch.emplace(this); + scratchGPR = scratch->gpr(); + } + bool negateResult = true; + m_jit.emitConvertValueToBoolean(JSValueRegs(arg1GPR), resultGPR, scratchGPR, valueFPR.fpr(), tempFPR.fpr(), shouldCheckMasqueradesAsUndefined, globalObject, negateResult); + m_jit.or32(TrustedImm32(ValueFalse), resultGPR); + jsValueResult(resultGPR, node, DataFormatJSBoolean); return; } case StringUse: return compileStringZeroLength(node); + case StringOrOtherUse: + return compileLogicalNotStringOrOther(node); + default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } } @@ -1982,32 +1975,36 @@ void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BasicBlock* taken, Ba { JSValueOperand value(this, nodeUse, ManualOperandSpeculation); GPRTemporary scratch(this); + GPRTemporary structure; GPRReg valueGPR = value.gpr(); GPRReg scratchGPR = scratch.gpr(); - - MacroAssembler::Jump notCell = m_jit.branchTest64(MacroAssembler::NonZero, valueGPR, GPRInfo::tagMaskRegister); + GPRReg structureGPR = InvalidGPRReg; + + if (!masqueradesAsUndefinedWatchpointIsStillValid()) { + GPRTemporary realStructure(this); + structure.adopt(realStructure); + structureGPR = structure.gpr(); + } + + MacroAssembler::Jump notCell = m_jit.branchIfNotCell(JSValueRegs(valueGPR)); if (masqueradesAsUndefinedWatchpointIsStillValid()) { DFG_TYPE_CHECK( - JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - MacroAssembler::Address(valueGPR, JSCell::structureOffset()), - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(valueGPR)); } else { - m_jit.loadPtr(MacroAssembler::Address(valueGPR, JSCell::structureOffset()), scratchGPR); - DFG_TYPE_CHECK( - JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchPtr( - MacroAssembler::Equal, - scratchGPR, - MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get()))); + JSValueRegs(valueGPR), nodeUse, (~SpecCell) | SpecObject, m_jit.branchIfNotObject(valueGPR)); - JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::Zero, MacroAssembler::Address(scratchGPR, Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)); + JITCompiler::Jump isNotMasqueradesAsUndefined = m_jit.branchTest8( + JITCompiler::Zero, + MacroAssembler::Address(valueGPR, JSCell::typeInfoFlagsOffset()), + TrustedImm32(MasqueradesAsUndefined)); + m_jit.emitLoadStructure(valueGPR, structureGPR, scratchGPR); speculationCheck(BadType, JSValueRegs(valueGPR), nodeUse, m_jit.branchPtr( MacroAssembler::Equal, - MacroAssembler::Address(scratchGPR, Structure::globalObjectOffset()), - MacroAssembler::TrustedImmPtr(m_jit.graph().globalObjectFor(m_currentNode->codeOrigin)))); + MacroAssembler::Address(structureGPR, Structure::globalObjectOffset()), + TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.graph().globalObjectFor(m_currentNode->origin.semantic)))); isNotMasqueradesAsUndefined.link(&m_jit); } @@ -2029,8 +2026,8 @@ void SpeculativeJIT::emitObjectOrOtherBranch(Edge nodeUse, BasicBlock* taken, Ba void SpeculativeJIT::emitBranch(Node* node) { - BasicBlock* taken = node->takenBlock(); - BasicBlock* notTaken = node->notTakenBlock(); + BasicBlock* taken = node->branchData()->taken.block; + BasicBlock* notTaken = node->branchData()->notTaken.block; switch (node->child1().useKind()) { case ObjectOrOtherUse: { @@ -2039,7 +2036,7 @@ void SpeculativeJIT::emitBranch(Node* node) } case Int32Use: - case NumberUse: { + case DoubleRepUse: { if (node->child1().useKind() == Int32Use) { bool invert = false; @@ -2064,12 +2061,23 @@ void SpeculativeJIT::emitBranch(Node* node) return; } + case StringUse: { + emitStringBranch(node->child1(), taken, notTaken); + return; + } + + case StringOrOtherUse: { + emitStringOrOtherBranch(node->child1(), taken, notTaken); + return; + } + case UntypedUse: - case BooleanUse: { + case BooleanUse: + case KnownBooleanUse: { JSValueOperand value(this, node->child1(), ManualOperandSpeculation); GPRReg valueGPR = value.gpr(); - if (node->child1().useKind() == BooleanUse) { + if (node->child1().useKind() == BooleanUse || node->child1().useKind() == KnownBooleanUse) { if (!needsTypeCheck(node->child1(), SpecBoolean)) { MacroAssembler::ResultCondition condition = MacroAssembler::NonZero; @@ -2091,9 +2099,22 @@ void SpeculativeJIT::emitBranch(Node* node) value.use(); } else { GPRTemporary result(this); + FPRTemporary fprValue(this); + FPRTemporary fprTemp(this); + std::optional<GPRTemporary> scratch; + + GPRReg scratchGPR = InvalidGPRReg; + bool shouldCheckMasqueradesAsUndefined = !masqueradesAsUndefinedWatchpointIsStillValid(); + if (shouldCheckMasqueradesAsUndefined) { + scratch.emplace(this); + scratchGPR = scratch->gpr(); + } + GPRReg resultGPR = result.gpr(); + FPRReg valueFPR = fprValue.fpr(); + FPRReg tempFPR = fprTemp.fpr(); - if (node->child1()->prediction() & SpecInt32) { + if (node->child1()->prediction() & SpecInt32Only) { branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsNumber(0))), notTaken); branch64(MacroAssembler::AboveOrEqual, valueGPR, GPRInfo::tagTypeNumberRegister, taken); } @@ -2104,10 +2125,9 @@ void SpeculativeJIT::emitBranch(Node* node) } value.use(); - - silentSpillAllRegisters(resultGPR); - callOperation(operationConvertJSValueToBoolean, resultGPR, valueGPR); - silentFillAllRegisters(resultGPR); + + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + m_jit.emitConvertValueToBoolean(JSValueRegs(valueGPR), resultGPR, scratchGPR, valueFPR, tempFPR, shouldCheckMasqueradesAsUndefined, globalObject); branchTest32(MacroAssembler::NonZero, resultGPR, taken); jump(notTaken); @@ -2118,7 +2138,7 @@ void SpeculativeJIT::emitBranch(Node* node) } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), m_currentNode, "Bad use kind"); } } @@ -2132,40 +2152,54 @@ void SpeculativeJIT::compile(Node* node) switch (op) { case JSConstant: + case DoubleConstant: + case Int52Constant: + case PhantomDirectArguments: + case PhantomClonedArguments: initConstantInfo(node); break; - case PhantomArguments: - initConstantInfo(node); + case LazyJSConstant: + compileLazyJSConstant(node); break; - case WeakJSConstant: - m_jit.addWeakReference(node->weakConstant()); - initConstantInfo(node); - break; - case Identity: { - // CSE should always eliminate this. - RELEASE_ASSERT_NOT_REACHED(); + speculate(node, node->child1()); + switch (node->child1().useKind()) { + case DoubleRepUse: + case DoubleRepRealUse: + case DoubleRepAnyIntUse: { + SpeculateDoubleOperand op(this, node->child1()); + FPRTemporary scratch(this, op); + m_jit.moveDouble(op.fpr(), scratch.fpr()); + doubleResult(scratch.fpr(), node); + break; + } + case Int52RepUse: { + SpeculateInt52Operand op(this, node->child1()); + GPRTemporary result(this, Reuse, op); + m_jit.move(op.gpr(), result.gpr()); + int52Result(result.gpr(), node); + break; + } + default: { + JSValueOperand op(this, node->child1()); + GPRTemporary result(this, Reuse, op); + m_jit.move(op.gpr(), result.gpr()); + jsValueResult(result.gpr(), node); + break; + } + } // switch break; } case GetLocal: { - SpeculatedType prediction = node->variableAccessData()->prediction(); AbstractValue& value = m_state.variables().operand(node->local()); - // If we have no prediction for this local, then don't attempt to compile. - if (prediction == SpecNone) { - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); - break; - } - // If the CFA is tracking this variable and it found that the variable // cannot have been assigned, then don't attempt to proceed. if (value.isClear()) { - // FIXME: We should trap instead. - // https://bugs.webkit.org/show_bug.cgi?id=110383 - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); + m_compileOkay = false; break; } @@ -2233,13 +2267,23 @@ void SpeculativeJIT::compile(Node* node) break; } - case MovHint: - case ZombieHint: - case Check: { - RELEASE_ASSERT_NOT_REACHED(); + case MovHint: { + compileMovHint(m_currentNode); + noResult(node); break; } - + + case ZombieHint: { + recordSetLocal(m_currentNode->unlinkedLocal(), VirtualRegister(), DataFormatDead); + noResult(node); + break; + } + + case ExitOK: { + noResult(node); + break; + } + case SetLocal: { switch (node->variableAccessData()->flushFormat()) { case FlushedDouble: { @@ -2286,8 +2330,7 @@ void SpeculativeJIT::compile(Node* node) break; } - case FlushedJSValue: - case FlushedArguments: { + case FlushedJSValue: { JSValueOperand value(this, node->child1()); m_jit.store64(value.gpr(), JITCompiler::addressFor(node->machineLocal())); noResult(node); @@ -2296,7 +2339,7 @@ void SpeculativeJIT::compile(Node* node) } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad flush format"); break; } @@ -2308,60 +2351,19 @@ void SpeculativeJIT::compile(Node* node) // But it may be profitable to use this as a hook to run speculation checks // on arguments, thereby allowing us to trivially eliminate such checks if // the argument is not used. + recordSetLocal(dataFormatFor(node->variableAccessData()->flushFormat())); break; case BitAnd: case BitOr: case BitXor: - if (isInt32Constant(node->child1().node())) { - SpeculateInt32Operand op2(this, node->child2()); - GPRTemporary result(this, Reuse, op2); - - bitOp(op, valueOfInt32Constant(node->child1().node()), op2.gpr(), result.gpr()); - - int32Result(result.gpr(), node); - } else if (isInt32Constant(node->child2().node())) { - SpeculateInt32Operand op1(this, node->child1()); - GPRTemporary result(this, Reuse, op1); - - bitOp(op, valueOfInt32Constant(node->child2().node()), op1.gpr(), result.gpr()); - - int32Result(result.gpr(), node); - } else { - SpeculateInt32Operand op1(this, node->child1()); - SpeculateInt32Operand op2(this, node->child2()); - GPRTemporary result(this, Reuse, op1, op2); - - GPRReg reg1 = op1.gpr(); - GPRReg reg2 = op2.gpr(); - bitOp(op, reg1, reg2, result.gpr()); - - int32Result(result.gpr(), node); - } + compileBitwiseOp(node); break; case BitRShift: case BitLShift: case BitURShift: - if (isInt32Constant(node->child2().node())) { - SpeculateInt32Operand op1(this, node->child1()); - GPRTemporary result(this, Reuse, op1); - - shiftOp(op, op1.gpr(), valueOfInt32Constant(node->child2().node()) & 0x1f, result.gpr()); - - int32Result(result.gpr(), node); - } else { - // Do not allow shift amount to be used as the result, MacroAssembler does not permit this. - SpeculateInt32Operand op1(this, node->child1()); - SpeculateInt32Operand op2(this, node->child2()); - GPRTemporary result(this, Reuse, op1); - - GPRReg reg1 = op1.gpr(); - GPRReg reg2 = op2.gpr(); - shiftOp(op, reg1, reg2, result.gpr()); - - int32Result(result.gpr(), node); - } + compileShiftOp(node); break; case UInt32ToNumber: { @@ -2379,48 +2381,101 @@ void SpeculativeJIT::compile(Node* node) break; } - case Int32ToDouble: { - compileInt32ToDouble(node); + case DoubleRep: { + compileDoubleRep(node); break; } - case Int52ToValue: { - JSValueOperand operand(this, node->child1()); - GPRTemporary result(this, Reuse, operand); - m_jit.move(operand.gpr(), result.gpr()); - jsValueResult(result.gpr(), node); + case ValueRep: { + compileValueRep(node); break; } - case Int52ToDouble: { - SpeculateDoubleOperand operand(this, node->child1()); - FPRTemporary result(this, operand); - m_jit.moveDouble(operand.fpr(), result.fpr()); - doubleResult(result.fpr(), node); + case Int52Rep: { + switch (node->child1().useKind()) { + case Int32Use: { + SpeculateInt32Operand operand(this, node->child1()); + GPRTemporary result(this, Reuse, operand); + + m_jit.signExtend32ToPtr(operand.gpr(), result.gpr()); + + strictInt52Result(result.gpr(), node); + break; + } + + case AnyIntUse: { + GPRTemporary result(this); + GPRReg resultGPR = result.gpr(); + + convertAnyInt(node->child1(), resultGPR); + + strictInt52Result(resultGPR, node); + break; + } + + case DoubleRepAnyIntUse: { + SpeculateDoubleOperand value(this, node->child1()); + FPRReg valueFPR = value.fpr(); + + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + + callOperation(operationConvertDoubleToInt52, resultGPR, valueFPR); + + DFG_TYPE_CHECK_WITH_EXIT_KIND(Int52Overflow, + JSValueRegs(), node->child1(), SpecAnyIntAsDouble, + m_jit.branch64( + JITCompiler::Equal, resultGPR, + JITCompiler::TrustedImm64(JSValue::notInt52))); + + strictInt52Result(resultGPR, node); + break; + } + + default: + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); + } break; } - - case ValueAdd: { - JSValueOperand op1(this, node->child1()); - JSValueOperand op2(this, node->child2()); + + case ValueAdd: + compileValueAdd(node); + break; + + case StrCat: { + JSValueOperand op1(this, node->child1(), ManualOperandSpeculation); + JSValueOperand op2(this, node->child2(), ManualOperandSpeculation); + JSValueOperand op3(this, node->child3(), ManualOperandSpeculation); GPRReg op1GPR = op1.gpr(); GPRReg op2GPR = op2.gpr(); + GPRReg op3GPR; + if (node->child3()) + op3GPR = op3.gpr(); + else + op3GPR = InvalidGPRReg; flushRegisters(); - - GPRResult result(this); - if (isKnownNotNumber(node->child1().node()) || isKnownNotNumber(node->child2().node())) - callOperation(operationValueAddNotNumber, result.gpr(), op1GPR, op2GPR); + + GPRFlushedCallResult result(this); + if (node->child3()) + callOperation(operationStrCat3, result.gpr(), op1GPR, op2GPR, op3GPR); else - callOperation(operationValueAdd, result.gpr(), op1GPR, op2GPR); + callOperation(operationStrCat2, result.gpr(), op1GPR, op2GPR); + m_jit.exceptionCheck(); - jsValueResult(result.gpr(), node); + cellResult(result.gpr(), node); break; } - + case ArithAdd: - compileAdd(node); + compileArithAdd(node); + break; + + case ArithClz32: + compileArithClz32(node); break; case MakeRope: @@ -2449,37 +2504,9 @@ void SpeculativeJIT::compile(Node* node) break; } - case ArithAbs: { - switch (node->child1().useKind()) { - case Int32Use: { - SpeculateStrictInt32Operand op1(this, node->child1()); - GPRTemporary result(this); - GPRTemporary scratch(this); - - m_jit.move(op1.gpr(), result.gpr()); - m_jit.rshift32(result.gpr(), MacroAssembler::TrustedImm32(31), scratch.gpr()); - m_jit.add32(scratch.gpr(), result.gpr()); - m_jit.xor32(scratch.gpr(), result.gpr()); - speculationCheck(Overflow, JSValueRegs(), 0, m_jit.branch32(MacroAssembler::Equal, result.gpr(), MacroAssembler::TrustedImm32(1 << 31))); - int32Result(result.gpr(), node); - break; - } - - case NumberUse: { - SpeculateDoubleOperand op1(this, node->child1()); - FPRTemporary result(this); - - m_jit.absDouble(op1.fpr(), result.fpr()); - doubleResult(result.fpr(), node); - break; - } - - default: - RELEASE_ASSERT_NOT_REACHED(); - break; - } + case ArithAbs: + compileArithAbs(node); break; - } case ArithMin: case ArithMax: { @@ -2503,7 +2530,7 @@ void SpeculativeJIT::compile(Node* node) break; } - case NumberUse: { + case DoubleRepUse: { SpeculateDoubleOperand op1(this, node->child1()); SpeculateDoubleOperand op2(this, node->child2()); FPRTemporary result(this, op1); @@ -2542,45 +2569,50 @@ void SpeculativeJIT::compile(Node* node) } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } break; } - - case ArithSqrt: { - SpeculateDoubleOperand op1(this, node->child1()); - FPRTemporary result(this, op1); - - m_jit.sqrtDouble(op1.fpr(), result.fpr()); - - doubleResult(result.fpr(), node); + + case ArithPow: + compileArithPow(node); break; - } - - case ArithSin: { - SpeculateDoubleOperand op1(this, node->child1()); - FPRReg op1FPR = op1.fpr(); - flushRegisters(); - - FPRResult result(this); - callOperation(sin, result.fpr(), op1FPR); - doubleResult(result.fpr(), node); + case ArithSqrt: + compileArithSqrt(node); break; - } - case ArithCos: { - SpeculateDoubleOperand op1(this, node->child1()); - FPRReg op1FPR = op1.fpr(); + case ArithFRound: + compileArithFRound(node); + break; - flushRegisters(); - - FPRResult result(this); - callOperation(cos, result.fpr(), op1FPR); - doubleResult(result.fpr(), node); + case ArithRandom: + compileArithRandom(node); + break; + + case ArithRound: + case ArithFloor: + case ArithCeil: + case ArithTrunc: + compileArithRounding(node); + break; + + case ArithSin: + compileArithSin(node); + break; + + case ArithCos: + compileArithCos(node); + break; + + case ArithTan: + compileArithTan(node); + break; + + case ArithLog: + compileArithLog(node); break; - } case LogicalNot: compileLogicalNot(node); @@ -2605,27 +2637,20 @@ void SpeculativeJIT::compile(Node* node) if (compare(node, JITCompiler::GreaterThanOrEqual, JITCompiler::DoubleGreaterThanOrEqual, operationCompareGreaterEq)) return; break; - - case CompareEqConstant: - ASSERT(isNullConstant(node->child2().node())); - if (nonSpeculativeCompareNull(node, node->child1())) - return; - break; case CompareEq: if (compare(node, JITCompiler::Equal, JITCompiler::DoubleEqual, operationCompareEq)) return; break; - case CompareStrictEqConstant: - if (compileStrictEqForConstant(node, node->child1(), valueOfJSConstant(node->child2().node()))) - return; - break; - case CompareStrictEq: if (compileStrictEq(node)) return; break; + + case CompareEqPtr: + compileCompareEqPtr(node); + break; case StringCharCodeAt: { compileGetCharCodeAt(node); @@ -2658,9 +2683,24 @@ void SpeculativeJIT::compile(Node* node) switch (node->arrayMode().type()) { case Array::SelectUsingPredictions: case Array::ForceExit: - RELEASE_ASSERT_NOT_REACHED(); - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); + DFG_CRASH(m_jit.graph(), node, "Bad array mode type"); break; + case Array::Undecided: { + SpeculateStrictInt32Operand index(this, node->child2()); + GPRTemporary result(this, Reuse, index); + GPRReg indexGPR = index.gpr(); + GPRReg resultGPR = result.gpr(); + + speculationCheck(OutOfBounds, JSValueRegs(), node, + m_jit.branch32(MacroAssembler::LessThan, indexGPR, MacroAssembler::TrustedImm32(0))); + + use(node->child1()); + index.use(); + + m_jit.move(MacroAssembler::TrustedImm64(ValueUndefined), resultGPR); + jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); + break; + } case Array::Generic: { JSValueOperand base(this, node->child1()); JSValueOperand property(this, node->child2()); @@ -2668,8 +2708,9 @@ void SpeculativeJIT::compile(Node* node) GPRReg propertyGPR = property.gpr(); flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation(operationGetByVal, result.gpr(), baseGPR, propertyGPR); + m_jit.exceptionCheck(); jsValueResult(result.gpr(), node); break; @@ -2690,7 +2731,17 @@ void SpeculativeJIT::compile(Node* node) GPRTemporary result(this); m_jit.load64(MacroAssembler::BaseIndex(storageReg, propertyReg, MacroAssembler::TimesEight), result.gpr()); - speculationCheck(LoadFromHole, JSValueRegs(), 0, m_jit.branchTest64(MacroAssembler::Zero, result.gpr())); + if (node->arrayMode().isSaneChain()) { + ASSERT(node->arrayMode().type() == Array::Contiguous); + JITCompiler::Jump notHole = m_jit.branchTest64( + MacroAssembler::NonZero, result.gpr()); + m_jit.move(TrustedImm64(JSValue::encode(jsUndefined())), result.gpr()); + notHole.link(&m_jit); + } else { + speculationCheck( + LoadFromHole, JSValueRegs(), 0, + m_jit.branchTest64(MacroAssembler::Zero, result.gpr())); + } jsValueResult(result.gpr(), node, node->arrayMode().type() == Array::Int32 ? DataFormatJSInt32 : DataFormatJS); break; } @@ -2833,8 +2884,11 @@ void SpeculativeJIT::compile(Node* node) case Array::String: compileGetByValOnString(node); break; - case Array::Arguments: - compileGetByValOnArguments(node); + case Array::DirectArguments: + compileGetByValOnDirectArguments(node); + break; + case Array::ScopedArguments: + compileGetByValOnScopedArguments(node); break; default: { TypedArrayType type = node->arrayMode().typedArrayType(); @@ -2846,6 +2900,25 @@ void SpeculativeJIT::compile(Node* node) break; } + case GetByValWithThis: { + JSValueOperand base(this, node->child1()); + GPRReg baseGPR = base.gpr(); + JSValueOperand thisValue(this, node->child2()); + GPRReg thisValueGPR = thisValue.gpr(); + JSValueOperand subscript(this, node->child3()); + GPRReg subscriptGPR = subscript.gpr(); + + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + callOperation(operationGetByValWithThis, resultGPR, baseGPR, thisValueGPR, subscriptGPR); + m_jit.exceptionCheck(); + + jsValueResult(resultGPR, node); + break; + } + case PutByValDirect: case PutByVal: case PutByValAlias: { @@ -2860,12 +2933,10 @@ void SpeculativeJIT::compile(Node* node) switch (arrayMode.type()) { case Array::SelectUsingPredictions: case Array::ForceExit: - RELEASE_ASSERT_NOT_REACHED(); - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); - alreadyHandled = true; + DFG_CRASH(m_jit.graph(), node, "Bad array mode type"); break; case Array::Generic: { - RELEASE_ASSERT(node->op() == PutByVal); + DFG_ASSERT(m_jit.graph(), node, node->op() == PutByVal || node->op() == PutByValDirect); JSValueOperand arg1(this, child1); JSValueOperand arg2(this, child2); @@ -2875,9 +2946,10 @@ void SpeculativeJIT::compile(Node* node) GPRReg arg3GPR = arg3.gpr(); flushRegisters(); if (node->op() == PutByValDirect) - callOperation(m_jit.isStrictModeFor(node->codeOrigin) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict, arg1GPR, arg2GPR, arg3GPR); + callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValDirectStrict : operationPutByValDirectNonStrict, arg1GPR, arg2GPR, arg3GPR); else - callOperation(m_jit.isStrictModeFor(node->codeOrigin) ? operationPutByValStrict : operationPutByValNonStrict, arg1GPR, arg2GPR, arg3GPR); + callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValStrict : operationPutByValNonStrict, arg1GPR, arg2GPR, arg3GPR); + m_jit.exceptionCheck(); noResult(node); alreadyHandled = true; @@ -2911,7 +2983,7 @@ void SpeculativeJIT::compile(Node* node) if (arrayMode.type() == Array::Int32) { DFG_TYPE_CHECK( - JSValueRegs(valueReg), child3, SpecInt32, + JSValueRegs(valueReg), child3, SpecInt32Only, m_jit.branch64( MacroAssembler::Below, valueReg, GPRInfo::tagTypeNumberRegister)); } @@ -3066,47 +3138,6 @@ void SpeculativeJIT::compile(Node* node) break; } - case Array::Arguments: { - JSValueOperand value(this, child3); - GPRTemporary scratch(this); - GPRTemporary scratch2(this); - - GPRReg valueReg = value.gpr(); - GPRReg scratchReg = scratch.gpr(); - GPRReg scratch2Reg = scratch2.gpr(); - - if (!m_compileOkay) - return; - - // Two really lame checks. - speculationCheck( - Uncountable, JSValueSource(), 0, - m_jit.branch32( - MacroAssembler::AboveOrEqual, propertyReg, - MacroAssembler::Address(baseReg, Arguments::offsetOfNumArguments()))); - speculationCheck( - Uncountable, JSValueSource(), 0, - m_jit.branchTestPtr( - MacroAssembler::NonZero, - MacroAssembler::Address( - baseReg, Arguments::offsetOfSlowArgumentData()))); - - m_jit.move(propertyReg, scratch2Reg); - m_jit.signExtend32ToPtr(scratch2Reg, scratch2Reg); - m_jit.loadPtr( - MacroAssembler::Address(baseReg, Arguments::offsetOfRegisters()), - scratchReg); - - m_jit.store64( - valueReg, - MacroAssembler::BaseIndex( - scratchReg, scratch2Reg, MacroAssembler::TimesEight, - CallFrame::thisArgumentOffset() * sizeof(Register) + sizeof(Register))); - - noResult(node); - break; - } - default: { TypedArrayType type = arrayMode.typedArrayType(); if (isInt(type)) @@ -3119,52 +3150,216 @@ void SpeculativeJIT::compile(Node* node) } case RegExpExec: { - if (compileRegExpExec(node)) - return; - if (!node->adjustedRefCount()) { - SpeculateCellOperand base(this, node->child1()); - SpeculateCellOperand argument(this, node->child2()); + bool sample = false; + + if (sample) + m_jit.incrementSuperSamplerCount(); + + SpeculateCellOperand globalObject(this, node->child1()); + GPRReg globalObjectGPR = globalObject.gpr(); + + if (node->child2().useKind() == RegExpObjectUse) { + if (node->child3().useKind() == StringUse) { + SpeculateCellOperand base(this, node->child2()); + SpeculateCellOperand argument(this, node->child3()); + GPRReg baseGPR = base.gpr(); + GPRReg argumentGPR = argument.gpr(); + speculateRegExpObject(node->child2(), baseGPR); + speculateString(node->child3(), argumentGPR); + + flushRegisters(); + GPRFlushedCallResult result(this); + callOperation(operationRegExpExecString, result.gpr(), globalObjectGPR, baseGPR, argumentGPR); + m_jit.exceptionCheck(); + + jsValueResult(result.gpr(), node); + + if (sample) + m_jit.decrementSuperSamplerCount(); + break; + } + + SpeculateCellOperand base(this, node->child2()); + JSValueOperand argument(this, node->child3()); GPRReg baseGPR = base.gpr(); GPRReg argumentGPR = argument.gpr(); - + speculateRegExpObject(node->child2(), baseGPR); + flushRegisters(); - GPRResult result(this); - callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR); - - // Must use jsValueResult because otherwise we screw up register - // allocation, which thinks that this node has a result. + GPRFlushedCallResult result(this); + callOperation(operationRegExpExec, result.gpr(), globalObjectGPR, baseGPR, argumentGPR); + m_jit.exceptionCheck(); + jsValueResult(result.gpr(), node); + + if (sample) + m_jit.decrementSuperSamplerCount(); break; } - - SpeculateCellOperand base(this, node->child1()); - SpeculateCellOperand argument(this, node->child2()); + + JSValueOperand base(this, node->child2()); + JSValueOperand argument(this, node->child3()); GPRReg baseGPR = base.gpr(); GPRReg argumentGPR = argument.gpr(); flushRegisters(); - GPRResult result(this); - callOperation(operationRegExpExec, result.gpr(), baseGPR, argumentGPR); + GPRFlushedCallResult result(this); + callOperation(operationRegExpExecGeneric, result.gpr(), globalObjectGPR, baseGPR, argumentGPR); + m_jit.exceptionCheck(); jsValueResult(result.gpr(), node); + + if (sample) + m_jit.decrementSuperSamplerCount(); break; } case RegExpTest: { - SpeculateCellOperand base(this, node->child1()); - SpeculateCellOperand argument(this, node->child2()); + SpeculateCellOperand globalObject(this, node->child1()); + GPRReg globalObjectGPR = globalObject.gpr(); + + if (node->child2().useKind() == RegExpObjectUse) { + if (node->child3().useKind() == StringUse) { + SpeculateCellOperand base(this, node->child2()); + SpeculateCellOperand argument(this, node->child3()); + GPRReg baseGPR = base.gpr(); + GPRReg argumentGPR = argument.gpr(); + speculateRegExpObject(node->child2(), baseGPR); + speculateString(node->child3(), argumentGPR); + + flushRegisters(); + GPRFlushedCallResult result(this); + callOperation(operationRegExpTestString, result.gpr(), globalObjectGPR, baseGPR, argumentGPR); + m_jit.exceptionCheck(); + + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + jsValueResult(result.gpr(), node); + break; + } + + SpeculateCellOperand base(this, node->child2()); + JSValueOperand argument(this, node->child3()); + GPRReg baseGPR = base.gpr(); + GPRReg argumentGPR = argument.gpr(); + speculateRegExpObject(node->child2(), baseGPR); + + flushRegisters(); + GPRFlushedCallResult result(this); + callOperation(operationRegExpTest, result.gpr(), globalObjectGPR, baseGPR, argumentGPR); + m_jit.exceptionCheck(); + + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + jsValueResult(result.gpr(), node); + break; + } + + JSValueOperand base(this, node->child2()); + JSValueOperand argument(this, node->child3()); GPRReg baseGPR = base.gpr(); GPRReg argumentGPR = argument.gpr(); flushRegisters(); - GPRResult result(this); - callOperation(operationRegExpTest, result.gpr(), baseGPR, argumentGPR); + GPRFlushedCallResult result(this); + callOperation(operationRegExpTestGeneric, result.gpr(), globalObjectGPR, baseGPR, argumentGPR); + m_jit.exceptionCheck(); - // If we add a DataFormatBool, we should use it here. m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); jsValueResult(result.gpr(), node, DataFormatJSBoolean); break; } + + case StringReplace: + case StringReplaceRegExp: { + bool sample = false; + + if (sample) + m_jit.incrementSuperSamplerCount(); + + if (node->child1().useKind() == StringUse + && node->child2().useKind() == RegExpObjectUse + && node->child3().useKind() == StringUse) { + if (JSString* replace = node->child3()->dynamicCastConstant<JSString*>(*m_jit.vm())) { + if (!replace->length()) { + SpeculateCellOperand string(this, node->child1()); + SpeculateCellOperand regExp(this, node->child2()); + GPRReg stringGPR = string.gpr(); + GPRReg regExpGPR = regExp.gpr(); + speculateString(node->child1(), stringGPR); + speculateRegExpObject(node->child2(), regExpGPR); + + flushRegisters(); + GPRFlushedCallResult result(this); + callOperation( + operationStringProtoFuncReplaceRegExpEmptyStr, result.gpr(), stringGPR, + regExpGPR); + m_jit.exceptionCheck(); + cellResult(result.gpr(), node); + if (sample) + m_jit.decrementSuperSamplerCount(); + break; + } + } + + SpeculateCellOperand string(this, node->child1()); + SpeculateCellOperand regExp(this, node->child2()); + SpeculateCellOperand replace(this, node->child3()); + GPRReg stringGPR = string.gpr(); + GPRReg regExpGPR = regExp.gpr(); + GPRReg replaceGPR = replace.gpr(); + speculateString(node->child1(), stringGPR); + speculateRegExpObject(node->child2(), regExpGPR); + speculateString(node->child3(), replaceGPR); + + flushRegisters(); + GPRFlushedCallResult result(this); + callOperation( + operationStringProtoFuncReplaceRegExpString, result.gpr(), stringGPR, regExpGPR, + replaceGPR); + m_jit.exceptionCheck(); + cellResult(result.gpr(), node); + if (sample) + m_jit.decrementSuperSamplerCount(); + break; + } + + // If we fixed up the edge of child2, we inserted a Check(@child2, String). + OperandSpeculationMode child2SpeculationMode = AutomaticOperandSpeculation; + if (node->child2().useKind() == StringUse) + child2SpeculationMode = ManualOperandSpeculation; + + JSValueOperand string(this, node->child1()); + JSValueOperand search(this, node->child2(), child2SpeculationMode); + JSValueOperand replace(this, node->child3()); + GPRReg stringGPR = string.gpr(); + GPRReg searchGPR = search.gpr(); + GPRReg replaceGPR = replace.gpr(); + + flushRegisters(); + GPRFlushedCallResult result(this); + callOperation( + operationStringProtoFuncReplaceGeneric, result.gpr(), stringGPR, searchGPR, + replaceGPR); + m_jit.exceptionCheck(); + cellResult(result.gpr(), node); + if (sample) + m_jit.decrementSuperSamplerCount(); + break; + } + + case GetRegExpObjectLastIndex: { + compileGetRegExpObjectLastIndex(node); + break; + } + + case SetRegExpObjectLastIndex: { + compileSetRegExpObjectLastIndex(node); + break; + } + + case RecordRegExpCachedResult: { + compileRecordRegExpCachedResult(node); + break; + } case ArrayPush: { ASSERT(node->arrayMode().isJSArray()); @@ -3186,7 +3381,7 @@ void SpeculativeJIT::compile(Node* node) if (node->arrayMode().type() == Array::Int32) { DFG_TYPE_CHECK( - JSValueRegs(valueGPR), node->child2(), SpecInt32, + JSValueRegs(valueGPR), node->child2(), SpecInt32Only, m_jit.branch64( MacroAssembler::Below, valueGPR, GPRInfo::tagTypeNumberRegister)); } @@ -3200,7 +3395,7 @@ void SpeculativeJIT::compile(Node* node) addSlowPathGenerator( slowPathCall( - slowPath, this, operationArrayPush, NoResult, storageLengthGPR, + slowPath, this, operationArrayPush, storageLengthGPR, valueGPR, baseGPR)); jsValueResult(storageLengthGPR, node); @@ -3212,7 +3407,7 @@ void SpeculativeJIT::compile(Node* node) FPRReg valueFPR = value.fpr(); DFG_TYPE_CHECK( - JSValueRegs(), node->child2(), SpecFullRealNumber, + JSValueRegs(), node->child2(), SpecDoubleReal, m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, valueFPR, valueFPR)); m_jit.load32(MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength()), storageLengthGPR); @@ -3224,7 +3419,7 @@ void SpeculativeJIT::compile(Node* node) addSlowPathGenerator( slowPathCall( - slowPath, this, operationArrayPushDouble, NoResult, storageLengthGPR, + slowPath, this, operationArrayPushDouble, storageLengthGPR, valueFPR, baseGPR)); jsValueResult(storageLengthGPR, node); @@ -3264,6 +3459,11 @@ void SpeculativeJIT::compile(Node* node) } break; } + + case ArraySlice: { + compileArraySlice(node); + break; + } case ArrayPop: { ASSERT(node->arrayMode().isJSArray()); @@ -3299,7 +3499,7 @@ void SpeculativeJIT::compile(Node* node) // FIXME: This would not have to be here if changing the publicLength also zeroed the values between the old // length and the new length. m_jit.store64( - MacroAssembler::TrustedImm64((int64_t)0), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight)); + MacroAssembler::TrustedImm64(bitwise_cast<int64_t>(PNaN)), MacroAssembler::BaseIndex(storageGPR, storageLengthGPR, MacroAssembler::TimesEight)); slowCase = m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, tempFPR, tempFPR); boxDouble(tempFPR, valueGPR); } else { @@ -3366,7 +3566,7 @@ void SpeculativeJIT::compile(Node* node) } case DFG::Jump: { - jump(node->takenBlock()); + jump(node->targetBlock()); noResult(node); break; } @@ -3388,12 +3588,8 @@ void SpeculativeJIT::compile(Node* node) JSValueOperand op1(this, node->child1()); m_jit.move(op1.gpr(), GPRInfo::returnValueGPR); - // Grab the return address. - m_jit.emitGetReturnPCFromCallFrameHeaderPtr(GPRInfo::regT1); - // Restore our caller's "r". - m_jit.emitGetCallerFrameFromCallFrameHeaderPtr(GPRInfo::callFrameRegister); - // Return. - m_jit.restoreReturnAddressBeforeReturn(GPRInfo::regT1); + m_jit.emitRestoreCalleeSaves(); + m_jit.emitFunctionEpilogue(); m_jit.ret(); noResult(node); @@ -3401,71 +3597,121 @@ void SpeculativeJIT::compile(Node* node) } case Throw: - case ThrowReferenceError: { + case ThrowStaticError: { // We expect that throw statements are rare and are intended to exit the code block // anyway, so we just OSR back to the old JIT for now. terminateSpeculativeExecution(Uncountable, JSValueRegs(), 0); break; } + case BooleanToNumber: { + switch (node->child1().useKind()) { + case BooleanUse: { + JSValueOperand value(this, node->child1(), ManualOperandSpeculation); + GPRTemporary result(this); // FIXME: We could reuse, but on speculation fail would need recovery to restore tag (akin to add). + + m_jit.move(value.gpr(), result.gpr()); + m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), result.gpr()); + DFG_TYPE_CHECK( + JSValueRegs(value.gpr()), node->child1(), SpecBoolean, m_jit.branchTest64( + JITCompiler::NonZero, result.gpr(), TrustedImm32(static_cast<int32_t>(~1)))); + + int32Result(result.gpr(), node); + break; + } + + case UntypedUse: { + JSValueOperand value(this, node->child1()); + GPRTemporary result(this); + + if (!m_interpreter.needsTypeCheck(node->child1(), SpecBoolInt32 | SpecBoolean)) { + m_jit.move(value.gpr(), result.gpr()); + m_jit.and32(TrustedImm32(1), result.gpr()); + int32Result(result.gpr(), node); + break; + } + + m_jit.move(value.gpr(), result.gpr()); + m_jit.xor64(TrustedImm32(static_cast<int32_t>(ValueFalse)), result.gpr()); + JITCompiler::Jump isBoolean = m_jit.branchTest64( + JITCompiler::Zero, result.gpr(), TrustedImm32(static_cast<int32_t>(~1))); + m_jit.move(value.gpr(), result.gpr()); + JITCompiler::Jump done = m_jit.jump(); + isBoolean.link(&m_jit); + m_jit.or64(GPRInfo::tagTypeNumberRegister, result.gpr()); + done.link(&m_jit); + + jsValueResult(result.gpr(), node); + break; + } + + default: + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); + break; + } + break; + } + case ToPrimitive: { - RELEASE_ASSERT(node->child1().useKind() == UntypedUse); - JSValueOperand op1(this, node->child1()); - GPRTemporary result(this, Reuse, op1); + DFG_ASSERT(m_jit.graph(), node, node->child1().useKind() == UntypedUse); + JSValueOperand argument(this, node->child1()); + GPRTemporary result(this, Reuse, argument); - GPRReg op1GPR = op1.gpr(); + GPRReg argumentGPR = argument.gpr(); GPRReg resultGPR = result.gpr(); - op1.use(); + argument.use(); - if (!(m_state.forNode(node->child1()).m_type & ~(SpecFullNumber | SpecBoolean))) - m_jit.move(op1GPR, resultGPR); - else { - MacroAssembler::Jump alreadyPrimitive = m_jit.branchTest64(MacroAssembler::NonZero, op1GPR, GPRInfo::tagMaskRegister); - MacroAssembler::Jump notPrimitive = m_jit.branchPtr(MacroAssembler::NotEqual, MacroAssembler::Address(op1GPR, JSCell::structureOffset()), MacroAssembler::TrustedImmPtr(m_jit.vm()->stringStructure.get())); - - alreadyPrimitive.link(&m_jit); - m_jit.move(op1GPR, resultGPR); - - addSlowPathGenerator( - slowPathCall(notPrimitive, this, operationToPrimitive, resultGPR, op1GPR)); - } + MacroAssembler::Jump alreadyPrimitive = m_jit.branchIfNotCell(JSValueRegs(argumentGPR)); + MacroAssembler::Jump notPrimitive = m_jit.branchIfObject(argumentGPR); + + alreadyPrimitive.link(&m_jit); + m_jit.move(argumentGPR, resultGPR); + + addSlowPathGenerator( + slowPathCall(notPrimitive, this, operationToPrimitive, resultGPR, argumentGPR)); jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); break; } - - case ToString: { - if (node->child1().useKind() == UntypedUse) { - JSValueOperand op1(this, node->child1()); - GPRReg op1GPR = op1.gpr(); - - GPRResult result(this); - GPRReg resultGPR = result.gpr(); - + + case ToNumber: { + JSValueOperand argument(this, node->child1()); + GPRTemporary result(this, Reuse, argument); + + GPRReg argumentGPR = argument.gpr(); + GPRReg resultGPR = result.gpr(); + + argument.use(); + + // We have several attempts to remove ToNumber. But ToNumber still exists. + // It means that converting non-numbers to numbers by this ToNumber is not rare. + // Instead of the slow path generator, we emit callOperation here. + if (!(m_state.forNode(node->child1()).m_type & SpecBytecodeNumber)) { flushRegisters(); - - JITCompiler::Jump done; - if (node->child1()->prediction() & SpecString) { - JITCompiler::Jump slowPath1 = m_jit.branchTest64( - JITCompiler::NonZero, op1GPR, GPRInfo::tagMaskRegister); - JITCompiler::Jump slowPath2 = m_jit.branchPtr( - JITCompiler::NotEqual, - JITCompiler::Address(op1GPR, JSCell::structureOffset()), - TrustedImmPtr(m_jit.vm()->stringStructure.get())); - m_jit.move(op1GPR, resultGPR); - done = m_jit.jump(); - slowPath1.link(&m_jit); - slowPath2.link(&m_jit); - } - callOperation(operationToString, resultGPR, op1GPR); - if (done.isSet()) - done.link(&m_jit); - cellResult(resultGPR, node); - break; + callOperation(operationToNumber, resultGPR, argumentGPR); + m_jit.exceptionCheck(); + } else { + MacroAssembler::Jump notNumber = m_jit.branchIfNotNumber(argumentGPR); + m_jit.move(argumentGPR, resultGPR); + MacroAssembler::Jump done = m_jit.jump(); + + notNumber.link(&m_jit); + silentSpillAllRegisters(resultGPR); + callOperation(operationToNumber, resultGPR, argumentGPR); + silentFillAllRegisters(resultGPR); + m_jit.exceptionCheck(); + + done.link(&m_jit); } + + jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); + break; + } - compileToStringOnCell(node); + case ToString: + case CallStringConstructor: { + compileToStringOrCallStringConstructor(node); break; } @@ -3475,10 +3721,10 @@ void SpeculativeJIT::compile(Node* node) } case NewArray: { - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); - if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) { - Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()); - RELEASE_ASSERT(structure->indexingType() == node->indexingType()); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) { + RegisteredStructure structure = m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())); + DFG_ASSERT(m_jit.graph(), node, structure->indexingType() == node->indexingType()); ASSERT( hasUndecided(structure->indexingType()) || hasInt32(structure->indexingType()) @@ -3493,7 +3739,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg resultGPR = result.gpr(); GPRReg storageGPR = storage.gpr(); - emitAllocateJSArray(resultGPR, structure, storageGPR, numElements); + emitAllocateRawObject(resultGPR, structure, storageGPR, numElements, numElements); // At this point, one way or another, resultGPR and storageGPR have pointers to // the JSArray and the Butterfly, respectively. @@ -3511,7 +3757,7 @@ void SpeculativeJIT::compile(Node* node) SpeculateDoubleOperand operand(this, use); FPRReg opFPR = operand.fpr(); DFG_TYPE_CHECK( - JSValueRegs(), use, SpecFullRealNumber, + JSValueRegs(), use, SpecDoubleReal, m_jit.branchDouble( MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR)); m_jit.storeDouble(opFPR, MacroAssembler::Address(storageGPR, sizeof(double) * operandIdx)); @@ -3523,7 +3769,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg opGPR = operand.gpr(); if (hasInt32(node->indexingType())) { DFG_TYPE_CHECK( - JSValueRegs(opGPR), use, SpecInt32, + JSValueRegs(opGPR), use, SpecInt32Only, m_jit.branch64( MacroAssembler::Below, opGPR, GPRInfo::tagTypeNumberRegister)); } @@ -3549,8 +3795,9 @@ void SpeculativeJIT::compile(Node* node) if (!node->numChildren()) { flushRegisters(); - GPRResult result(this); - callOperation(operationNewEmptyArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())); + GPRFlushedCallResult result(this); + callOperation(operationNewEmptyArray, result.gpr(), m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()))); + m_jit.exceptionCheck(); cellResult(result.gpr(), node); break; } @@ -3576,7 +3823,7 @@ void SpeculativeJIT::compile(Node* node) FPRReg opFPR = operand.fpr(); GPRReg scratchGPR = scratch.gpr(); DFG_TYPE_CHECK( - JSValueRegs(), use, SpecFullRealNumber, + JSValueRegs(), use, SpecDoubleReal, m_jit.branchDouble( MacroAssembler::DoubleNotEqualOrUnordered, opFPR, opFPR)); m_jit.boxDouble(opFPR, scratchGPR); @@ -3588,7 +3835,7 @@ void SpeculativeJIT::compile(Node* node) GPRReg opGPR = operand.gpr(); if (hasInt32(node->indexingType())) { DFG_TYPE_CHECK( - JSValueRegs(opGPR), use, SpecInt32, + JSValueRegs(opGPR), use, SpecInt32Only, m_jit.branch64( MacroAssembler::Below, opGPR, GPRInfo::tagTypeNumberRegister)); } @@ -3628,11 +3875,12 @@ void SpeculativeJIT::compile(Node* node) m_jit.storePtr(TrustedImmPtr(scratchSize), scratch.gpr()); } - GPRResult result(this); + GPRFlushedCallResult result(this); callOperation( - operationNewArray, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), + operationNewArray, result.gpr(), m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), static_cast<void*>(buffer), node->numChildren()); + m_jit.exceptionCheck(); if (scratchSize) { GPRTemporary scratch(this); @@ -3644,56 +3892,25 @@ void SpeculativeJIT::compile(Node* node) cellResult(result.gpr(), node, UseChildrenCalledExplicitly); break; } + + case NewArrayWithSpread: { + compileNewArrayWithSpread(node); + break; + } + + case Spread: { + compileSpread(node); + break; + } case NewArrayWithSize: { - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); - if (!globalObject->isHavingABadTime() && !hasArrayStorage(node->indexingType())) { + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(node->indexingType())) { SpeculateStrictInt32Operand size(this, node->child1()); GPRTemporary result(this); - GPRTemporary storage(this); - GPRTemporary scratch(this); - GPRTemporary scratch2(this); - GPRReg sizeGPR = size.gpr(); GPRReg resultGPR = result.gpr(); - GPRReg storageGPR = storage.gpr(); - GPRReg scratchGPR = scratch.gpr(); - GPRReg scratch2GPR = scratch2.gpr(); - - MacroAssembler::JumpList slowCases; - slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX))); - - ASSERT((1 << 3) == sizeof(JSValue)); - m_jit.move(sizeGPR, scratchGPR); - m_jit.lshift32(TrustedImm32(3), scratchGPR); - m_jit.add32(TrustedImm32(sizeof(IndexingHeader)), scratchGPR, resultGPR); - slowCases.append( - emitAllocateBasicStorage(resultGPR, storageGPR)); - m_jit.subPtr(scratchGPR, storageGPR); - Structure* structure = globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()); - emitAllocateJSObject<JSArray>(resultGPR, ImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases); - - m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength())); - m_jit.store32(sizeGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfVectorLength())); - - if (hasDouble(node->indexingType())) { - m_jit.move(TrustedImm64(bitwise_cast<int64_t>(QNaN)), scratchGPR); - m_jit.move(sizeGPR, scratch2GPR); - MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, scratch2GPR); - MacroAssembler::Label loop = m_jit.label(); - m_jit.sub32(TrustedImm32(1), scratch2GPR); - m_jit.store64(scratchGPR, MacroAssembler::BaseIndex(storageGPR, scratch2GPR, MacroAssembler::TimesEight)); - m_jit.branchTest32(MacroAssembler::NonZero, scratch2GPR).linkTo(loop, &m_jit); - done.link(&m_jit); - } - - addSlowPathGenerator(adoptPtr( - new CallArrayAllocatorWithVariableSizeSlowPathGenerator( - slowCases, this, operationNewArrayWithSize, resultGPR, - globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), - globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage), - sizeGPR))); - + compileAllocateNewArrayWithSize(globalObject, resultGPR, sizeGPR, node->indexingType()); cellResult(resultGPR, node); break; } @@ -3701,24 +3918,25 @@ void SpeculativeJIT::compile(Node* node) SpeculateStrictInt32Operand size(this, node->child1()); GPRReg sizeGPR = size.gpr(); flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); GPRReg structureGPR = selectScratchGPR(sizeGPR); - MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_SPARSE_ARRAY_INDEX)); - m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), structureGPR); + MacroAssembler::Jump bigLength = m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)); + m_jit.move(TrustedImmPtr(m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()))), structureGPR); MacroAssembler::Jump done = m_jit.jump(); bigLength.link(&m_jit); - m_jit.move(TrustedImmPtr(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)), structureGPR); + m_jit.move(TrustedImmPtr(m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage))), structureGPR); done.link(&m_jit); - callOperation(operationNewArrayWithSize, resultGPR, structureGPR, sizeGPR); + callOperation(operationNewArrayWithSize, resultGPR, structureGPR, sizeGPR, nullptr); + m_jit.exceptionCheck(); cellResult(resultGPR, node); break; } case NewArrayBuffer: { - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); IndexingType indexingType = node->indexingType(); - if (!globalObject->isHavingABadTime() && !hasArrayStorage(indexingType)) { + if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(indexingType)) { unsigned numElements = node->numConstants(); GPRTemporary result(this); @@ -3727,9 +3945,9 @@ void SpeculativeJIT::compile(Node* node) GPRReg resultGPR = result.gpr(); GPRReg storageGPR = storage.gpr(); - emitAllocateJSArray(resultGPR, globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType), storageGPR, numElements); + emitAllocateRawObject(resultGPR, m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType)), storageGPR, numElements, numElements); - RELEASE_ASSERT(indexingType & IsArray); + DFG_ASSERT(m_jit.graph(), node, indexingType & IsArray); JSValue* data = m_jit.codeBlock()->constantBuffer(node->startConstant()); if (indexingType == ArrayWithDouble) { for (unsigned index = 0; index < node->numConstants(); ++index) { @@ -3751,9 +3969,10 @@ void SpeculativeJIT::compile(Node* node) } flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); - callOperation(operationNewArrayBuffer, result.gpr(), globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType()), node->startConstant(), node->numConstants()); + callOperation(operationNewArrayBuffer, result.gpr(), m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(node->indexingType())), node->startConstant(), node->numConstants()); + m_jit.exceptionCheck(); cellResult(result.gpr(), node); break; @@ -3770,20 +3989,21 @@ void SpeculativeJIT::compile(Node* node) flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); - JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->codeOrigin); + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); callOperation( operationNewTypedArrayWithOneArgumentForType(node->typedArrayType()), - resultGPR, globalObject->typedArrayStructure(node->typedArrayType()), + resultGPR, m_jit.graph().registerStructure(globalObject->typedArrayStructureConcurrently(node->typedArrayType())), argumentGPR); + m_jit.exceptionCheck(); cellResult(resultGPR, node); break; } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } break; @@ -3791,14 +4011,21 @@ void SpeculativeJIT::compile(Node* node) case NewRegexp: { flushRegisters(); - GPRResult result(this); + GPRFlushedCallResult result(this); - callOperation(operationNewRegexp, result.gpr(), m_jit.codeBlock()->regexp(node->regexpIndex())); + RegExp* regexp = node->castOperand<RegExp*>(); + callOperation(operationNewRegexp, result.gpr(), regexp); + m_jit.exceptionCheck(); cellResult(result.gpr(), node); break; } - + + case CallObjectConstructor: { + compileCallObjectConstructor(node); + break; + } + case ToThis: { ASSERT(node->child1().useKind() == UntypedUse); JSValueOperand thisValue(this, node->child1()); @@ -3807,17 +4034,15 @@ void SpeculativeJIT::compile(Node* node) GPRReg tempGPR = temp.gpr(); MacroAssembler::JumpList slowCases; - slowCases.append(m_jit.branchTest64( - MacroAssembler::NonZero, thisValueGPR, GPRInfo::tagMaskRegister)); - m_jit.loadPtr( - MacroAssembler::Address(thisValueGPR, JSCell::structureOffset()), tempGPR); - slowCases.append(m_jit.branch8( - MacroAssembler::NotEqual, - MacroAssembler::Address(tempGPR, Structure::typeInfoTypeOffset()), - TrustedImm32(FinalObjectType))); + slowCases.append(m_jit.branchIfNotCell(JSValueRegs(thisValueGPR))); + slowCases.append( + m_jit.branchTest8( + MacroAssembler::NonZero, + MacroAssembler::Address(thisValueGPR, JSCell::typeInfoFlagsOffset()), + MacroAssembler::TrustedImm32(OverridesToThis))); m_jit.move(thisValueGPR, tempGPR); J_JITOperation_EJ function; - if (m_jit.graph().executableFor(node->codeOrigin)->isStrictMode()) + if (m_jit.graph().executableFor(node->origin.semantic)->isStrictMode()) function = operationToThisStrict; else function = operationToThis; @@ -3846,13 +4071,26 @@ void SpeculativeJIT::compile(Node* node) GPRReg allocatorGPR = allocator.gpr(); GPRReg structureGPR = structure.gpr(); GPRReg scratchGPR = scratch.gpr(); + // Rare data is only used to access the allocator & structure + // We can avoid using an additional GPR this way + GPRReg rareDataGPR = structureGPR; + GPRReg inlineCapacityGPR = rareDataGPR; MacroAssembler::JumpList slowPath; - m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR); - m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR); + slowPath.append(m_jit.branch8(JITCompiler::NotEqual, + JITCompiler::Address(calleeGPR, JSCell::typeInfoTypeOffset()), TrustedImm32(JSFunctionType))); + m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfRareData()), rareDataGPR); + slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, rareDataGPR)); + m_jit.loadPtr(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfObjectAllocationProfile() + ObjectAllocationProfile::offsetOfAllocator()), allocatorGPR); + m_jit.loadPtr(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfObjectAllocationProfile() + ObjectAllocationProfile::offsetOfStructure()), structureGPR); slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, allocatorGPR)); - emitAllocateJSObject(resultGPR, allocatorGPR, structureGPR, TrustedImmPtr(0), scratchGPR, slowPath); + emitAllocateJSObject(resultGPR, nullptr, allocatorGPR, structureGPR, TrustedImmPtr(0), scratchGPR, slowPath); + + m_jit.loadPtr(JITCompiler::Address(calleeGPR, JSFunction::offsetOfRareData()), rareDataGPR); + m_jit.load32(JITCompiler::Address(rareDataGPR, FunctionRareData::offsetOfObjectAllocationProfile() + ObjectAllocationProfile::offsetOfInlineCapacity()), inlineCapacityGPR); + m_jit.emitInitializeInlineStorage(resultGPR, inlineCapacityGPR); + m_jit.mutatorFence(); addSlowPathGenerator(slowPathCall(slowPath, this, operationCreateThis, resultGPR, calleeGPR, node->inlineCapacity())); @@ -3860,12 +4098,6 @@ void SpeculativeJIT::compile(Node* node) break; } - case AllocationProfileWatchpoint: - case TypedArrayWatchpoint: { - noResult(node); - break; - } - case NewObject: { GPRTemporary result(this); GPRTemporary allocator(this); @@ -3877,12 +4109,14 @@ void SpeculativeJIT::compile(Node* node) MacroAssembler::JumpList slowPath; - Structure* structure = node->structure(); + RegisteredStructure structure = node->structure(); size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity()); - MarkedAllocator* allocatorPtr = &m_jit.vm()->heap.allocatorForObjectWithoutDestructor(allocationSize); + MarkedAllocator* allocatorPtr = subspaceFor<JSFinalObject>(*m_jit.vm())->allocatorFor(allocationSize); m_jit.move(TrustedImmPtr(allocatorPtr), allocatorGPR); - emitAllocateJSObject(resultGPR, allocatorGPR, TrustedImmPtr(structure), TrustedImmPtr(0), scratchGPR, slowPath); + emitAllocateJSObject(resultGPR, allocatorPtr, allocatorGPR, TrustedImmPtr(structure), TrustedImmPtr(0), scratchGPR, slowPath); + m_jit.emitInitializeInlineStorage(resultGPR, structure->inlineCapacity()); + m_jit.mutatorFence(); addSlowPathGenerator(slowPathCall(slowPath, this, operationNewObject, resultGPR, structure)); @@ -3892,96 +4126,65 @@ void SpeculativeJIT::compile(Node* node) case GetCallee: { GPRTemporary result(this); - m_jit.loadPtr(JITCompiler::addressFor(JSStack::Callee), result.gpr()); + m_jit.loadPtr(JITCompiler::addressFor(CallFrameSlot::callee), result.gpr()); cellResult(result.gpr(), node); break; } - case GetScope: { - SpeculateCellOperand function(this, node->child1()); - GPRTemporary result(this, Reuse, function); - m_jit.loadPtr(JITCompiler::Address(function.gpr(), JSFunction::offsetOfScopeChain()), result.gpr()); - cellResult(result.gpr(), node); + case GetArgumentCountIncludingThis: { + GPRTemporary result(this); + m_jit.load32(JITCompiler::payloadFor(CallFrameSlot::argumentCount), result.gpr()); + int32Result(result.gpr(), node); break; } - - case GetMyScope: { - GPRTemporary result(this); - GPRReg resultGPR = result.gpr(); - m_jit.loadPtr(JITCompiler::addressFor(JSStack::ScopeChain), resultGPR); - cellResult(resultGPR, node); + case GetRestLength: { + compileGetRestLength(node); break; } - case SkipTopScope: { - SpeculateCellOperand scope(this, node->child1()); - GPRTemporary result(this, Reuse, scope); - GPRReg resultGPR = result.gpr(); - m_jit.move(scope.gpr(), resultGPR); - JITCompiler::Jump activationNotCreated = - m_jit.branchTest64( - JITCompiler::Zero, - JITCompiler::addressFor( - static_cast<VirtualRegister>(m_jit.graph().machineActivationRegister()))); - m_jit.loadPtr(JITCompiler::Address(resultGPR, JSScope::offsetOfNext()), resultGPR); - activationNotCreated.link(&m_jit); - cellResult(resultGPR, node); + case GetScope: + compileGetScope(node); break; - } - - case SkipScope: { - SpeculateCellOperand scope(this, node->child1()); - GPRTemporary result(this, Reuse, scope); - m_jit.loadPtr(JITCompiler::Address(scope.gpr(), JSScope::offsetOfNext()), result.gpr()); - cellResult(result.gpr(), node); + + case SkipScope: + compileSkipScope(node); break; - } - - case GetClosureRegisters: { - if (WriteBarrierBase<Unknown>* registers = m_jit.graph().tryGetRegisters(node->child1().node())) { - GPRTemporary result(this); - GPRReg resultGPR = result.gpr(); - m_jit.move(TrustedImmPtr(registers), resultGPR); - storageResult(resultGPR, node); - break; - } - - SpeculateCellOperand scope(this, node->child1()); - GPRTemporary result(this); - GPRReg scopeGPR = scope.gpr(); - GPRReg resultGPR = result.gpr(); - m_jit.loadPtr(JITCompiler::Address(scopeGPR, JSVariableObject::offsetOfRegisters()), resultGPR); - storageResult(resultGPR, node); + case GetGlobalObject: + compileGetGlobalObject(node); break; - } + case GetClosureVar: { - StorageOperand registers(this, node->child1()); + SpeculateCellOperand base(this, node->child1()); GPRTemporary result(this); - GPRReg registersGPR = registers.gpr(); + GPRReg baseGPR = base.gpr(); GPRReg resultGPR = result.gpr(); - m_jit.load64(JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register)), resultGPR); + m_jit.load64(JITCompiler::Address(baseGPR, JSEnvironmentRecord::offsetOfVariable(node->scopeOffset())), resultGPR); jsValueResult(resultGPR, node); break; } case PutClosureVar: { - StorageOperand registers(this, node->child2()); - JSValueOperand value(this, node->child3()); + SpeculateCellOperand base(this, node->child1()); + JSValueOperand value(this, node->child2()); - GPRReg registersGPR = registers.gpr(); + GPRReg baseGPR = base.gpr(); GPRReg valueGPR = value.gpr(); - speculate(node, node->child1()); - - m_jit.store64(valueGPR, JITCompiler::Address(registersGPR, node->varNumber() * sizeof(Register))); + m_jit.store64(valueGPR, JITCompiler::Address(baseGPR, JSEnvironmentRecord::offsetOfVariable(node->scopeOffset()))); noResult(node); break; } - case GetById: { - ASSERT(node->prediction()); + case TryGetById: { + compileTryGetById(node); + break; + } + + case GetById: { + // FIXME https://bugs.webkit.org/show_bug.cgi?id=161158 + // dedup with SpeculativeJIT::compileTryGetById and 32-bit version of this. switch (node->child1().useKind()) { case CellUse: { SpeculateCellOperand base(this, node->child1()); @@ -3992,7 +4195,7 @@ void SpeculativeJIT::compile(Node* node) base.use(); - cachedGetById(node->codeOrigin, baseGPR, resultGPR, node->identifierNumber()); + cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber()); jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); break; @@ -4007,16 +4210,16 @@ void SpeculativeJIT::compile(Node* node) base.use(); - JITCompiler::Jump notCell = m_jit.branchTest64(JITCompiler::NonZero, baseGPR, GPRInfo::tagMaskRegister); + JITCompiler::Jump notCell = m_jit.branchIfNotCell(JSValueRegs(baseGPR)); - cachedGetById(node->codeOrigin, baseGPR, resultGPR, node->identifierNumber(), notCell); + cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber(), notCell); jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); break; } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } break; @@ -4033,7 +4236,7 @@ void SpeculativeJIT::compile(Node* node) SpeculateCellOperand base(this, node->child1()); GPRReg baseGPR = base.gpr(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); @@ -4041,7 +4244,7 @@ void SpeculativeJIT::compile(Node* node) flushRegisters(); - cachedGetById(node->codeOrigin, baseGPR, resultGPR, node->identifierNumber(), JITCompiler::Jump(), DontSpill); + cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber(), JITCompiler::Jump(), DontSpill); jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); break; @@ -4051,124 +4254,106 @@ void SpeculativeJIT::compile(Node* node) JSValueOperand base(this, node->child1()); GPRReg baseGPR = base.gpr(); - GPRResult result(this); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); base.use(); flushRegisters(); - JITCompiler::Jump notCell = m_jit.branchTest64(JITCompiler::NonZero, baseGPR, GPRInfo::tagMaskRegister); + JITCompiler::Jump notCell = m_jit.branchIfNotCell(JSValueRegs(baseGPR)); - cachedGetById(node->codeOrigin, baseGPR, resultGPR, node->identifierNumber(), notCell, DontSpill); + cachedGetById(node->origin.semantic, baseGPR, resultGPR, node->identifierNumber(), notCell, DontSpill); jsValueResult(resultGPR, node, UseChildrenCalledExplicitly); break; } default: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Bad use kind"); break; } break; } + case GetByIdWithThis: { + JSValueOperand base(this, node->child1()); + GPRReg baseGPR = base.gpr(); + JSValueOperand thisValue(this, node->child2()); + GPRReg thisValueGPR = thisValue.gpr(); + + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + callOperation(operationGetByIdWithThis, resultGPR, baseGPR, thisValueGPR, identifierUID(node->identifierNumber())); + m_jit.exceptionCheck(); + + jsValueResult(resultGPR, node); + break; + } + case GetArrayLength: compileGetArrayLength(node); break; - - case CheckFunction: { - SpeculateCellOperand function(this, node->child1()); - speculationCheck(BadFunction, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, function.gpr(), node->function())); - noResult(node); + + case DeleteById: { + compileDeleteById(node); break; } - - case CheckExecutable: { - SpeculateCellOperand function(this, node->child1()); - speculationCheck(BadExecutable, JSValueSource::unboxedCell(function.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, JITCompiler::Address(function.gpr(), JSFunction::offsetOfExecutable()), node->executable())); - noResult(node); + + case DeleteByVal: { + compileDeleteByVal(node); break; } - case CheckStructure: { - SpeculateCellOperand base(this, node->child1()); - - ASSERT(node->structureSet().size()); - - ExitKind exitKind; - if (node->child1()->op() == WeakJSConstant) - exitKind = BadWeakConstantCache; - else - exitKind = BadCache; - - if (node->structureSet().size() == 1) { - speculationCheck( - exitKind, JSValueSource::unboxedCell(base.gpr()), 0, - m_jit.branchWeakPtr( - JITCompiler::NotEqual, - JITCompiler::Address(base.gpr(), JSCell::structureOffset()), - node->structureSet()[0])); - } else { - GPRTemporary structure(this); - - m_jit.loadPtr(JITCompiler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr()); - - JITCompiler::JumpList done; - - for (size_t i = 0; i < node->structureSet().size() - 1; ++i) - done.append(m_jit.branchWeakPtr(JITCompiler::Equal, structure.gpr(), node->structureSet()[i])); - - speculationCheck( - exitKind, JSValueSource::unboxedCell(base.gpr()), 0, - m_jit.branchWeakPtr( - JITCompiler::NotEqual, structure.gpr(), node->structureSet().last())); - - done.link(&m_jit); - } - + case CheckCell: { + SpeculateCellOperand cell(this, node->child1()); + speculationCheck(BadCell, JSValueSource::unboxedCell(cell.gpr()), node->child1(), m_jit.branchWeakPtr(JITCompiler::NotEqual, cell.gpr(), node->cellOperand()->cell())); noResult(node); break; } - - case StructureTransitionWatchpoint: { - // There is a fascinating question here of what to do about array profiling. - // We *could* try to tell the OSR exit about where the base of the access is. - // The DFG will have kept it alive, though it may not be in a register, and - // we shouldn't really load it since that could be a waste. For now though, - // we'll just rely on the fact that when a watchpoint fires then that's - // quite a hint already. - - m_jit.addWeakReference(node->structure()); -#if !ASSERT_DISABLED - SpeculateCellOperand op1(this, node->child1()); - JITCompiler::Jump isOK = m_jit.branchPtr(JITCompiler::Equal, JITCompiler::Address(op1.gpr(), JSCell::structureOffset()), TrustedImmPtr(node->structure())); - m_jit.breakpoint(); - isOK.link(&m_jit); -#else - speculateCell(node->child1()); -#endif - + case CheckNotEmpty: { + JSValueOperand operand(this, node->child1()); + GPRReg gpr = operand.gpr(); + speculationCheck(TDZFailure, JSValueSource(), nullptr, m_jit.branchTest64(JITCompiler::Zero, gpr)); noResult(node); break; } + + case CheckStringIdent: + compileCheckStringIdent(node); + break; + + case GetExecutable: { + SpeculateCellOperand function(this, node->child1()); + GPRTemporary result(this, Reuse, function); + GPRReg functionGPR = function.gpr(); + GPRReg resultGPR = result.gpr(); + speculateCellType(node->child1(), functionGPR, SpecFunction, JSFunctionType); + m_jit.loadPtr(JITCompiler::Address(functionGPR, JSFunction::offsetOfExecutable()), resultGPR); + cellResult(resultGPR, node); + break; + } - case PhantomPutStructure: { - ASSERT(isKnownCell(node->child1().node())); - m_jit.jitCode()->common.notifyCompilingStructureTransition(m_jit.graph().m_plan, m_jit.codeBlock(), node); - noResult(node); + case CheckStructure: { + compileCheckStructure(node); break; } case PutStructure: { + RegisteredStructure oldStructure = node->transition()->previous; + RegisteredStructure newStructure = node->transition()->next; + m_jit.jitCode()->common.notifyCompilingStructureTransition(m_jit.graph().m_plan, m_jit.codeBlock(), node); SpeculateCellOperand base(this, node->child1()); - GPRTemporary scratch1(this); - GPRTemporary scratch2(this); GPRReg baseGPR = base.gpr(); - m_jit.storePtr(MacroAssembler::TrustedImmPtr(node->structureTransitionData().newStructure), MacroAssembler::Address(baseGPR, JSCell::structureOffset())); + ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType()); + ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type()); + ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags()); + m_jit.store32(MacroAssembler::TrustedImm32(newStructure->id()), MacroAssembler::Address(baseGPR, JSCell::structureIDOffset())); noResult(node); break; @@ -4182,18 +4367,13 @@ void SpeculativeJIT::compile(Node* node) compileReallocatePropertyStorage(node); break; - case GetButterfly: { - SpeculateCellOperand base(this, node->child1()); - GPRTemporary result(this, Reuse, base); - - GPRReg baseGPR = base.gpr(); - GPRReg resultGPR = result.gpr(); - - m_jit.loadPtr(JITCompiler::Address(baseGPR, JSObject::butterflyOffset()), resultGPR); + case NukeStructureAndSetButterfly: + compileNukeStructureAndSetButterfly(node); + break; - storageResult(resultGPR, node); + case GetButterfly: + compileGetButterfly(node); break; - } case GetIndexedPropertyStorage: { compileGetIndexedPropertyStorage(node); @@ -4210,14 +4390,15 @@ void SpeculativeJIT::compile(Node* node) break; } - case GetByOffset: { + case GetByOffset: + case GetGetterSetterByOffset: { StorageOperand storage(this, node->child1()); GPRTemporary result(this, Reuse, storage); GPRReg storageGPR = storage.gpr(); GPRReg resultGPR = result.gpr(); - StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()]; + StorageAccessData& storageAccessData = node->storageAccessData(); m_jit.load64(JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset)), resultGPR); @@ -4225,24 +4406,64 @@ void SpeculativeJIT::compile(Node* node) break; } + case GetGetter: { + SpeculateCellOperand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); + + GPRReg op1GPR = op1.gpr(); + GPRReg resultGPR = result.gpr(); + + m_jit.loadPtr(JITCompiler::Address(op1GPR, GetterSetter::offsetOfGetter()), resultGPR); + + cellResult(resultGPR, node); + break; + } + + case GetSetter: { + SpeculateCellOperand op1(this, node->child1()); + GPRTemporary result(this, Reuse, op1); + + GPRReg op1GPR = op1.gpr(); + GPRReg resultGPR = result.gpr(); + + m_jit.loadPtr(JITCompiler::Address(op1GPR, GetterSetter::offsetOfSetter()), resultGPR); + + cellResult(resultGPR, node); + break; + } + case PutByOffset: { StorageOperand storage(this, node->child1()); JSValueOperand value(this, node->child3()); - GPRTemporary scratch1(this); - GPRTemporary scratch2(this); GPRReg storageGPR = storage.gpr(); GPRReg valueGPR = value.gpr(); speculate(node, node->child2()); - StorageAccessData& storageAccessData = m_jit.graph().m_storageAccessData[node->storageAccessDataIndex()]; + StorageAccessData& storageAccessData = node->storageAccessData(); m_jit.store64(valueGPR, JITCompiler::Address(storageGPR, offsetRelativeToBase(storageAccessData.offset))); noResult(node); break; } + + case PutByIdFlush: { + SpeculateCellOperand base(this, node->child1()); + JSValueOperand value(this, node->child2()); + GPRTemporary scratch(this); + + GPRReg baseGPR = base.gpr(); + GPRReg valueGPR = value.gpr(); + GPRReg scratchGPR = scratch.gpr(); + flushRegisters(); + + cachedPutById(node->origin.semantic, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), NotDirect, MacroAssembler::Jump(), DontSpill); + + noResult(node); + break; + } case PutById: { SpeculateCellOperand base(this, node->child1()); @@ -4253,7 +4474,41 @@ void SpeculativeJIT::compile(Node* node) GPRReg valueGPR = value.gpr(); GPRReg scratchGPR = scratch.gpr(); - cachedPutById(node->codeOrigin, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), NotDirect); + cachedPutById(node->origin.semantic, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), NotDirect); + + noResult(node); + break; + } + + case PutByIdWithThis: { + JSValueOperand base(this, node->child1()); + GPRReg baseGPR = base.gpr(); + JSValueOperand thisValue(this, node->child2()); + GPRReg thisValueGPR = thisValue.gpr(); + JSValueOperand value(this, node->child3()); + GPRReg valueGPR = value.gpr(); + + flushRegisters(); + callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByIdWithThisStrict : operationPutByIdWithThis, NoResult, baseGPR, thisValueGPR, valueGPR, identifierUID(node->identifierNumber())); + m_jit.exceptionCheck(); + + noResult(node); + break; + } + + case PutByValWithThis: { + JSValueOperand base(this, m_jit.graph().varArgChild(node, 0)); + GPRReg baseGPR = base.gpr(); + JSValueOperand thisValue(this, m_jit.graph().varArgChild(node, 1)); + GPRReg thisValueGPR = thisValue.gpr(); + JSValueOperand property(this, m_jit.graph().varArgChild(node, 2)); + GPRReg propertyGPR = property.gpr(); + JSValueOperand value(this, m_jit.graph().varArgChild(node, 3)); + GPRReg valueGPR = value.gpr(); + + flushRegisters(); + callOperation(m_jit.isStrictModeFor(node->origin.semantic) ? operationPutByValWithThisStrict : operationPutByValWithThis, NoResult, baseGPR, thisValueGPR, propertyGPR, valueGPR); + m_jit.exceptionCheck(); noResult(node); break; @@ -4268,90 +4523,121 @@ void SpeculativeJIT::compile(Node* node) GPRReg valueGPR = value.gpr(); GPRReg scratchGPR = scratch.gpr(); - cachedPutById(node->codeOrigin, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), Direct); + cachedPutById(node->origin.semantic, baseGPR, valueGPR, scratchGPR, node->identifierNumber(), Direct); noResult(node); break; } + case PutGetterById: + case PutSetterById: { + compilePutAccessorById(node); + break; + } + + case PutGetterSetterById: { + compilePutGetterSetterById(node); + break; + } + + case PutGetterByVal: + case PutSetterByVal: { + compilePutAccessorByVal(node); + break; + } + + case DefineDataProperty: { + compileDefineDataProperty(node); + break; + } + + case DefineAccessorProperty: { + compileDefineAccessorProperty(node); + break; + } + + case GetGlobalLexicalVariable: case GetGlobalVar: { GPRTemporary result(this); - m_jit.load64(node->registerPointer(), result.gpr()); + m_jit.load64(node->variablePointer(), result.gpr()); jsValueResult(result.gpr(), node); break; } - case PutGlobalVar: { - JSValueOperand value(this, node->child1()); + case PutGlobalVariable: { + JSValueOperand value(this, node->child2()); - m_jit.store64(value.gpr(), node->registerPointer()); + m_jit.store64(value.gpr(), node->variablePointer()); noResult(node); break; } + case PutDynamicVar: { + compilePutDynamicVar(node); + break; + } + + case GetDynamicVar: { + compileGetDynamicVar(node); + break; + } + + case ResolveScope: { + compileResolveScope(node); + break; + } + case NotifyWrite: { - VariableWatchpointSet* set = node->variableWatchpointSet(); - - JSValueOperand value(this, node->child1()); - GPRReg valueGPR = value.gpr(); - - GPRTemporary temp(this); - GPRReg tempGPR = temp.gpr(); - - m_jit.load8(set->addressOfState(), tempGPR); - - JITCompiler::JumpList ready; - - ready.append(m_jit.branch32(JITCompiler::Equal, tempGPR, TrustedImm32(IsInvalidated))); - - if (set->state() == ClearWatchpoint) { - JITCompiler::Jump isWatched = - m_jit.branch32(JITCompiler::NotEqual, tempGPR, TrustedImm32(ClearWatchpoint)); - - m_jit.store64(valueGPR, set->addressOfInferredValue()); - m_jit.store8(TrustedImm32(IsWatched), set->addressOfState()); - ready.append(m_jit.jump()); - - isWatched.link(&m_jit); - } - - ready.append(m_jit.branch64( - JITCompiler::Equal, - JITCompiler::AbsoluteAddress(set->addressOfInferredValue()), valueGPR)); - - JITCompiler::Jump slowCase = m_jit.branchTest8( - JITCompiler::NonZero, JITCompiler::AbsoluteAddress(set->addressOfSetIsNotEmpty())); - m_jit.store8(TrustedImm32(IsInvalidated), set->addressOfState()); - m_jit.move(TrustedImm64(JSValue::encode(JSValue())), tempGPR); - m_jit.store64(tempGPR, set->addressOfInferredValue()); + compileNotifyWrite(node); + break; + } - ready.link(&m_jit); - - addSlowPathGenerator( - slowPathCall(slowCase, this, operationInvalidate, NoResult, set)); - - noResult(node); + case CheckTypeInfoFlags: { + compileCheckTypeInfoFlags(node); break; } - case VarInjectionWatchpoint: - case VariableWatchpoint: { - noResult(node); + case ParseInt: { + compileParseInt(node); break; } - case CheckHasInstance: { + case OverridesHasInstance: { + + Node* hasInstanceValueNode = node->child2().node(); + JSFunction* defaultHasInstanceFunction = jsCast<JSFunction*>(node->cellOperand()->value()); + + MacroAssembler::Jump notDefault; SpeculateCellOperand base(this, node->child1()); - GPRTemporary structure(this); + JSValueOperand hasInstanceValue(this, node->child2()); + GPRTemporary result(this); - // Speculate that base 'ImplementsDefaultHasInstance'. - m_jit.loadPtr(MacroAssembler::Address(base.gpr(), JSCell::structureOffset()), structure.gpr()); - speculationCheck(Uncountable, JSValueRegs(), 0, m_jit.branchTest8(MacroAssembler::Zero, MacroAssembler::Address(structure.gpr(), Structure::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance))); + GPRReg resultGPR = result.gpr(); + GPRReg baseGPR = base.gpr(); - noResult(node); + // It would be great if constant folding handled automatically the case where we knew the hasInstance function + // was a constant. Unfortunately, the folding rule for OverridesHasInstance is in the strength reduction phase + // since it relies on OSR information. https://bugs.webkit.org/show_bug.cgi?id=154832 + if (!hasInstanceValueNode->isCellConstant() || defaultHasInstanceFunction != hasInstanceValueNode->asCell()) { + GPRReg hasInstanceValueGPR = hasInstanceValue.gpr(); + notDefault = m_jit.branchPtr(MacroAssembler::NotEqual, hasInstanceValueGPR, TrustedImmPtr(node->cellOperand())); + } + + // Check that base 'ImplementsDefaultHasInstance'. + m_jit.test8(MacroAssembler::Zero, MacroAssembler::Address(baseGPR, JSCell::typeInfoFlagsOffset()), MacroAssembler::TrustedImm32(ImplementsDefaultHasInstance), resultGPR); + m_jit.or32(TrustedImm32(ValueFalse), resultGPR); + MacroAssembler::Jump done = m_jit.jump(); + + if (notDefault.isSet()) { + notDefault.link(&m_jit); + moveTrueTo(resultGPR); + } + + done.link(&m_jit); + jsValueResult(resultGPR, node, DataFormatJSBoolean); break; } @@ -4359,14 +4645,31 @@ void SpeculativeJIT::compile(Node* node) compileInstanceOf(node); break; } + + case InstanceOfCustom: { + compileInstanceOfCustom(node); + break; + } + + case IsEmpty: { + JSValueOperand value(this, node->child1()); + GPRTemporary result(this, Reuse, value); + + m_jit.comparePtr(JITCompiler::Equal, value.gpr(), TrustedImm32(JSValue::encode(JSValue())), result.gpr()); + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + + jsValueResult(result.gpr(), node, DataFormatJSBoolean); + break; + } case IsUndefined: { JSValueOperand value(this, node->child1()); GPRTemporary result(this); GPRTemporary localGlobalObject(this); GPRTemporary remoteGlobalObject(this); + GPRTemporary scratch(this); - JITCompiler::Jump isCell = m_jit.branchTest64(JITCompiler::Zero, value.gpr(), GPRInfo::tagMaskRegister); + JITCompiler::Jump isCell = m_jit.branchIfCell(value.jsValueRegs()); m_jit.compare64(JITCompiler::Equal, value.gpr(), TrustedImm32(ValueUndefined), result.gpr()); JITCompiler::Jump done = m_jit.jump(); @@ -4377,15 +4680,18 @@ void SpeculativeJIT::compile(Node* node) m_jit.move(TrustedImm32(0), result.gpr()); notMasqueradesAsUndefined = m_jit.jump(); } else { - m_jit.loadPtr(JITCompiler::Address(value.gpr(), JSCell::structureOffset()), result.gpr()); - JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8(JITCompiler::NonZero, JITCompiler::Address(result.gpr(), Structure::typeInfoFlagsOffset()), TrustedImm32(MasqueradesAsUndefined)); + JITCompiler::Jump isMasqueradesAsUndefined = m_jit.branchTest8( + JITCompiler::NonZero, + JITCompiler::Address(value.gpr(), JSCell::typeInfoFlagsOffset()), + TrustedImm32(MasqueradesAsUndefined)); m_jit.move(TrustedImm32(0), result.gpr()); notMasqueradesAsUndefined = m_jit.jump(); isMasqueradesAsUndefined.link(&m_jit); GPRReg localGlobalObjectGPR = localGlobalObject.gpr(); GPRReg remoteGlobalObjectGPR = remoteGlobalObject.gpr(); - m_jit.move(TrustedImmPtr(m_jit.globalObjectFor(node->codeOrigin)), localGlobalObjectGPR); + m_jit.move(TrustedImmPtr::weakPointer(m_jit.graph(), m_jit.globalObjectFor(node->origin.semantic)), localGlobalObjectGPR); + m_jit.emitLoadStructure(value.gpr(), result.gpr(), scratch.gpr()); m_jit.loadPtr(JITCompiler::Address(result.gpr(), Structure::globalObjectOffset()), remoteGlobalObjectGPR); m_jit.comparePtr(JITCompiler::Equal, localGlobalObjectGPR, remoteGlobalObjectGPR, result.gpr()); } @@ -4419,105 +4725,336 @@ void SpeculativeJIT::compile(Node* node) break; } - case IsString: { - JSValueOperand value(this, node->child1()); - GPRTemporary result(this, Reuse, value); - - JITCompiler::Jump isNotCell = m_jit.branchTest64(JITCompiler::NonZero, value.gpr(), GPRInfo::tagMaskRegister); - - m_jit.loadPtr(JITCompiler::Address(value.gpr(), JSCell::structureOffset()), result.gpr()); - m_jit.compare8(JITCompiler::Equal, JITCompiler::Address(result.gpr(), Structure::typeInfoTypeOffset()), TrustedImm32(StringType), result.gpr()); - m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); - JITCompiler::Jump done = m_jit.jump(); + case MapHash: { + switch (node->child1().useKind()) { + case BooleanUse: + case Int32Use: + case SymbolUse: + case ObjectUse: { + JSValueOperand input(this, node->child1(), ManualOperandSpeculation); + GPRTemporary result(this, Reuse, input); + GPRTemporary temp(this); + + GPRReg inputGPR = input.gpr(); + GPRReg resultGPR = result.gpr(); + GPRReg tempGPR = temp.gpr(); + + speculate(node, node->child1()); + + m_jit.move(inputGPR, resultGPR); + m_jit.wangsInt64Hash(resultGPR, tempGPR); + int32Result(resultGPR, node); + break; + } + case CellUse: + case StringUse: { + SpeculateCellOperand input(this, node->child1()); + GPRTemporary result(this); + std::optional<GPRTemporary> temp; + + GPRReg tempGPR = InvalidGPRReg; + if (node->child1().useKind() == CellUse) { + temp.emplace(this); + tempGPR = temp->gpr(); + } + + GPRReg inputGPR = input.gpr(); + GPRReg resultGPR = result.gpr(); + + MacroAssembler::JumpList slowPath; + MacroAssembler::JumpList done; + + if (node->child1().useKind() == StringUse) + speculateString(node->child1(), inputGPR); + else { + auto isString = m_jit.branch8(MacroAssembler::Equal, MacroAssembler::Address(inputGPR, JSCell::typeInfoTypeOffset()), TrustedImm32(StringType)); + m_jit.move(inputGPR, resultGPR); + m_jit.wangsInt64Hash(resultGPR, tempGPR); + done.append(m_jit.jump()); + isString.link(&m_jit); + } + + m_jit.loadPtr(MacroAssembler::Address(inputGPR, JSString::offsetOfValue()), resultGPR); + slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, resultGPR)); + m_jit.load32(MacroAssembler::Address(resultGPR, StringImpl::flagsOffset()), resultGPR); + m_jit.urshift32(MacroAssembler::TrustedImm32(StringImpl::s_flagCount), resultGPR); + slowPath.append(m_jit.branchTest32(MacroAssembler::Zero, resultGPR)); + done.append(m_jit.jump()); + + slowPath.link(&m_jit); + silentSpillAllRegisters(resultGPR); + callOperation(operationMapHash, resultGPR, JSValueRegs(inputGPR)); + silentFillAllRegisters(resultGPR); + m_jit.exceptionCheck(); + + done.link(&m_jit); + int32Result(resultGPR, node); + break; + } + default: + RELEASE_ASSERT(node->child1().useKind() == UntypedUse); + break; + } + if (node->child1().useKind() != UntypedUse) + break; + + JSValueOperand input(this, node->child1()); + GPRTemporary temp(this); + GPRTemporary result(this); + + GPRReg inputGPR = input.gpr(); + GPRReg resultGPR = result.gpr(); + GPRReg tempGPR = temp.gpr(); + + MacroAssembler::JumpList straightHash; + MacroAssembler::JumpList done; + auto isNotCell = m_jit.branchIfNotCell(inputGPR); + MacroAssembler::JumpList slowPath; + straightHash.append(m_jit.branch8(MacroAssembler::NotEqual, MacroAssembler::Address(inputGPR, JSCell::typeInfoTypeOffset()), TrustedImm32(StringType))); + m_jit.loadPtr(MacroAssembler::Address(inputGPR, JSString::offsetOfValue()), resultGPR); + slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, resultGPR)); + m_jit.load32(MacroAssembler::Address(resultGPR, StringImpl::flagsOffset()), resultGPR); + m_jit.urshift32(MacroAssembler::TrustedImm32(StringImpl::s_flagCount), resultGPR); + slowPath.append(m_jit.branchTest32(MacroAssembler::Zero, resultGPR)); + done.append(m_jit.jump()); + isNotCell.link(&m_jit); - m_jit.move(TrustedImm32(ValueFalse), result.gpr()); - + straightHash.append(m_jit.branchIfNotNumber(inputGPR)); + straightHash.append(m_jit.branchIfInt32(JSValueRegs(inputGPR))); + slowPath.append(m_jit.jump()); + + straightHash.link(&m_jit); + m_jit.move(inputGPR, resultGPR); + m_jit.wangsInt64Hash(resultGPR, tempGPR); + done.append(m_jit.jump()); + + slowPath.link(&m_jit); + silentSpillAllRegisters(resultGPR); + callOperation(operationMapHash, resultGPR, JSValueRegs(inputGPR)); + silentFillAllRegisters(resultGPR); + m_jit.exceptionCheck(); + done.link(&m_jit); - jsValueResult(result.gpr(), node, DataFormatJSBoolean); + int32Result(resultGPR, node); break; } - - case IsObject: { - JSValueOperand value(this, node->child1()); - GPRReg valueGPR = value.gpr(); - GPRResult result(this); + case GetMapBucket: { + SpeculateCellOperand map(this, node->child1()); + JSValueOperand key(this, node->child2(), ManualOperandSpeculation); + SpeculateInt32Operand hash(this, node->child3()); + GPRTemporary mask(this); + GPRTemporary index(this); + GPRTemporary buffer(this); + GPRTemporary bucket(this); + GPRTemporary result(this); + + GPRReg hashGPR = hash.gpr(); + GPRReg mapGPR = map.gpr(); + GPRReg maskGPR = mask.gpr(); + GPRReg indexGPR = index.gpr(); + GPRReg bufferGPR = buffer.gpr(); + GPRReg bucketGPR = bucket.gpr(); + GPRReg keyGPR = key.gpr(); GPRReg resultGPR = result.gpr(); - flushRegisters(); - callOperation(operationIsObject, resultGPR, valueGPR); - m_jit.or32(TrustedImm32(ValueFalse), resultGPR); - jsValueResult(result.gpr(), node, DataFormatJSBoolean); + + if (node->child1().useKind() == MapObjectUse) + speculateMapObject(node->child1(), mapGPR); + else if (node->child1().useKind() == SetObjectUse) + speculateSetObject(node->child1(), mapGPR); + else + RELEASE_ASSERT_NOT_REACHED(); + + if (node->child2().useKind() != UntypedUse) + speculate(node, node->child2()); + + m_jit.loadPtr(MacroAssembler::Address(mapGPR, node->child1().useKind() == MapObjectUse ? JSMap::offsetOfHashMapImpl() : JSSet::offsetOfHashMapImpl()), bufferGPR); + m_jit.load32(MacroAssembler::Address(bufferGPR, HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::offsetOfCapacity()), maskGPR); + m_jit.loadPtr(MacroAssembler::Address(bufferGPR, HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::offsetOfBuffer()), bufferGPR); + m_jit.sub32(TrustedImm32(1), maskGPR); + m_jit.move(hashGPR, indexGPR); + + MacroAssembler::Label loop = m_jit.label(); + MacroAssembler::JumpList done; + MacroAssembler::JumpList slowPathCases; + MacroAssembler::JumpList loopAround; + + m_jit.and32(maskGPR, indexGPR); + m_jit.loadPtr(MacroAssembler::BaseIndex(bufferGPR, indexGPR, MacroAssembler::TimesEight), bucketGPR); + m_jit.move(bucketGPR, resultGPR); + auto notPresentInTable = m_jit.branchPtr(MacroAssembler::Equal, + bucketGPR, TrustedImmPtr(bitwise_cast<size_t>(HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::emptyValue()))); + loopAround.append(m_jit.branchPtr(MacroAssembler::Equal, + bucketGPR, TrustedImmPtr(bitwise_cast<size_t>(HashMapImpl<HashMapBucket<HashMapBucketDataKey>>::deletedValue())))); + + m_jit.load64(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKey>::offsetOfKey()), bucketGPR); + + // Perform Object.is() + switch (node->child2().useKind()) { + case BooleanUse: + case Int32Use: + case SymbolUse: + case ObjectUse: { + done.append(m_jit.branch64(MacroAssembler::Equal, bucketGPR, keyGPR)); // They're definitely the same value, we found the bucket we were looking for! + // Otherwise, loop around. + break; + } + case CellUse: { + done.append(m_jit.branch64(MacroAssembler::Equal, bucketGPR, keyGPR)); + loopAround.append(m_jit.branchIfNotCell(JSValueRegs(bucketGPR))); + loopAround.append(m_jit.branch8(JITCompiler::NotEqual, + JITCompiler::Address(bucketGPR, JSCell::typeInfoTypeOffset()), TrustedImm32(StringType))); + loopAround.append(m_jit.branch8(JITCompiler::NotEqual, + JITCompiler::Address(keyGPR, JSCell::typeInfoTypeOffset()), TrustedImm32(StringType))); + // They're both strings. + slowPathCases.append(m_jit.jump()); + break; + } + case StringUse: { + done.append(m_jit.branch64(MacroAssembler::Equal, bucketGPR, keyGPR)); // They're definitely the same value, we found the bucket we were looking for! + loopAround.append(m_jit.branchIfNotCell(JSValueRegs(bucketGPR))); + loopAround.append(m_jit.branch8(JITCompiler::NotEqual, + JITCompiler::Address(bucketGPR, JSCell::typeInfoTypeOffset()), TrustedImm32(StringType))); + slowPathCases.append(m_jit.jump()); + break; + } + case UntypedUse: { + done.append(m_jit.branch64(MacroAssembler::Equal, bucketGPR, keyGPR)); // They're definitely the same value, we found the bucket we were looking for! + auto oneIsntCell = m_jit.branchIfNotCell(JSValueRegs(bucketGPR)); + // first is a cell here. + loopAround.append(m_jit.branchIfNotCell(JSValueRegs(keyGPR))); + // Both are cells here. + loopAround.append(m_jit.branch8(JITCompiler::NotEqual, + JITCompiler::Address(bucketGPR, JSCell::typeInfoTypeOffset()), TrustedImm32(StringType))); + // The first is a string here. + slowPathCases.append(m_jit.branch8(JITCompiler::Equal, + JITCompiler::Address(keyGPR, JSCell::typeInfoTypeOffset()), TrustedImm32(StringType))); + // The first is a string, but the second is not, we continue to loop around. + loopAround.append(m_jit.jump()); + + oneIsntCell.link(&m_jit); + // We've already done a 64-bit compare at this point, so if one is not a number, they're definitely not equal. + loopAround.append(m_jit.branchIfNotNumber(bucketGPR)); + loopAround.append(m_jit.branchIfNotNumber(keyGPR)); + // Both are definitely numbers. If we see a double, we go to the slow path. + slowPathCases.append(m_jit.branchIfNotInt32(bucketGPR)); + slowPathCases.append(m_jit.branchIfNotInt32(keyGPR)); + break; + } + default: + RELEASE_ASSERT_NOT_REACHED(); + } + + + if (!loopAround.empty()) + loopAround.link(&m_jit); + + m_jit.add32(TrustedImm32(1), indexGPR); + m_jit.jump().linkTo(loop, &m_jit); + + if (!slowPathCases.empty()) { + slowPathCases.link(&m_jit); + silentSpillAllRegisters(indexGPR); + if (node->child1().useKind() == MapObjectUse) + callOperation(operationJSMapFindBucket, resultGPR, mapGPR, keyGPR, hashGPR); + else + callOperation(operationJSSetFindBucket, resultGPR, mapGPR, keyGPR, hashGPR); + silentFillAllRegisters(indexGPR); + m_jit.exceptionCheck(); + done.append(m_jit.jump()); + } + + notPresentInTable.link(&m_jit); + m_jit.move(TrustedImmPtr(nullptr), resultGPR); + done.link(&m_jit); + cellResult(resultGPR, node); break; } - case IsFunction: { - JSValueOperand value(this, node->child1()); - GPRReg valueGPR = value.gpr(); - GPRResult result(this); + case LoadFromJSMapBucket: { + SpeculateCellOperand bucket(this, node->child1()); + GPRTemporary result(this); + + GPRReg bucketGPR = bucket.gpr(); GPRReg resultGPR = result.gpr(); - flushRegisters(); - callOperation(operationIsFunction, resultGPR, valueGPR); - m_jit.or32(TrustedImm32(ValueFalse), resultGPR); - jsValueResult(result.gpr(), node, DataFormatJSBoolean); + + auto notBucket = m_jit.branchTestPtr(MacroAssembler::Zero, bucketGPR); + m_jit.load64(MacroAssembler::Address(bucketGPR, HashMapBucket<HashMapBucketDataKeyValue>::offsetOfValue()), resultGPR); + auto done = m_jit.jump(); + + notBucket.link(&m_jit); + m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), resultGPR); + done.link(&m_jit); + jsValueResult(resultGPR, node); break; } - case TypeOf: { - JSValueOperand value(this, node->child1(), ManualOperandSpeculation); - GPRReg valueGPR = value.gpr(); - GPRTemporary temp(this); - GPRReg tempGPR = temp.gpr(); - GPRResult result(this); + case IsNonEmptyMapBucket: { + SpeculateCellOperand bucket(this, node->child1()); + GPRTemporary result(this); + + GPRReg bucketGPR = bucket.gpr(); GPRReg resultGPR = result.gpr(); - JITCompiler::JumpList doneJumps; - flushRegisters(); - - ASSERT(node->child1().useKind() == UntypedUse || node->child1().useKind() == CellUse || node->child1().useKind() == StringUse); + m_jit.comparePtr(MacroAssembler::NotEqual, bucketGPR, TrustedImm32(0), resultGPR); + m_jit.or32(TrustedImm32(ValueFalse), resultGPR); + jsValueResult(resultGPR, node, DataFormatJSBoolean); + break; + } - JITCompiler::Jump isNotCell = m_jit.branchTest64(JITCompiler::NonZero, valueGPR, GPRInfo::tagMaskRegister); - if (node->child1().useKind() != UntypedUse) - DFG_TYPE_CHECK(JSValueSource(valueGPR), node->child1(), SpecCell, isNotCell); + case ToLowerCase: { + compileToLowerCase(node); + break; + } - if (!node->child1()->shouldSpeculateObject() || node->child1().useKind() == StringUse) { - m_jit.loadPtr(JITCompiler::Address(valueGPR, JSCell::structureOffset()), tempGPR); - JITCompiler::Jump notString = m_jit.branch8(JITCompiler::NotEqual, JITCompiler::Address(tempGPR, Structure::typeInfoTypeOffset()), TrustedImm32(StringType)); - if (node->child1().useKind() == StringUse) - DFG_TYPE_CHECK(JSValueSource(valueGPR), node->child1(), SpecString, notString); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.stringString()), resultGPR); - doneJumps.append(m_jit.jump()); - if (node->child1().useKind() != StringUse) { - notString.link(&m_jit); - callOperation(operationTypeOf, resultGPR, valueGPR); - doneJumps.append(m_jit.jump()); - } - } else { - callOperation(operationTypeOf, resultGPR, valueGPR); - doneJumps.append(m_jit.jump()); - } + case NumberToStringWithRadix: { + compileNumberToStringWithRadix(node); + break; + } - if (node->child1().useKind() == UntypedUse) { - isNotCell.link(&m_jit); - JITCompiler::Jump notNumber = m_jit.branchTest64(JITCompiler::Zero, valueGPR, GPRInfo::tagTypeNumberRegister); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.numberString()), resultGPR); - doneJumps.append(m_jit.jump()); - notNumber.link(&m_jit); + case IsObject: { + JSValueOperand value(this, node->child1()); + GPRTemporary result(this, Reuse, value); + + JITCompiler::Jump isNotCell = m_jit.branchIfNotCell(value.jsValueRegs()); - JITCompiler::Jump notUndefined = m_jit.branch64(JITCompiler::NotEqual, valueGPR, JITCompiler::TrustedImm64(ValueUndefined)); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.undefinedString()), resultGPR); - doneJumps.append(m_jit.jump()); - notUndefined.link(&m_jit); + m_jit.compare8(JITCompiler::AboveOrEqual, + JITCompiler::Address(value.gpr(), JSCell::typeInfoTypeOffset()), + TrustedImm32(ObjectType), + result.gpr()); + m_jit.or32(TrustedImm32(ValueFalse), result.gpr()); + JITCompiler::Jump done = m_jit.jump(); - JITCompiler::Jump notNull = m_jit.branch64(JITCompiler::NotEqual, valueGPR, JITCompiler::TrustedImm64(ValueNull)); - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.objectString()), resultGPR); - doneJumps.append(m_jit.jump()); - notNull.link(&m_jit); + isNotCell.link(&m_jit); + m_jit.move(TrustedImm32(ValueFalse), result.gpr()); - // Only boolean left - m_jit.move(TrustedImmPtr(m_jit.vm()->smallStrings.booleanString()), resultGPR); - } - doneJumps.link(&m_jit); - cellResult(resultGPR, node); + done.link(&m_jit); + jsValueResult(result.gpr(), node, DataFormatJSBoolean); + break; + } + + case IsObjectOrNull: { + compileIsObjectOrNull(node); + break; + } + + case IsFunction: { + compileIsFunction(node); + break; + } + + case IsCellWithType: { + compileIsCellWithType(node); + break; + } + + case IsTypedArrayView: { + compileIsTypedArrayView(node); + break; + } + + case TypeOf: { + compileTypeOf(node); break; } @@ -4525,434 +5062,674 @@ void SpeculativeJIT::compile(Node* node) break; case Call: + case TailCall: + case TailCallInlinedCaller: case Construct: + case CallVarargs: + case TailCallVarargs: + case TailCallVarargsInlinedCaller: + case CallForwardVarargs: + case ConstructVarargs: + case ConstructForwardVarargs: + case TailCallForwardVarargs: + case TailCallForwardVarargsInlinedCaller: + case CallEval: + case DirectCall: + case DirectConstruct: + case DirectTailCall: + case DirectTailCallInlinedCaller: emitCall(node); break; - case CreateActivation: { - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); + case LoadVarargs: { + LoadVarargsData* data = node->loadVarargsData(); - JSValueOperand value(this, node->child1()); - GPRTemporary result(this, Reuse, value); + GPRReg argumentsGPR; + { + JSValueOperand arguments(this, node->child1()); + argumentsGPR = arguments.gpr(); + flushRegisters(); + } - GPRReg valueGPR = value.gpr(); - GPRReg resultGPR = result.gpr(); + callOperation(operationSizeOfVarargs, GPRInfo::returnValueGPR, argumentsGPR, data->offset); + m_jit.exceptionCheck(); - m_jit.move(valueGPR, resultGPR); + lock(GPRInfo::returnValueGPR); + { + JSValueOperand arguments(this, node->child1()); + argumentsGPR = arguments.gpr(); + flushRegisters(); + } + unlock(GPRInfo::returnValueGPR); - JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR); + // FIXME: There is a chance that we will call an effectful length property twice. This is safe + // from the standpoint of the VM's integrity, but it's subtly wrong from a spec compliance + // standpoint. The best solution would be one where we can exit *into* the op_call_varargs right + // past the sizing. + // https://bugs.webkit.org/show_bug.cgi?id=141448 + + GPRReg argCountIncludingThisGPR = + JITCompiler::selectScratchGPR(GPRInfo::returnValueGPR, argumentsGPR); - addSlowPathGenerator( - slowPathCall( - notCreated, this, operationCreateActivation, resultGPR, - framePointerOffsetToGetActivationRegisters())); + m_jit.add32(TrustedImm32(1), GPRInfo::returnValueGPR, argCountIncludingThisGPR); + speculationCheck( + VarargsOverflow, JSValueSource(), Edge(), m_jit.branch32( + MacroAssembler::Above, + argCountIncludingThisGPR, + TrustedImm32(data->limit))); - cellResult(resultGPR, node); + m_jit.store32(argCountIncludingThisGPR, JITCompiler::payloadFor(data->machineCount)); + + callOperation(operationLoadVarargs, data->machineStart.offset(), argumentsGPR, data->offset, GPRInfo::returnValueGPR, data->mandatoryMinimum); + m_jit.exceptionCheck(); + + noResult(node); break; } - case FunctionReentryWatchpoint: { - noResult(node); + case ForwardVarargs: { + compileForwardVarargs(node); break; } - case CreateArguments: { - JSValueOperand value(this, node->child1()); - GPRTemporary result(this, Reuse, value); + case CreateActivation: { + compileCreateActivation(node); + break; + } - GPRReg valueGPR = value.gpr(); - GPRReg resultGPR = result.gpr(); + case CreateDirectArguments: { + compileCreateDirectArguments(node); + break; + } - m_jit.move(valueGPR, resultGPR); + case GetFromArguments: { + compileGetFromArguments(node); + break; + } - JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR); + case PutToArguments: { + compilePutToArguments(node); + break; + } + + case GetArgument: { + compileGetArgument(node); + break; + } - if (node->codeOrigin.inlineCallFrame) { - addSlowPathGenerator( - slowPathCall( - notCreated, this, operationCreateInlinedArguments, resultGPR, - node->codeOrigin.inlineCallFrame)); - } else { - addSlowPathGenerator( - slowPathCall(notCreated, this, operationCreateArguments, resultGPR)); - } + case CreateScopedArguments: { + compileCreateScopedArguments(node); + break; + } - cellResult(resultGPR, node); + case CreateClonedArguments: { + compileCreateClonedArguments(node); break; } + case CreateRest: { + compileCreateRest(node); + break; + } + + case NewFunction: + case NewGeneratorFunction: + case NewAsyncFunction: + compileNewFunction(node); + break; - case TearOffActivation: { - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); + case SetFunctionName: + compileSetFunctionName(node); + break; - JSValueOperand activationValue(this, node->child1()); - GPRTemporary scratch(this); - GPRReg activationValueGPR = activationValue.gpr(); - GPRReg scratchGPR = scratch.gpr(); + case In: + compileIn(node); + break; - JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, activationValueGPR); - - SymbolTable* symbolTable = m_jit.symbolTableFor(node->codeOrigin); - int registersOffset = JSActivation::registersOffset(symbolTable); - - int bytecodeCaptureStart = symbolTable->captureStart(); - int machineCaptureStart = m_jit.graph().m_machineCaptureStart; - for (int i = symbolTable->captureCount(); i--;) { - m_jit.load64( - JITCompiler::Address( - GPRInfo::callFrameRegister, - (machineCaptureStart - i) * sizeof(Register)), - scratchGPR); - m_jit.store64( - scratchGPR, - JITCompiler::Address( - activationValueGPR, - registersOffset + (bytecodeCaptureStart - i) * sizeof(Register))); + case HasOwnProperty: { + SpeculateCellOperand object(this, node->child1()); + GPRTemporary uniquedStringImpl(this); + GPRTemporary temp(this); + GPRTemporary hash(this); + GPRTemporary structureID(this); + GPRTemporary result(this); + + std::optional<SpeculateCellOperand> keyAsCell; + std::optional<JSValueOperand> keyAsValue; + GPRReg keyGPR; + if (node->child2().useKind() == UntypedUse) { + keyAsValue.emplace(this, node->child2()); + keyGPR = keyAsValue->gpr(); + } else { + ASSERT(node->child2().useKind() == StringUse || node->child2().useKind() == SymbolUse); + keyAsCell.emplace(this, node->child2()); + keyGPR = keyAsCell->gpr(); } - m_jit.addPtr(TrustedImm32(registersOffset), activationValueGPR, scratchGPR); - m_jit.storePtr(scratchGPR, JITCompiler::Address(activationValueGPR, JSActivation::offsetOfRegisters())); - notCreated.link(&m_jit); - noResult(node); + GPRReg objectGPR = object.gpr(); + GPRReg implGPR = uniquedStringImpl.gpr(); + GPRReg tempGPR = temp.gpr(); + GPRReg hashGPR = hash.gpr(); + GPRReg structureIDGPR = structureID.gpr(); + GPRReg resultGPR = result.gpr(); + + speculateObject(node->child1()); + + MacroAssembler::JumpList slowPath; + switch (node->child2().useKind()) { + case SymbolUse: { + speculateSymbol(node->child2(), keyGPR); + m_jit.loadPtr(MacroAssembler::Address(keyGPR, Symbol::offsetOfSymbolImpl()), implGPR); + break; + } + case StringUse: { + speculateString(node->child2(), keyGPR); + m_jit.loadPtr(MacroAssembler::Address(keyGPR, JSString::offsetOfValue()), implGPR); + slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, implGPR)); + slowPath.append(m_jit.branchTest32( + MacroAssembler::Zero, MacroAssembler::Address(implGPR, StringImpl::flagsOffset()), + MacroAssembler::TrustedImm32(StringImpl::flagIsAtomic()))); + break; + } + case UntypedUse: { + slowPath.append(m_jit.branchIfNotCell(JSValueRegs(keyGPR))); + auto isNotString = m_jit.branchIfNotString(keyGPR); + m_jit.loadPtr(MacroAssembler::Address(keyGPR, JSString::offsetOfValue()), implGPR); + slowPath.append(m_jit.branchTestPtr(MacroAssembler::Zero, implGPR)); + slowPath.append(m_jit.branchTest32( + MacroAssembler::Zero, MacroAssembler::Address(implGPR, StringImpl::flagsOffset()), + MacroAssembler::TrustedImm32(StringImpl::flagIsAtomic()))); + auto hasUniquedImpl = m_jit.jump(); + + isNotString.link(&m_jit); + slowPath.append(m_jit.branchIfNotSymbol(keyGPR)); + m_jit.loadPtr(MacroAssembler::Address(keyGPR, Symbol::offsetOfSymbolImpl()), implGPR); + + hasUniquedImpl.link(&m_jit); + break; + } + default: + RELEASE_ASSERT_NOT_REACHED(); + } + + // Note that we don't test if the hash is zero here. AtomicStringImpl's can't have a zero + // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only + // ever load the result from the cache if the cache entry matches what we are querying for. + // So we either get super lucky and use zero for the hash and somehow collide with the entity + // we're looking for, or we realize we're comparing against another entity, and go to the + // slow path anyways. + m_jit.load32(MacroAssembler::Address(implGPR, UniquedStringImpl::flagsOffset()), hashGPR); + m_jit.urshift32(MacroAssembler::TrustedImm32(StringImpl::s_flagCount), hashGPR); + m_jit.load32(MacroAssembler::Address(objectGPR, JSCell::structureIDOffset()), structureIDGPR); + m_jit.add32(structureIDGPR, hashGPR); + m_jit.and32(TrustedImm32(HasOwnPropertyCache::mask), hashGPR); + static_assert(sizeof(HasOwnPropertyCache::Entry) == 16, "Strong assumption of that here."); + m_jit.lshift32(TrustedImm32(4), hashGPR); + ASSERT(m_jit.vm()->hasOwnPropertyCache()); + m_jit.move(TrustedImmPtr(m_jit.vm()->hasOwnPropertyCache()), tempGPR); + slowPath.append(m_jit.branchPtr(MacroAssembler::NotEqual, + MacroAssembler::BaseIndex(tempGPR, hashGPR, MacroAssembler::TimesOne, HasOwnPropertyCache::Entry::offsetOfImpl()), implGPR)); + m_jit.load8(MacroAssembler::BaseIndex(tempGPR, hashGPR, MacroAssembler::TimesOne, HasOwnPropertyCache::Entry::offsetOfResult()), resultGPR); + m_jit.load32(MacroAssembler::BaseIndex(tempGPR, hashGPR, MacroAssembler::TimesOne, HasOwnPropertyCache::Entry::offsetOfStructureID()), tempGPR); + slowPath.append(m_jit.branch32(MacroAssembler::NotEqual, tempGPR, structureIDGPR)); + auto done = m_jit.jump(); + + slowPath.link(&m_jit); + silentSpillAllRegisters(resultGPR); + callOperation(operationHasOwnProperty, resultGPR, objectGPR, keyGPR); + silentFillAllRegisters(resultGPR); + m_jit.exceptionCheck(); + + done.link(&m_jit); + m_jit.or32(TrustedImm32(ValueFalse), resultGPR); + jsValueResult(resultGPR, node, DataFormatJSBoolean); break; } + + case CountExecution: + m_jit.add64(TrustedImm32(1), MacroAssembler::AbsoluteAddress(node->executionCounter()->address())); + break; - case TearOffArguments: { - JSValueOperand unmodifiedArgumentsValue(this, node->child1()); - JSValueOperand activationValue(this, node->child2()); - GPRReg unmodifiedArgumentsValueGPR = unmodifiedArgumentsValue.gpr(); - GPRReg activationValueGPR = activationValue.gpr(); + case ForceOSRExit: { + terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); + break; + } + + case InvalidationPoint: + emitInvalidationPoint(node); + break; - JITCompiler::Jump created = m_jit.branchTest64(JITCompiler::NonZero, unmodifiedArgumentsValueGPR); + case CheckWatchdogTimer: { + ASSERT(m_jit.vm()->watchdog()); + GPRTemporary unused(this); + GPRReg unusedGPR = unused.gpr(); - if (node->codeOrigin.inlineCallFrame) { - addSlowPathGenerator( - slowPathCall( - created, this, operationTearOffInlinedArguments, NoResult, - unmodifiedArgumentsValueGPR, activationValueGPR, node->codeOrigin.inlineCallFrame)); - } else { - addSlowPathGenerator( - slowPathCall( - created, this, operationTearOffArguments, NoResult, unmodifiedArgumentsValueGPR, activationValueGPR)); - } + JITCompiler::Jump timerDidFire = m_jit.branchTest8(JITCompiler::NonZero, + JITCompiler::AbsoluteAddress(m_jit.vm()->watchdog()->timerDidFireAddress())); + + addSlowPathGenerator(slowPathCall(timerDidFire, this, operationHandleWatchdogTimer, unusedGPR)); + break; + } + + case Phantom: + case Check: + DFG_NODE_DO_TO_CHILDREN(m_jit.graph(), node, speculate); + noResult(node); + break; + case PhantomLocal: + case LoopHint: + // This is a no-op. noResult(node); break; + + case Unreachable: + unreachable(node); + break; + + case StoreBarrier: + case FencedStoreBarrier: { + compileStoreBarrier(node); + break; } - case GetMyArgumentsLength: { - GPRTemporary result(this); + case GetEnumerableLength: { + SpeculateCellOperand enumerator(this, node->child1()); + GPRFlushedCallResult result(this); GPRReg resultGPR = result.gpr(); - - if (!isEmptySpeculation( - m_state.variables().operand( - m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) { - speculationCheck( - ArgumentsEscaped, JSValueRegs(), 0, - m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin)))); - } - - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); - m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultGPR); - m_jit.sub32(TrustedImm32(1), resultGPR); + + m_jit.load32(MacroAssembler::Address(enumerator.gpr(), JSPropertyNameEnumerator::indexedLengthOffset()), resultGPR); int32Result(resultGPR, node); break; } - - case GetMyArgumentsLengthSafe: { + case HasGenericProperty: { + JSValueOperand base(this, node->child1()); + SpeculateCellOperand property(this, node->child2()); + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + callOperation(operationHasGenericProperty, resultGPR, base.gpr(), property.gpr()); + m_jit.exceptionCheck(); + jsValueResult(resultGPR, node, DataFormatJSBoolean); + break; + } + case HasStructureProperty: { + JSValueOperand base(this, node->child1()); + SpeculateCellOperand property(this, node->child2()); + SpeculateCellOperand enumerator(this, node->child3()); GPRTemporary result(this); + + GPRReg baseGPR = base.gpr(); + GPRReg propertyGPR = property.gpr(); GPRReg resultGPR = result.gpr(); - - JITCompiler::Jump created = m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin))); - - if (node->codeOrigin.inlineCallFrame) { - m_jit.move( - Imm64(JSValue::encode(jsNumber(node->codeOrigin.inlineCallFrame->arguments.size() - 1))), - resultGPR); - } else { - m_jit.load32(JITCompiler::payloadFor(JSStack::ArgumentCount), resultGPR); - m_jit.sub32(TrustedImm32(1), resultGPR); - m_jit.or64(GPRInfo::tagTypeNumberRegister, resultGPR); - } - - // FIXME: the slow path generator should perform a forward speculation that the - // result is an integer. For now we postpone the speculation by having this return - // a JSValue. - - addSlowPathGenerator( - slowPathCall( - created, this, operationGetArgumentsLength, resultGPR, - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin).offset())); - - jsValueResult(resultGPR, node); + + m_jit.load32(MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()), resultGPR); + MacroAssembler::Jump wrongStructure = m_jit.branch32(MacroAssembler::NotEqual, + resultGPR, + MacroAssembler::Address(enumerator.gpr(), JSPropertyNameEnumerator::cachedStructureIDOffset())); + + moveTrueTo(resultGPR); + MacroAssembler::Jump done = m_jit.jump(); + + done.link(&m_jit); + + addSlowPathGenerator(slowPathCall(wrongStructure, this, operationHasGenericProperty, resultGPR, baseGPR, propertyGPR)); + jsValueResult(resultGPR, node, DataFormatJSBoolean); break; } - - case GetMyArgumentByVal: { - SpeculateStrictInt32Operand index(this, node->child1()); + case HasIndexedProperty: { + SpeculateCellOperand base(this, node->child1()); + SpeculateStrictInt32Operand index(this, node->child2()); GPRTemporary result(this); + + GPRReg baseGPR = base.gpr(); GPRReg indexGPR = index.gpr(); GPRReg resultGPR = result.gpr(); - if (!isEmptySpeculation( - m_state.variables().operand( - m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)) { - speculationCheck( - ArgumentsEscaped, JSValueRegs(), 0, - m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin)))); - } + MacroAssembler::JumpList slowCases; + ArrayMode mode = node->arrayMode(); + switch (mode.type()) { + case Array::Int32: + case Array::Contiguous: { + ASSERT(!!node->child3()); + StorageOperand storage(this, node->child3()); + GPRTemporary scratch(this); + + GPRReg storageGPR = storage.gpr(); + GPRReg scratchGPR = scratch.gpr(); - m_jit.add32(TrustedImm32(1), indexGPR, resultGPR); - if (node->codeOrigin.inlineCallFrame) { - speculationCheck( - Uncountable, JSValueRegs(), 0, - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultGPR, - Imm32(node->codeOrigin.inlineCallFrame->arguments.size()))); - } else { - speculationCheck( - Uncountable, JSValueRegs(), 0, - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultGPR, - JITCompiler::payloadFor(JSStack::ArgumentCount))); + MacroAssembler::Jump outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength())); + if (mode.isInBounds()) + speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds); + else + slowCases.append(outOfBounds); + + m_jit.load64(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), scratchGPR); + slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, scratchGPR)); + moveTrueTo(resultGPR); + break; } + case Array::Double: { + ASSERT(!!node->child3()); + StorageOperand storage(this, node->child3()); + FPRTemporary scratch(this); + FPRReg scratchFPR = scratch.fpr(); + GPRReg storageGPR = storage.gpr(); - JITCompiler::JumpList slowArgument; - JITCompiler::JumpList slowArgumentOutOfBounds; - if (m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) { - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); - const SlowArgument* slowArguments = m_jit.graph().m_slowArguments.get(); - - slowArgumentOutOfBounds.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, indexGPR, - Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount()))); + MacroAssembler::Jump outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, Butterfly::offsetOfPublicLength())); + if (mode.isInBounds()) + speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds); + else + slowCases.append(outOfBounds); - COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes); - m_jit.move(ImmPtr(slowArguments), resultGPR); - m_jit.load32( - JITCompiler::BaseIndex( - resultGPR, indexGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(SlowArgument, index)), - resultGPR); - m_jit.signExtend32ToPtr(resultGPR, resultGPR); - m_jit.load64( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight), - resultGPR); - slowArgument.append(m_jit.jump()); + m_jit.loadDouble(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight), scratchFPR); + slowCases.append(m_jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, scratchFPR, scratchFPR)); + moveTrueTo(resultGPR); + break; } - slowArgumentOutOfBounds.link(&m_jit); + case Array::ArrayStorage: { + ASSERT(!!node->child3()); + StorageOperand storage(this, node->child3()); + GPRTemporary scratch(this); - m_jit.signExtend32ToPtr(resultGPR, resultGPR); - - m_jit.load64( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight, m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin)), - resultGPR); + GPRReg storageGPR = storage.gpr(); + GPRReg scratchGPR = scratch.gpr(); - slowArgument.link(&m_jit); - jsValueResult(resultGPR, node); + MacroAssembler::Jump outOfBounds = m_jit.branch32(MacroAssembler::AboveOrEqual, indexGPR, MacroAssembler::Address(storageGPR, ArrayStorage::vectorLengthOffset())); + if (mode.isInBounds()) + speculationCheck(OutOfBounds, JSValueRegs(), 0, outOfBounds); + else + slowCases.append(outOfBounds); + + m_jit.load64(MacroAssembler::BaseIndex(storageGPR, indexGPR, MacroAssembler::TimesEight, ArrayStorage::vectorOffset()), scratchGPR); + slowCases.append(m_jit.branchTest64(MacroAssembler::Zero, scratchGPR)); + moveTrueTo(resultGPR); + break; + } + default: { + slowCases.append(m_jit.jump()); + break; + } + } + + addSlowPathGenerator(slowPathCall(slowCases, this, operationHasIndexedProperty, resultGPR, baseGPR, indexGPR, static_cast<int32_t>(node->internalMethodType()))); + + jsValueResult(resultGPR, node, DataFormatJSBoolean); break; } - - case GetMyArgumentByValSafe: { - SpeculateStrictInt32Operand index(this, node->child1()); + case GetDirectPname: { + Edge& baseEdge = m_jit.graph().varArgChild(node, 0); + Edge& propertyEdge = m_jit.graph().varArgChild(node, 1); + Edge& indexEdge = m_jit.graph().varArgChild(node, 2); + Edge& enumeratorEdge = m_jit.graph().varArgChild(node, 3); + + SpeculateCellOperand base(this, baseEdge); + SpeculateCellOperand property(this, propertyEdge); + SpeculateStrictInt32Operand index(this, indexEdge); + SpeculateCellOperand enumerator(this, enumeratorEdge); GPRTemporary result(this); + GPRTemporary scratch1(this); + GPRTemporary scratch2(this); + + GPRReg baseGPR = base.gpr(); + GPRReg propertyGPR = property.gpr(); GPRReg indexGPR = index.gpr(); + GPRReg enumeratorGPR = enumerator.gpr(); GPRReg resultGPR = result.gpr(); - - JITCompiler::JumpList slowPath; + GPRReg scratch1GPR = scratch1.gpr(); + GPRReg scratch2GPR = scratch2.gpr(); + + MacroAssembler::JumpList slowPath; + + // Check the structure + m_jit.load32(MacroAssembler::Address(baseGPR, JSCell::structureIDOffset()), scratch1GPR); slowPath.append( - m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin)))); - - m_jit.add32(TrustedImm32(1), indexGPR, resultGPR); - if (node->codeOrigin.inlineCallFrame) { - slowPath.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultGPR, - Imm32(node->codeOrigin.inlineCallFrame->arguments.size()))); - } else { - slowPath.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, - resultGPR, - JITCompiler::payloadFor(JSStack::ArgumentCount))); - } + m_jit.branch32( + MacroAssembler::NotEqual, + scratch1GPR, + MacroAssembler::Address( + enumeratorGPR, JSPropertyNameEnumerator::cachedStructureIDOffset()))); - JITCompiler::JumpList slowArgument; - JITCompiler::JumpList slowArgumentOutOfBounds; - if (m_jit.symbolTableFor(node->codeOrigin)->slowArguments()) { - RELEASE_ASSERT(!node->codeOrigin.inlineCallFrame); - const SlowArgument* slowArguments = m_jit.graph().m_slowArguments.get(); + // Compute the offset + // If index is less than the enumerator's cached inline storage, then it's an inline access + MacroAssembler::Jump outOfLineAccess = m_jit.branch32(MacroAssembler::AboveOrEqual, + indexGPR, MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedInlineCapacityOffset())); - slowArgumentOutOfBounds.append( - m_jit.branch32( - JITCompiler::AboveOrEqual, indexGPR, - Imm32(m_jit.symbolTableFor(node->codeOrigin)->parameterCount()))); + m_jit.load64(MacroAssembler::BaseIndex(baseGPR, indexGPR, MacroAssembler::TimesEight, JSObject::offsetOfInlineStorage()), resultGPR); - COMPILE_ASSERT(sizeof(SlowArgument) == 8, SlowArgument_size_is_eight_bytes); - m_jit.move(ImmPtr(slowArguments), resultGPR); - m_jit.load32( - JITCompiler::BaseIndex( - resultGPR, indexGPR, JITCompiler::TimesEight, - OBJECT_OFFSETOF(SlowArgument, index)), - resultGPR); - m_jit.signExtend32ToPtr(resultGPR, resultGPR); - m_jit.load64( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight), - resultGPR); - slowArgument.append(m_jit.jump()); - } - slowArgumentOutOfBounds.link(&m_jit); - - m_jit.signExtend32ToPtr(resultGPR, resultGPR); - - m_jit.load64( - JITCompiler::BaseIndex( - GPRInfo::callFrameRegister, resultGPR, JITCompiler::TimesEight, m_jit.offsetOfArgumentsIncludingThis(node->codeOrigin)), - resultGPR); - - if (node->codeOrigin.inlineCallFrame) { - addSlowPathGenerator( - slowPathCall( - slowPath, this, operationGetInlinedArgumentByVal, resultGPR, - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin).offset(), - node->codeOrigin.inlineCallFrame, - indexGPR)); - } else { - addSlowPathGenerator( - slowPathCall( - slowPath, this, operationGetArgumentByVal, resultGPR, - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin).offset(), - indexGPR)); - } + MacroAssembler::Jump done = m_jit.jump(); - slowArgument.link(&m_jit); + // Otherwise it's out of line + outOfLineAccess.link(&m_jit); + m_jit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratch2GPR); + m_jit.move(indexGPR, scratch1GPR); + m_jit.sub32(MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedInlineCapacityOffset()), scratch1GPR); + m_jit.neg32(scratch1GPR); + m_jit.signExtend32ToPtr(scratch1GPR, scratch1GPR); + int32_t offsetOfFirstProperty = static_cast<int32_t>(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue); + m_jit.load64(MacroAssembler::BaseIndex(scratch2GPR, scratch1GPR, MacroAssembler::TimesEight, offsetOfFirstProperty), resultGPR); + + done.link(&m_jit); + + addSlowPathGenerator(slowPathCall(slowPath, this, operationGetByVal, resultGPR, baseGPR, propertyGPR)); + jsValueResult(resultGPR, node); break; } - - case CheckArgumentsNotCreated: { - ASSERT(!isEmptySpeculation( - m_state.variables().operand( - m_jit.graph().argumentsRegisterFor(node->codeOrigin)).m_type)); - speculationCheck( - ArgumentsEscaped, JSValueRegs(), 0, - m_jit.branchTest64( - JITCompiler::NonZero, - JITCompiler::addressFor( - m_jit.graph().machineArgumentsRegisterFor(node->codeOrigin)))); - noResult(node); + case GetPropertyEnumerator: { + SpeculateCellOperand base(this, node->child1()); + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + callOperation(operationGetPropertyEnumerator, resultGPR, base.gpr()); + m_jit.exceptionCheck(); + cellResult(resultGPR, node); break; } - - case NewFunctionNoCheck: - compileNewFunctionNoCheck(node); + case GetEnumeratorStructurePname: + case GetEnumeratorGenericPname: { + SpeculateCellOperand enumerator(this, node->child1()); + SpeculateStrictInt32Operand index(this, node->child2()); + GPRTemporary scratch1(this); + GPRTemporary result(this); + + GPRReg enumeratorGPR = enumerator.gpr(); + GPRReg indexGPR = index.gpr(); + GPRReg scratch1GPR = scratch1.gpr(); + GPRReg resultGPR = result.gpr(); + + MacroAssembler::Jump inBounds = m_jit.branch32(MacroAssembler::Below, indexGPR, + MacroAssembler::Address(enumeratorGPR, (op == GetEnumeratorStructurePname) + ? JSPropertyNameEnumerator::endStructurePropertyIndexOffset() + : JSPropertyNameEnumerator::endGenericPropertyIndexOffset())); + + m_jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsNull())), resultGPR); + + MacroAssembler::Jump done = m_jit.jump(); + inBounds.link(&m_jit); + + m_jit.loadPtr(MacroAssembler::Address(enumeratorGPR, JSPropertyNameEnumerator::cachedPropertyNamesVectorOffset()), scratch1GPR); + m_jit.load64(MacroAssembler::BaseIndex(scratch1GPR, indexGPR, MacroAssembler::TimesEight), resultGPR); + + done.link(&m_jit); + jsValueResult(resultGPR, node); break; - - case NewFunction: { + } + case ToIndexString: { + SpeculateInt32Operand index(this, node->child1()); + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + callOperation(operationToIndexString, resultGPR, index.gpr()); + m_jit.exceptionCheck(); + cellResult(resultGPR, node); + break; + } + case ProfileType: { JSValueOperand value(this, node->child1()); - GPRTemporary result(this, Reuse, value); - + GPRTemporary scratch1(this); + GPRTemporary scratch2(this); + GPRTemporary scratch3(this); + + GPRReg scratch1GPR = scratch1.gpr(); + GPRReg scratch2GPR = scratch2.gpr(); + GPRReg scratch3GPR = scratch3.gpr(); GPRReg valueGPR = value.gpr(); - GPRReg resultGPR = result.gpr(); - - m_jit.move(valueGPR, resultGPR); - - JITCompiler::Jump notCreated = m_jit.branchTest64(JITCompiler::Zero, resultGPR); - + + MacroAssembler::JumpList jumpToEnd; + + jumpToEnd.append(m_jit.branchTest64(JITCompiler::Zero, valueGPR)); + + TypeLocation* cachedTypeLocation = node->typeLocation(); + // Compile in a predictive type check, if possible, to see if we can skip writing to the log. + // These typechecks are inlined to match those of the 64-bit JSValue type checks. + if (cachedTypeLocation->m_lastSeenType == TypeUndefined) + jumpToEnd.append(m_jit.branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())))); + else if (cachedTypeLocation->m_lastSeenType == TypeNull) + jumpToEnd.append(m_jit.branch64(MacroAssembler::Equal, valueGPR, MacroAssembler::TrustedImm64(JSValue::encode(jsNull())))); + else if (cachedTypeLocation->m_lastSeenType == TypeBoolean) { + m_jit.move(valueGPR, scratch2GPR); + m_jit.and64(TrustedImm32(~1), scratch2GPR); + jumpToEnd.append(m_jit.branch64(MacroAssembler::Equal, scratch2GPR, MacroAssembler::TrustedImm64(ValueFalse))); + } else if (cachedTypeLocation->m_lastSeenType == TypeAnyInt) + jumpToEnd.append(m_jit.branch64(MacroAssembler::AboveOrEqual, valueGPR, GPRInfo::tagTypeNumberRegister)); + else if (cachedTypeLocation->m_lastSeenType == TypeNumber) + jumpToEnd.append(m_jit.branchTest64(MacroAssembler::NonZero, valueGPR, GPRInfo::tagTypeNumberRegister)); + else if (cachedTypeLocation->m_lastSeenType == TypeString) { + MacroAssembler::Jump isNotCell = m_jit.branchIfNotCell(JSValueRegs(valueGPR)); + jumpToEnd.append(m_jit.branchIfString(valueGPR)); + isNotCell.link(&m_jit); + } + + // Load the TypeProfilerLog into Scratch2. + TypeProfilerLog* cachedTypeProfilerLog = m_jit.vm()->typeProfilerLog(); + m_jit.move(TrustedImmPtr(cachedTypeProfilerLog), scratch2GPR); + + // Load the next LogEntry into Scratch1. + m_jit.loadPtr(MacroAssembler::Address(scratch2GPR, TypeProfilerLog::currentLogEntryOffset()), scratch1GPR); + + // Store the JSValue onto the log entry. + m_jit.store64(valueGPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::valueOffset())); + + // Store the structureID of the cell if valueGPR is a cell, otherwise, store 0 on the log entry. + MacroAssembler::Jump isNotCell = m_jit.branchIfNotCell(JSValueRegs(valueGPR)); + m_jit.load32(MacroAssembler::Address(valueGPR, JSCell::structureIDOffset()), scratch3GPR); + m_jit.store32(scratch3GPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::structureIDOffset())); + MacroAssembler::Jump skipIsCell = m_jit.jump(); + isNotCell.link(&m_jit); + m_jit.store32(TrustedImm32(0), MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::structureIDOffset())); + skipIsCell.link(&m_jit); + + // Store the typeLocation on the log entry. + m_jit.move(TrustedImmPtr(cachedTypeLocation), scratch3GPR); + m_jit.storePtr(scratch3GPR, MacroAssembler::Address(scratch1GPR, TypeProfilerLog::LogEntry::locationOffset())); + + // Increment the current log entry. + m_jit.addPtr(TrustedImm32(sizeof(TypeProfilerLog::LogEntry)), scratch1GPR); + m_jit.storePtr(scratch1GPR, MacroAssembler::Address(scratch2GPR, TypeProfilerLog::currentLogEntryOffset())); + MacroAssembler::Jump clearLog = m_jit.branchPtr(MacroAssembler::Equal, scratch1GPR, TrustedImmPtr(cachedTypeProfilerLog->logEndPtr())); addSlowPathGenerator( - slowPathCall( - notCreated, this, operationNewFunction, - resultGPR, m_jit.codeBlock()->functionDecl(node->functionDeclIndex()))); - - jsValueResult(resultGPR, node); + slowPathCall(clearLog, this, operationProcessTypeProfilerLogDFG, NoResult)); + + jumpToEnd.link(&m_jit); + + noResult(node); break; } - - case NewFunctionExpression: - compileNewFunctionExpression(node); - break; - - case In: - compileIn(node); + case ProfileControlFlow: { + BasicBlockLocation* basicBlockLocation = node->basicBlockLocation(); + basicBlockLocation->emitExecuteCode(m_jit); + noResult(node); break; + } - case CountExecution: - m_jit.add64(TrustedImm32(1), MacroAssembler::AbsoluteAddress(node->executionCounter()->address())); - break; + case LogShadowChickenPrologue: { + flushRegisters(); + prepareForExternalCall(); + m_jit.emitStoreCodeOrigin(node->origin.semantic); - case ForceOSRExit: { - terminateSpeculativeExecution(InadequateCoverage, JSValueRegs(), 0); + GPRTemporary scratch1(this, GPRInfo::nonArgGPR0); // This must be a non-argument GPR. + GPRReg scratch1Reg = scratch1.gpr(); + GPRTemporary scratch2(this); + GPRReg scratch2Reg = scratch2.gpr(); + GPRTemporary shadowPacket(this); + GPRReg shadowPacketReg = shadowPacket.gpr(); + + m_jit.ensureShadowChickenPacket(shadowPacketReg, scratch1Reg, scratch2Reg); + + SpeculateCellOperand scope(this, node->child1()); + GPRReg scopeReg = scope.gpr(); + + m_jit.logShadowChickenProloguePacket(shadowPacketReg, scratch1Reg, scopeReg); + noResult(node); break; } - - case InvalidationPoint: - emitInvalidationPoint(node); - break; - case CheckWatchdogTimer: - speculationCheck( - WatchdogTimerFired, JSValueRegs(), 0, - m_jit.branchTest8( - JITCompiler::NonZero, - JITCompiler::AbsoluteAddress(m_jit.vm()->watchdog.timerDidFireAddress()))); - break; + case LogShadowChickenTail: { + flushRegisters(); + prepareForExternalCall(); + CallSiteIndex callSiteIndex = m_jit.emitStoreCodeOrigin(node->origin.semantic); - case Phantom: - DFG_NODE_DO_TO_CHILDREN(m_jit.graph(), node, speculate); + GPRTemporary scratch1(this, GPRInfo::nonArgGPR0); // This must be a non-argument GPR. + GPRReg scratch1Reg = scratch1.gpr(); + GPRTemporary scratch2(this); + GPRReg scratch2Reg = scratch2.gpr(); + GPRTemporary shadowPacket(this); + GPRReg shadowPacketReg = shadowPacket.gpr(); + + m_jit.ensureShadowChickenPacket(shadowPacketReg, scratch1Reg, scratch2Reg); + + JSValueOperand thisValue(this, node->child1()); + JSValueRegs thisRegs = JSValueRegs(thisValue.gpr()); + SpeculateCellOperand scope(this, node->child2()); + GPRReg scopeReg = scope.gpr(); + + m_jit.logShadowChickenTailPacket(shadowPacketReg, thisRegs, scopeReg, m_jit.codeBlock(), callSiteIndex); noResult(node); break; - - case Breakpoint: - case ProfileWillCall: - case ProfileDidCall: - case PhantomLocal: - case LoopHint: - // This is a no-op. - noResult(node); + } + + case MaterializeNewObject: + compileMaterializeNewObject(node); break; - case Unreachable: - RELEASE_ASSERT_NOT_REACHED(); + case CallDOM: + compileCallDOM(node); break; - case StoreBarrier: - case ConditionalStoreBarrier: - case StoreBarrierWithNullCheck: { - compileStoreBarrier(node); + case CallDOMGetter: + compileCallDOMGetter(node); + break; + + case CheckDOM: + compileCheckDOM(node); break; - } #if ENABLE(FTL_JIT) case CheckTierUpInLoop: { - MacroAssembler::Jump done = m_jit.branchAdd32( - MacroAssembler::Signed, + MacroAssembler::Jump callTierUp = m_jit.branchAdd32( + MacroAssembler::PositiveOrZero, TrustedImm32(Options::ftlTierUpCounterIncrementForLoop()), MacroAssembler::AbsoluteAddress(&m_jit.jitCode()->tierUpCounter.m_counter)); - - silentSpillAllRegisters(InvalidGPRReg); - m_jit.setupArgumentsExecState(); - appendCall(triggerTierUpNow); - silentFillAllRegisters(InvalidGPRReg); - - done.link(&m_jit); + + MacroAssembler::Label toNextOperation = m_jit.label(); + + Vector<SilentRegisterSavePlan> savePlans; + silentSpillAllRegistersImpl(false, savePlans, InvalidGPRReg); + unsigned bytecodeIndex = node->origin.semantic.bytecodeIndex; + + addSlowPathGenerator([=]() { + callTierUp.link(&m_jit); + + silentSpill(savePlans); + m_jit.setupArgumentsWithExecState(TrustedImm32(bytecodeIndex)); + appendCall(triggerTierUpNowInLoop); + silentFill(savePlans); + + m_jit.jump().linkTo(toNextOperation, &m_jit); + }); break; } @@ -4972,45 +5749,88 @@ void SpeculativeJIT::compile(Node* node) } case CheckTierUpAndOSREnter: { - ASSERT(!node->codeOrigin.inlineCallFrame); - + ASSERT(!node->origin.semantic.inlineCallFrame); + GPRTemporary temp(this); GPRReg tempGPR = temp.gpr(); - - MacroAssembler::Jump done = m_jit.branchAdd32( - MacroAssembler::Signed, + + unsigned bytecodeIndex = node->origin.semantic.bytecodeIndex; + auto triggerIterator = m_jit.jitCode()->tierUpEntryTriggers.find(bytecodeIndex); + DFG_ASSERT(m_jit.graph(), node, triggerIterator != m_jit.jitCode()->tierUpEntryTriggers.end()); + JITCode::TriggerReason* forceEntryTrigger = &(m_jit.jitCode()->tierUpEntryTriggers.find(bytecodeIndex)->value); + static_assert(!static_cast<uint8_t>(JITCode::TriggerReason::DontTrigger), "the JIT code assumes non-zero means 'enter'"); + static_assert(sizeof(JITCode::TriggerReason) == 1, "branchTest8 assumes this size"); + + MacroAssembler::Jump forceOSREntry = m_jit.branchTest8(MacroAssembler::NonZero, MacroAssembler::AbsoluteAddress(forceEntryTrigger)); + MacroAssembler::Jump overflowedCounter = m_jit.branchAdd32( + MacroAssembler::PositiveOrZero, TrustedImm32(Options::ftlTierUpCounterIncrementForLoop()), MacroAssembler::AbsoluteAddress(&m_jit.jitCode()->tierUpCounter.m_counter)); - - silentSpillAllRegisters(tempGPR); - m_jit.setupArgumentsWithExecState( - TrustedImm32(node->codeOrigin.bytecodeIndex), - TrustedImm32(m_stream->size())); - appendCallSetResult(triggerOSREntryNow, tempGPR); - MacroAssembler::Jump dontEnter = m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR); - m_jit.jump(tempGPR); - dontEnter.link(&m_jit); - silentFillAllRegisters(tempGPR); - - done.link(&m_jit); + MacroAssembler::Label toNextOperation = m_jit.label(); + + Vector<SilentRegisterSavePlan> savePlans; + silentSpillAllRegistersImpl(false, savePlans, tempGPR); + + unsigned streamIndex = m_stream->size(); + m_jit.jitCode()->bytecodeIndexToStreamIndex.add(bytecodeIndex, streamIndex); + + addSlowPathGenerator([=]() { + forceOSREntry.link(&m_jit); + overflowedCounter.link(&m_jit); + + silentSpill(savePlans); + m_jit.setupArgumentsWithExecState(TrustedImm32(bytecodeIndex)); + appendCallSetResult(triggerOSREntryNow, tempGPR); + + if (savePlans.isEmpty()) + m_jit.branchTestPtr(MacroAssembler::Zero, tempGPR).linkTo(toNextOperation, &m_jit); + else { + MacroAssembler::Jump osrEnter = m_jit.branchTestPtr(MacroAssembler::NonZero, tempGPR); + silentFill(savePlans); + m_jit.jump().linkTo(toNextOperation, &m_jit); + osrEnter.link(&m_jit); + } + m_jit.emitRestoreCalleeSaves(); + m_jit.jump(tempGPR); + }); break; } #else // ENABLE(FTL_JIT) case CheckTierUpInLoop: case CheckTierUpAtReturn: case CheckTierUpAndOSREnter: - RELEASE_ASSERT_NOT_REACHED(); + DFG_CRASH(m_jit.graph(), node, "Unexpected tier-up node"); break; #endif // ENABLE(FTL_JIT) - + case LastNodeType: case Phi: case Upsilon: - case GetArgument: case ExtractOSREntryLocal: case CheckInBounds: case ArithIMul: - RELEASE_ASSERT_NOT_REACHED(); + case MultiGetByOffset: + case MultiPutByOffset: + case FiatInt52: + case CheckBadCell: + case BottomValue: + case PhantomNewObject: + case PhantomNewFunction: + case PhantomNewGeneratorFunction: + case PhantomNewAsyncFunction: + case PhantomCreateActivation: + case GetMyArgumentByVal: + case GetMyArgumentByValOutOfBounds: + case PutHint: + case CheckStructureImmediate: + case MaterializeCreateActivation: + case PutStack: + case KillStack: + case GetStack: + case PhantomCreateRest: + case PhantomSpread: + case PhantomNewArrayWithSpread: + DFG_CRASH(m_jit.graph(), node, "Unexpected node"); break; } @@ -5021,35 +5841,135 @@ void SpeculativeJIT::compile(Node* node) use(node); } -#if ENABLE(GGC) -void SpeculativeJIT::writeBarrier(GPRReg ownerGPR, GPRReg valueGPR, Edge valueUse, GPRReg scratch1, GPRReg scratch2) +void SpeculativeJIT::moveTrueTo(GPRReg gpr) { - JITCompiler::Jump isNotCell; - if (!isKnownCell(valueUse.node())) - isNotCell = m_jit.branchTest64(JITCompiler::NonZero, valueGPR, GPRInfo::tagMaskRegister); + m_jit.move(TrustedImm32(ValueTrue), gpr); +} - JITCompiler::Jump definitelyNotMarked = genericWriteBarrier(m_jit, ownerGPR, scratch1, scratch2); - storeToWriteBarrierBuffer(ownerGPR, scratch1, scratch2); - definitelyNotMarked.link(&m_jit); +void SpeculativeJIT::moveFalseTo(GPRReg gpr) +{ + m_jit.move(TrustedImm32(ValueFalse), gpr); +} - if (!isKnownCell(valueUse.node())) - isNotCell.link(&m_jit); +void SpeculativeJIT::blessBoolean(GPRReg gpr) +{ + m_jit.or32(TrustedImm32(ValueFalse), gpr); } -void SpeculativeJIT::writeBarrier(JSCell* owner, GPRReg valueGPR, Edge valueUse, GPRReg scratch1, GPRReg scratch2) +void SpeculativeJIT::convertAnyInt(Edge valueEdge, GPRReg resultGPR) { - JITCompiler::Jump isNotCell; - if (!isKnownCell(valueUse.node())) - isNotCell = m_jit.branchTest64(JITCompiler::NonZero, valueGPR, GPRInfo::tagMaskRegister); + JSValueOperand value(this, valueEdge, ManualOperandSpeculation); + GPRReg valueGPR = value.gpr(); + + JITCompiler::Jump notInt32 = + m_jit.branch64(JITCompiler::Below, valueGPR, GPRInfo::tagTypeNumberRegister); + + m_jit.signExtend32ToPtr(valueGPR, resultGPR); + JITCompiler::Jump done = m_jit.jump(); + + notInt32.link(&m_jit); + silentSpillAllRegisters(resultGPR); + callOperation(operationConvertBoxedDoubleToInt52, resultGPR, valueGPR); + silentFillAllRegisters(resultGPR); + + DFG_TYPE_CHECK( + JSValueRegs(valueGPR), valueEdge, SpecInt32Only | SpecAnyIntAsDouble, + m_jit.branch64( + JITCompiler::Equal, resultGPR, + JITCompiler::TrustedImm64(JSValue::notInt52))); + done.link(&m_jit); +} - JITCompiler::Jump definitelyNotMarked = genericWriteBarrier(m_jit, owner); - storeToWriteBarrierBuffer(owner, scratch1, scratch2); - definitelyNotMarked.link(&m_jit); +void SpeculativeJIT::speculateAnyInt(Edge edge) +{ + if (!needsTypeCheck(edge, SpecInt32Only | SpecAnyIntAsDouble)) + return; + + GPRTemporary temp(this); + convertAnyInt(edge, temp.gpr()); +} - if (!isKnownCell(valueUse.node())) - isNotCell.link(&m_jit); +void SpeculativeJIT::speculateDoubleRepAnyInt(Edge edge) +{ + if (!needsTypeCheck(edge, SpecAnyIntAsDouble)) + return; + + SpeculateDoubleOperand value(this, edge); + FPRReg valueFPR = value.fpr(); + + GPRFlushedCallResult result(this); + GPRReg resultGPR = result.gpr(); + + flushRegisters(); + + callOperation(operationConvertDoubleToInt52, resultGPR, valueFPR); + + DFG_TYPE_CHECK( + JSValueRegs(), edge, SpecAnyIntAsDouble, + m_jit.branch64( + JITCompiler::Equal, resultGPR, + JITCompiler::TrustedImm64(JSValue::notInt52))); +} + +void SpeculativeJIT::compileArithRandom(Node* node) +{ + JSGlobalObject* globalObject = m_jit.graph().globalObjectFor(node->origin.semantic); + GPRTemporary temp1(this); + GPRTemporary temp2(this); + GPRTemporary temp3(this); + FPRTemporary result(this); + m_jit.emitRandomThunk(globalObject, temp1.gpr(), temp2.gpr(), temp3.gpr(), result.fpr()); + doubleResult(result.fpr(), node); +} + +void SpeculativeJIT::emitInitializeButterfly(GPRReg storageGPR, GPRReg sizeGPR, JSValueRegs emptyValueRegs, GPRReg scratchGPR) +{ + m_jit.zeroExtend32ToPtr(sizeGPR, scratchGPR); + MacroAssembler::Jump done = m_jit.branchTest32(MacroAssembler::Zero, scratchGPR); + MacroAssembler::Label loop = m_jit.label(); + m_jit.sub32(TrustedImm32(1), scratchGPR); + m_jit.store64(emptyValueRegs.gpr(), MacroAssembler::BaseIndex(storageGPR, scratchGPR, MacroAssembler::TimesEight)); + m_jit.branchTest32(MacroAssembler::NonZero, scratchGPR).linkTo(loop, &m_jit); + done.link(&m_jit); +} + +void SpeculativeJIT::compileAllocateNewArrayWithSize(JSGlobalObject* globalObject, GPRReg resultGPR, GPRReg sizeGPR, IndexingType indexingType, bool shouldConvertLargeSizeToArrayStorage) +{ + GPRTemporary storage(this); + GPRTemporary scratch(this); + GPRTemporary scratch2(this); + + GPRReg storageGPR = storage.gpr(); + GPRReg scratchGPR = scratch.gpr(); + GPRReg scratch2GPR = scratch2.gpr(); + + m_jit.move(TrustedImmPtr(0), storageGPR); + + MacroAssembler::JumpList slowCases; + if (shouldConvertLargeSizeToArrayStorage) + slowCases.append(m_jit.branch32(MacroAssembler::AboveOrEqual, sizeGPR, TrustedImm32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH))); + + // We can use resultGPR as a scratch right now. + emitAllocateButterfly(storageGPR, sizeGPR, resultGPR, scratchGPR, scratch2GPR, slowCases); + + if (hasDouble(indexingType)) + m_jit.move(TrustedImm64(bitwise_cast<int64_t>(PNaN)), scratchGPR); + else + m_jit.move(TrustedImm64(JSValue::encode(JSValue())), scratchGPR); + emitInitializeButterfly(storageGPR, sizeGPR, JSValueRegs(scratchGPR), scratch2GPR); + + RegisteredStructure structure = m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType)); + + emitAllocateJSObject<JSArray>(resultGPR, TrustedImmPtr(structure), storageGPR, scratchGPR, scratch2GPR, slowCases); + + m_jit.mutatorFence(); + + addSlowPathGenerator(std::make_unique<CallArrayAllocatorWithVariableSizeSlowPathGenerator>( + slowCases, this, operationNewArrayWithSize, resultGPR, + structure, + shouldConvertLargeSizeToArrayStorage ? m_jit.graph().registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)) : structure, + sizeGPR, storageGPR)); } -#endif // ENABLE(GGC) #endif |