/* * Copyright (C) 2013-2017 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY APPLE INC. ``AS IS'' AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include "config.h" #include "FTLLowerDFGToB3.h" #if ENABLE(FTL_JIT) #include "AirGenerationContext.h" #include "AllowMacroScratchRegisterUsage.h" #include "B3CheckValue.h" #include "B3FenceValue.h" #include "B3PatchpointValue.h" #include "B3SlotBaseValue.h" #include "B3StackmapGenerationParams.h" #include "B3ValueInlines.h" #include "CallFrameShuffler.h" #include "CodeBlockWithJITType.h" #include "DFGAbstractInterpreterInlines.h" #include "DFGCapabilities.h" #include "DFGDominators.h" #include "DFGInPlaceAbstractState.h" #include "DFGOSRAvailabilityAnalysisPhase.h" #include "DFGOSRExitFuzz.h" #include "DOMJITPatchpoint.h" #include "DirectArguments.h" #include "FTLAbstractHeapRepository.h" #include "FTLAvailableRecovery.h" #include "FTLDOMJITPatchpointParams.h" #include "FTLExceptionTarget.h" #include "FTLForOSREntryJITCode.h" #include "FTLFormattedValue.h" #include "FTLLazySlowPathCall.h" #include "FTLLoweredNodeValue.h" #include "FTLOperations.h" #include "FTLOutput.h" #include "FTLPatchpointExceptionHandle.h" #include "FTLThunks.h" #include "FTLWeightedTarget.h" #include "JITAddGenerator.h" #include "JITBitAndGenerator.h" #include "JITBitOrGenerator.h" #include "JITBitXorGenerator.h" #include "JITDivGenerator.h" #include "JITInlineCacheGenerator.h" #include "JITLeftShiftGenerator.h" #include "JITMathIC.h" #include "JITMulGenerator.h" #include "JITRightShiftGenerator.h" #include "JITSubGenerator.h" #include "JSCInlines.h" #include "JSGeneratorFunction.h" #include "JSLexicalEnvironment.h" #include "JSMap.h" #include "OperandsInlines.h" #include "ScopedArguments.h" #include "ScopedArgumentsTable.h" #include "ScratchRegisterAllocator.h" #include "SetupVarargsFrame.h" #include "ShadowChicken.h" #include "StructureStubInfo.h" #include "VirtualRegister.h" #include "Watchdog.h" #include #if !OS(WINDOWS) #include #endif #include #include #include namespace JSC { namespace FTL { using namespace B3; using namespace DFG; namespace { std::atomic compileCounter; #if !ASSERT_DISABLED NO_RETURN_DUE_TO_CRASH static void ftlUnreachable( CodeBlock* codeBlock, BlockIndex blockIndex, unsigned nodeIndex) { dataLog("Crashing in thought-to-be-unreachable FTL-generated code for ", pointerDump(codeBlock), " at basic block #", blockIndex); if (nodeIndex != UINT_MAX) dataLog(", node @", nodeIndex); dataLog(".\n"); CRASH(); } #endif // Using this instead of typeCheck() helps to reduce the load on B3, by creating // significantly less dead code. #define FTL_TYPE_CHECK_WITH_EXIT_KIND(exitKind, lowValue, highValue, typesPassedThrough, failCondition) do { \ FormattedValue _ftc_lowValue = (lowValue); \ Edge _ftc_highValue = (highValue); \ SpeculatedType _ftc_typesPassedThrough = (typesPassedThrough); \ if (!m_interpreter.needsTypeCheck(_ftc_highValue, _ftc_typesPassedThrough)) \ break; \ typeCheck(_ftc_lowValue, _ftc_highValue, _ftc_typesPassedThrough, (failCondition), exitKind); \ } while (false) #define FTL_TYPE_CHECK(lowValue, highValue, typesPassedThrough, failCondition) \ FTL_TYPE_CHECK_WITH_EXIT_KIND(BadType, lowValue, highValue, typesPassedThrough, failCondition) class LowerDFGToB3 { WTF_MAKE_NONCOPYABLE(LowerDFGToB3); public: LowerDFGToB3(State& state) : m_graph(state.graph) , m_ftlState(state) , m_out(state) , m_proc(*state.proc) , m_availabilityCalculator(m_graph) , m_state(state.graph) , m_interpreter(state.graph, m_state) { } void lower() { State* state = &m_ftlState; CString name; if (verboseCompilationEnabled()) { name = toCString( "jsBody_", ++compileCounter, "_", codeBlock()->inferredName(), "_", codeBlock()->hash()); } else name = "jsBody"; m_graph.ensureDominators(); if (verboseCompilationEnabled()) dataLog("Function ready, beginning lowering.\n"); m_out.initialize(m_heaps); // We use prologue frequency for all of the initialization code. m_out.setFrequency(1); m_prologue = m_out.newBlock(); m_handleExceptions = m_out.newBlock(); for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) { m_highBlock = m_graph.block(blockIndex); if (!m_highBlock) continue; m_out.setFrequency(m_highBlock->executionCount); m_blocks.add(m_highBlock, m_out.newBlock()); } // Back to prologue frequency for any bocks that get sneakily created in the initialization code. m_out.setFrequency(1); m_out.appendTo(m_prologue, m_handleExceptions); m_out.initializeConstants(m_proc, m_prologue); createPhiVariables(); size_t sizeOfCaptured = sizeof(JSValue) * m_graph.m_nextMachineLocal; B3::SlotBaseValue* capturedBase = m_out.lockedStackSlot(sizeOfCaptured); m_captured = m_out.add(capturedBase, m_out.constIntPtr(sizeOfCaptured)); state->capturedValue = capturedBase->slot(); auto preOrder = m_graph.blocksInPreOrder(); m_callFrame = m_out.framePointer(); m_tagTypeNumber = m_out.constInt64(TagTypeNumber); m_tagMask = m_out.constInt64(TagMask); // Make sure that B3 knows that we really care about the mask registers. This forces the // constants to be materialized in registers. m_proc.addFastConstant(m_tagTypeNumber->key()); m_proc.addFastConstant(m_tagMask->key()); // We don't want the CodeBlock to have a weak pointer to itself because // that would cause it to always get collected. m_out.storePtr(m_out.constIntPtr(bitwise_cast(codeBlock())), addressFor(CallFrameSlot::codeBlock)); // Stack Overflow Check. unsigned exitFrameSize = m_graph.requiredRegisterCountForExit() * sizeof(Register); MacroAssembler::AbsoluteAddress addressOfStackLimit(vm().addressOfSoftStackLimit()); PatchpointValue* stackOverflowHandler = m_out.patchpoint(Void); CallSiteIndex callSiteIndex = callSiteIndexForCodeOrigin(m_ftlState, CodeOrigin(0)); stackOverflowHandler->appendSomeRegister(m_callFrame); stackOverflowHandler->clobber(RegisterSet::macroScratchRegisters()); stackOverflowHandler->numGPScratchRegisters = 1; stackOverflowHandler->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); GPRReg fp = params[0].gpr(); GPRReg scratch = params.gpScratch(0); unsigned ftlFrameSize = params.proc().frameSize(); jit.addPtr(MacroAssembler::TrustedImm32(-std::max(exitFrameSize, ftlFrameSize)), fp, scratch); MacroAssembler::Jump stackOverflow = jit.branchPtr(MacroAssembler::Above, addressOfStackLimit, scratch); params.addLatePath([=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); stackOverflow.link(&jit); jit.store32( MacroAssembler::TrustedImm32(callSiteIndex.bits()), CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount))); jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(); jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0); jit.move(CCallHelpers::TrustedImmPtr(jit.codeBlock()), GPRInfo::argumentGPR1); CCallHelpers::Call throwCall = jit.call(); jit.move(CCallHelpers::TrustedImmPtr(jit.vm()), GPRInfo::argumentGPR0); jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); CCallHelpers::Call lookupExceptionHandlerCall = jit.call(); jit.jumpToExceptionHandler(); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { linkBuffer.link(throwCall, FunctionPtr(operationThrowStackOverflowError)); linkBuffer.link(lookupExceptionHandlerCall, FunctionPtr(lookupExceptionHandlerFromCallerFrame)); }); }); }); LBasicBlock firstDFGBasicBlock = lowBlock(m_graph.block(0)); // Check Arguments. availabilityMap().clear(); availabilityMap().m_locals = Operands(codeBlock()->numParameters(), 0); for (unsigned i = codeBlock()->numParameters(); i--;) { availabilityMap().m_locals.argument(i) = Availability(FlushedAt(FlushedJSValue, virtualRegisterForArgument(i))); } m_node = nullptr; m_origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), true); for (unsigned i = codeBlock()->numParameters(); i--;) { Node* node = m_graph.m_arguments[i]; VirtualRegister operand = virtualRegisterForArgument(i); LValue jsValue = m_out.load64(addressFor(operand)); if (node) { DFG_ASSERT(m_graph, node, operand == node->stackAccessData()->machineLocal); // This is a hack, but it's an effective one. It allows us to do CSE on the // primordial load of arguments. This assumes that the GetLocal that got put in // place of the original SetArgument doesn't have any effects before it. This // should hold true. m_loadedArgumentValues.add(node, jsValue); } switch (m_graph.m_argumentFormats[i]) { case FlushedInt32: speculate(BadType, jsValueValue(jsValue), node, isNotInt32(jsValue)); break; case FlushedBoolean: speculate(BadType, jsValueValue(jsValue), node, isNotBoolean(jsValue)); break; case FlushedCell: speculate(BadType, jsValueValue(jsValue), node, isNotCell(jsValue)); break; case FlushedJSValue: break; default: DFG_CRASH(m_graph, node, "Bad flush format for argument"); break; } } m_out.jump(firstDFGBasicBlock); m_out.appendTo(m_handleExceptions, firstDFGBasicBlock); Box exceptionHandler = state->exceptionHandler; m_out.patchpoint(Void)->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams&) { CCallHelpers::Jump jump = jit.jump(); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { linkBuffer.link(jump, linkBuffer.locationOf(*exceptionHandler)); }); }); m_out.unreachable(); for (DFG::BasicBlock* block : preOrder) compileBlock(block); // Make sure everything is decorated. This does a bunch of deferred decorating. This has // to happen last because our abstract heaps are generated lazily. They have to be // generated lazily because we have an infiniten number of numbered, indexed, and // absolute heaps. We only become aware of the ones we actually mention while lowering. m_heaps.computeRangesAndDecorateInstructions(); // We create all Phi's up front, but we may then decide not to compile the basic block // that would have contained one of them. So this creates orphans, which triggers B3 // validation failures. Calling this fixes the issue. // // Note that you should avoid the temptation to make this call conditional upon // validation being enabled. B3 makes no guarantees of any kind of correctness when // dealing with IR that would have failed validation. For example, it would be valid to // write a B3 phase that so aggressively assumes the lack of orphans that it would crash // if any orphans were around. We might even have such phases already. m_proc.deleteOrphans(); // We put the blocks into the B3 procedure in a super weird order. Now we reorder them. m_out.applyBlockOrder(); } private: void createPhiVariables() { for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { DFG::BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; for (unsigned nodeIndex = block->size(); nodeIndex--;) { Node* node = block->at(nodeIndex); if (node->op() != DFG::Phi) continue; LType type; switch (node->flags() & NodeResultMask) { case NodeResultDouble: type = Double; break; case NodeResultInt32: type = Int32; break; case NodeResultInt52: type = Int64; break; case NodeResultBoolean: type = Int32; break; case NodeResultJS: type = Int64; break; default: DFG_CRASH(m_graph, node, "Bad Phi node result type"); break; } m_phis.add(node, m_proc.add(B3::Phi, type, Origin(node))); } } } void compileBlock(DFG::BasicBlock* block) { if (!block) return; if (verboseCompilationEnabled()) dataLog("Compiling block ", *block, "\n"); m_highBlock = block; // Make sure that any blocks created while lowering code in the high block have the frequency of // the high block. This is appropriate because B3 doesn't need precise frequencies. It just needs // something roughly approximate for things like register allocation. m_out.setFrequency(m_highBlock->executionCount); LBasicBlock lowBlock = m_blocks.get(m_highBlock); m_nextHighBlock = 0; for (BlockIndex nextBlockIndex = m_highBlock->index + 1; nextBlockIndex < m_graph.numBlocks(); ++nextBlockIndex) { m_nextHighBlock = m_graph.block(nextBlockIndex); if (m_nextHighBlock) break; } m_nextLowBlock = m_nextHighBlock ? m_blocks.get(m_nextHighBlock) : 0; // All of this effort to find the next block gives us the ability to keep the // generated IR in roughly program order. This ought not affect the performance // of the generated code (since we expect B3 to reorder things) but it will // make IR dumps easier to read. m_out.appendTo(lowBlock, m_nextLowBlock); if (Options::ftlCrashes()) m_out.trap(); if (!m_highBlock->cfaHasVisited) { if (verboseCompilationEnabled()) dataLog("Bailing because CFA didn't reach.\n"); crash(m_highBlock, nullptr); return; } m_availabilityCalculator.beginBlock(m_highBlock); m_state.reset(); m_state.beginBasicBlock(m_highBlock); for (m_nodeIndex = 0; m_nodeIndex < m_highBlock->size(); ++m_nodeIndex) { if (!compileNode(m_nodeIndex)) break; } } void safelyInvalidateAfterTermination() { if (verboseCompilationEnabled()) dataLog("Bailing.\n"); crash(); // Invalidate dominated blocks. Under normal circumstances we would expect // them to be invalidated already. But you can have the CFA become more // precise over time because the structures of objects change on the main // thread. Failing to do this would result in weird crashes due to a value // being used but not defined. Race conditions FTW! for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { DFG::BasicBlock* target = m_graph.block(blockIndex); if (!target) continue; if (m_graph.m_dominators->dominates(m_highBlock, target)) { if (verboseCompilationEnabled()) dataLog("Block ", *target, " will bail also.\n"); target->cfaHasVisited = false; } } } bool compileNode(unsigned nodeIndex) { if (!m_state.isValid()) { safelyInvalidateAfterTermination(); return false; } m_node = m_highBlock->at(nodeIndex); m_origin = m_node->origin; m_out.setOrigin(m_node); if (verboseCompilationEnabled()) dataLog("Lowering ", m_node, "\n"); m_availableRecoveries.resize(0); m_interpreter.startExecuting(); m_interpreter.executeKnownEdgeTypes(m_node); switch (m_node->op()) { case DFG::Upsilon: compileUpsilon(); break; case DFG::Phi: compilePhi(); break; case JSConstant: break; case DoubleConstant: compileDoubleConstant(); break; case Int52Constant: compileInt52Constant(); break; case LazyJSConstant: compileLazyJSConstant(); break; case DoubleRep: compileDoubleRep(); break; case DoubleAsInt32: compileDoubleAsInt32(); break; case DFG::ValueRep: compileValueRep(); break; case Int52Rep: compileInt52Rep(); break; case ValueToInt32: compileValueToInt32(); break; case BooleanToNumber: compileBooleanToNumber(); break; case ExtractOSREntryLocal: compileExtractOSREntryLocal(); break; case GetStack: compileGetStack(); break; case PutStack: compilePutStack(); break; case DFG::Check: compileNoOp(); break; case CallObjectConstructor: compileCallObjectConstructor(); break; case ToThis: compileToThis(); break; case ValueAdd: compileValueAdd(); break; case StrCat: compileStrCat(); break; case ArithAdd: case ArithSub: compileArithAddOrSub(); break; case ArithClz32: compileArithClz32(); break; case ArithMul: compileArithMul(); break; case ArithDiv: compileArithDiv(); break; case ArithMod: compileArithMod(); break; case ArithMin: case ArithMax: compileArithMinOrMax(); break; case ArithAbs: compileArithAbs(); break; case ArithSin: compileArithSin(); break; case ArithCos: compileArithCos(); break; case ArithTan: compileArithTan(); break; case ArithPow: compileArithPow(); break; case ArithRandom: compileArithRandom(); break; case ArithRound: compileArithRound(); break; case ArithFloor: compileArithFloor(); break; case ArithCeil: compileArithCeil(); break; case ArithTrunc: compileArithTrunc(); break; case ArithSqrt: compileArithSqrt(); break; case ArithLog: compileArithLog(); break; case ArithFRound: compileArithFRound(); break; case ArithNegate: compileArithNegate(); break; case DFG::BitAnd: compileBitAnd(); break; case DFG::BitOr: compileBitOr(); break; case DFG::BitXor: compileBitXor(); break; case BitRShift: compileBitRShift(); break; case BitLShift: compileBitLShift(); break; case BitURShift: compileBitURShift(); break; case UInt32ToNumber: compileUInt32ToNumber(); break; case CheckStructure: compileCheckStructure(); break; case CheckCell: compileCheckCell(); break; case CheckNotEmpty: compileCheckNotEmpty(); break; case CheckBadCell: compileCheckBadCell(); break; case CheckStringIdent: compileCheckStringIdent(); break; case GetExecutable: compileGetExecutable(); break; case ArrayifyToStructure: compileArrayifyToStructure(); break; case PutStructure: compilePutStructure(); break; case TryGetById: compileGetById(AccessType::TryGet); break; case GetById: case GetByIdFlush: compileGetById(AccessType::Get); break; case GetByIdWithThis: compileGetByIdWithThis(); break; case In: compileIn(); break; case HasOwnProperty: compileHasOwnProperty(); break; case PutById: case PutByIdDirect: case PutByIdFlush: compilePutById(); break; case PutByIdWithThis: compilePutByIdWithThis(); break; case PutGetterById: case PutSetterById: compilePutAccessorById(); break; case PutGetterSetterById: compilePutGetterSetterById(); break; case PutGetterByVal: case PutSetterByVal: compilePutAccessorByVal(); break; case GetButterfly: compileGetButterfly(); break; case ConstantStoragePointer: compileConstantStoragePointer(); break; case GetIndexedPropertyStorage: compileGetIndexedPropertyStorage(); break; case CheckArray: compileCheckArray(); break; case GetArrayLength: compileGetArrayLength(); break; case CheckInBounds: compileCheckInBounds(); break; case GetByVal: compileGetByVal(); break; case GetMyArgumentByVal: case GetMyArgumentByValOutOfBounds: compileGetMyArgumentByVal(); break; case GetByValWithThis: compileGetByValWithThis(); break; case PutByVal: case PutByValAlias: case PutByValDirect: compilePutByVal(); break; case PutByValWithThis: compilePutByValWithThis(); break; case DefineDataProperty: compileDefineDataProperty(); break; case DefineAccessorProperty: compileDefineAccessorProperty(); break; case ArrayPush: compileArrayPush(); break; case ArrayPop: compileArrayPop(); break; case ArraySlice: compileArraySlice(); break; case CreateActivation: compileCreateActivation(); break; case NewFunction: case NewGeneratorFunction: case NewAsyncFunction: compileNewFunction(); break; case CreateDirectArguments: compileCreateDirectArguments(); break; case CreateScopedArguments: compileCreateScopedArguments(); break; case CreateClonedArguments: compileCreateClonedArguments(); break; case NewObject: compileNewObject(); break; case NewArray: compileNewArray(); break; case NewArrayWithSpread: compileNewArrayWithSpread(); break; case Spread: compileSpread(); break; case NewArrayBuffer: compileNewArrayBuffer(); break; case NewArrayWithSize: compileNewArrayWithSize(); break; case NewTypedArray: compileNewTypedArray(); break; case GetTypedArrayByteOffset: compileGetTypedArrayByteOffset(); break; case AllocatePropertyStorage: compileAllocatePropertyStorage(); break; case ReallocatePropertyStorage: compileReallocatePropertyStorage(); break; case NukeStructureAndSetButterfly: compileNukeStructureAndSetButterfly(); break; case ToNumber: compileToNumber(); break; case ToString: case CallStringConstructor: compileToStringOrCallStringConstructor(); break; case ToPrimitive: compileToPrimitive(); break; case MakeRope: compileMakeRope(); break; case StringCharAt: compileStringCharAt(); break; case StringCharCodeAt: compileStringCharCodeAt(); break; case StringFromCharCode: compileStringFromCharCode(); break; case GetByOffset: case GetGetterSetterByOffset: compileGetByOffset(); break; case GetGetter: compileGetGetter(); break; case GetSetter: compileGetSetter(); break; case MultiGetByOffset: compileMultiGetByOffset(); break; case PutByOffset: compilePutByOffset(); break; case MultiPutByOffset: compileMultiPutByOffset(); break; case GetGlobalVar: case GetGlobalLexicalVariable: compileGetGlobalVariable(); break; case PutGlobalVariable: compilePutGlobalVariable(); break; case NotifyWrite: compileNotifyWrite(); break; case GetCallee: compileGetCallee(); break; case GetArgumentCountIncludingThis: compileGetArgumentCountIncludingThis(); break; case GetScope: compileGetScope(); break; case SkipScope: compileSkipScope(); break; case GetGlobalObject: compileGetGlobalObject(); break; case GetClosureVar: compileGetClosureVar(); break; case PutClosureVar: compilePutClosureVar(); break; case GetFromArguments: compileGetFromArguments(); break; case PutToArguments: compilePutToArguments(); break; case GetArgument: compileGetArgument(); break; case CompareEq: compileCompareEq(); break; case CompareStrictEq: compileCompareStrictEq(); break; case CompareLess: compileCompareLess(); break; case CompareLessEq: compileCompareLessEq(); break; case CompareGreater: compileCompareGreater(); break; case CompareGreaterEq: compileCompareGreaterEq(); break; case CompareEqPtr: compileCompareEqPtr(); break; case LogicalNot: compileLogicalNot(); break; case Call: case TailCallInlinedCaller: case Construct: compileCallOrConstruct(); break; case DirectCall: case DirectTailCallInlinedCaller: case DirectConstruct: case DirectTailCall: compileDirectCallOrConstruct(); break; case TailCall: compileTailCall(); break; case CallVarargs: case CallForwardVarargs: case TailCallVarargs: case TailCallVarargsInlinedCaller: case TailCallForwardVarargs: case TailCallForwardVarargsInlinedCaller: case ConstructVarargs: case ConstructForwardVarargs: compileCallOrConstructVarargs(); break; case CallEval: compileCallEval(); break; case LoadVarargs: compileLoadVarargs(); break; case ForwardVarargs: compileForwardVarargs(); break; case DFG::Jump: compileJump(); break; case DFG::Branch: compileBranch(); break; case DFG::Switch: compileSwitch(); break; case DFG::Return: compileReturn(); break; case ForceOSRExit: compileForceOSRExit(); break; case Throw: case ThrowStaticError: compileThrow(); break; case InvalidationPoint: compileInvalidationPoint(); break; case IsEmpty: compileIsEmpty(); break; case IsUndefined: compileIsUndefined(); break; case IsBoolean: compileIsBoolean(); break; case IsNumber: compileIsNumber(); break; case IsCellWithType: compileIsCellWithType(); break; case MapHash: compileMapHash(); break; case GetMapBucket: compileGetMapBucket(); break; case LoadFromJSMapBucket: compileLoadFromJSMapBucket(); break; case IsNonEmptyMapBucket: compileIsNonEmptyMapBucket(); break; case IsObject: compileIsObject(); break; case IsObjectOrNull: compileIsObjectOrNull(); break; case IsFunction: compileIsFunction(); break; case IsTypedArrayView: compileIsTypedArrayView(); break; case ParseInt: compileParseInt(); break; case TypeOf: compileTypeOf(); break; case CheckTypeInfoFlags: compileCheckTypeInfoFlags(); break; case OverridesHasInstance: compileOverridesHasInstance(); break; case InstanceOf: compileInstanceOf(); break; case InstanceOfCustom: compileInstanceOfCustom(); break; case CountExecution: compileCountExecution(); break; case StoreBarrier: case FencedStoreBarrier: compileStoreBarrier(); break; case HasIndexedProperty: compileHasIndexedProperty(); break; case HasGenericProperty: compileHasGenericProperty(); break; case HasStructureProperty: compileHasStructureProperty(); break; case GetDirectPname: compileGetDirectPname(); break; case GetEnumerableLength: compileGetEnumerableLength(); break; case GetPropertyEnumerator: compileGetPropertyEnumerator(); break; case GetEnumeratorStructurePname: compileGetEnumeratorStructurePname(); break; case GetEnumeratorGenericPname: compileGetEnumeratorGenericPname(); break; case ToIndexString: compileToIndexString(); break; case CheckStructureImmediate: compileCheckStructureImmediate(); break; case MaterializeNewObject: compileMaterializeNewObject(); break; case MaterializeCreateActivation: compileMaterializeCreateActivation(); break; case CheckWatchdogTimer: compileCheckWatchdogTimer(); break; case CreateRest: compileCreateRest(); break; case GetRestLength: compileGetRestLength(); break; case RegExpExec: compileRegExpExec(); break; case RegExpTest: compileRegExpTest(); break; case NewRegexp: compileNewRegexp(); break; case SetFunctionName: compileSetFunctionName(); break; case StringReplace: case StringReplaceRegExp: compileStringReplace(); break; case GetRegExpObjectLastIndex: compileGetRegExpObjectLastIndex(); break; case SetRegExpObjectLastIndex: compileSetRegExpObjectLastIndex(); break; case LogShadowChickenPrologue: compileLogShadowChickenPrologue(); break; case LogShadowChickenTail: compileLogShadowChickenTail(); break; case RecordRegExpCachedResult: compileRecordRegExpCachedResult(); break; case ResolveScope: compileResolveScope(); break; case GetDynamicVar: compileGetDynamicVar(); break; case PutDynamicVar: compilePutDynamicVar(); break; case Unreachable: compileUnreachable(); break; case ToLowerCase: compileToLowerCase(); break; case NumberToStringWithRadix: compileNumberToStringWithRadix(); break; case CheckDOM: compileCheckDOM(); break; case CallDOM: compileCallDOM(); break; case CallDOMGetter: compileCallDOMGetter(); break; case PhantomLocal: case LoopHint: case MovHint: case ZombieHint: case ExitOK: case PhantomNewObject: case PhantomNewFunction: case PhantomNewGeneratorFunction: case PhantomNewAsyncFunction: case PhantomCreateActivation: case PhantomDirectArguments: case PhantomCreateRest: case PhantomSpread: case PhantomNewArrayWithSpread: case PhantomClonedArguments: case PutHint: case BottomValue: case KillStack: break; default: DFG_CRASH(m_graph, m_node, "Unrecognized node in FTL backend"); break; } if (m_node->isTerminal()) return false; if (!m_state.isValid()) { safelyInvalidateAfterTermination(); return false; } m_availabilityCalculator.executeNode(m_node); m_interpreter.executeEffects(nodeIndex); return true; } void compileUpsilon() { LValue upsilonValue = nullptr; switch (m_node->child1().useKind()) { case DoubleRepUse: upsilonValue = lowDouble(m_node->child1()); break; case Int32Use: case KnownInt32Use: upsilonValue = lowInt32(m_node->child1()); break; case Int52RepUse: upsilonValue = lowInt52(m_node->child1()); break; case BooleanUse: case KnownBooleanUse: upsilonValue = lowBoolean(m_node->child1()); break; case CellUse: case KnownCellUse: upsilonValue = lowCell(m_node->child1()); break; case UntypedUse: upsilonValue = lowJSValue(m_node->child1()); break; default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } ValueFromBlock upsilon = m_out.anchor(upsilonValue); LValue phiNode = m_phis.get(m_node->phi()); m_out.addIncomingToPhi(phiNode, upsilon); } void compilePhi() { LValue phi = m_phis.get(m_node); m_out.m_block->append(phi); switch (m_node->flags() & NodeResultMask) { case NodeResultDouble: setDouble(phi); break; case NodeResultInt32: setInt32(phi); break; case NodeResultInt52: setInt52(phi); break; case NodeResultBoolean: setBoolean(phi); break; case NodeResultJS: setJSValue(phi); break; default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } } void compileDoubleConstant() { setDouble(m_out.constDouble(m_node->asNumber())); } void compileInt52Constant() { int64_t value = m_node->asAnyInt(); setInt52(m_out.constInt64(value << JSValue::int52ShiftAmount)); setStrictInt52(m_out.constInt64(value)); } void compileLazyJSConstant() { PatchpointValue* patchpoint = m_out.patchpoint(Int64); LazyJSValue value = m_node->lazyJSValue(); patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { value.emit(jit, JSValueRegs(params[0].gpr())); }); patchpoint->effects = Effects::none(); setJSValue(patchpoint); } void compileDoubleRep() { switch (m_node->child1().useKind()) { case RealNumberUse: { LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation); LValue doubleValue = unboxDouble(value); LBasicBlock intCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock fastResult = m_out.anchor(doubleValue); m_out.branch( m_out.doubleEqual(doubleValue, doubleValue), usually(continuation), rarely(intCase)); LBasicBlock lastNext = m_out.appendTo(intCase, continuation); FTL_TYPE_CHECK( jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber, isNotInt32(value, provenType(m_node->child1()) & ~SpecDoubleReal)); ValueFromBlock slowResult = m_out.anchor(m_out.intToDouble(unboxInt32(value))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setDouble(m_out.phi(Double, fastResult, slowResult)); return; } case NotCellUse: case NumberUse: { bool shouldConvertNonNumber = m_node->child1().useKind() == NotCellUse; LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation); LBasicBlock intCase = m_out.newBlock(); LBasicBlock doubleTesting = m_out.newBlock(); LBasicBlock doubleCase = m_out.newBlock(); LBasicBlock nonDoubleCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( isNotInt32(value, provenType(m_node->child1())), unsure(doubleTesting), unsure(intCase)); LBasicBlock lastNext = m_out.appendTo(intCase, doubleTesting); ValueFromBlock intToDouble = m_out.anchor( m_out.intToDouble(unboxInt32(value))); m_out.jump(continuation); m_out.appendTo(doubleTesting, doubleCase); LValue valueIsNumber = isNumber(value, provenType(m_node->child1())); m_out.branch(valueIsNumber, usually(doubleCase), rarely(nonDoubleCase)); m_out.appendTo(doubleCase, nonDoubleCase); ValueFromBlock unboxedDouble = m_out.anchor(unboxDouble(value)); m_out.jump(continuation); if (shouldConvertNonNumber) { LBasicBlock undefinedCase = m_out.newBlock(); LBasicBlock testNullCase = m_out.newBlock(); LBasicBlock nullCase = m_out.newBlock(); LBasicBlock testBooleanTrueCase = m_out.newBlock(); LBasicBlock convertBooleanTrueCase = m_out.newBlock(); LBasicBlock convertBooleanFalseCase = m_out.newBlock(); m_out.appendTo(nonDoubleCase, undefinedCase); LValue valueIsUndefined = m_out.equal(value, m_out.constInt64(ValueUndefined)); m_out.branch(valueIsUndefined, unsure(undefinedCase), unsure(testNullCase)); m_out.appendTo(undefinedCase, testNullCase); ValueFromBlock convertedUndefined = m_out.anchor(m_out.constDouble(PNaN)); m_out.jump(continuation); m_out.appendTo(testNullCase, nullCase); LValue valueIsNull = m_out.equal(value, m_out.constInt64(ValueNull)); m_out.branch(valueIsNull, unsure(nullCase), unsure(testBooleanTrueCase)); m_out.appendTo(nullCase, testBooleanTrueCase); ValueFromBlock convertedNull = m_out.anchor(m_out.constDouble(0)); m_out.jump(continuation); m_out.appendTo(testBooleanTrueCase, convertBooleanTrueCase); LValue valueIsBooleanTrue = m_out.equal(value, m_out.constInt64(ValueTrue)); m_out.branch(valueIsBooleanTrue, unsure(convertBooleanTrueCase), unsure(convertBooleanFalseCase)); m_out.appendTo(convertBooleanTrueCase, convertBooleanFalseCase); ValueFromBlock convertedTrue = m_out.anchor(m_out.constDouble(1)); m_out.jump(continuation); m_out.appendTo(convertBooleanFalseCase, continuation); LValue valueIsNotBooleanFalse = m_out.notEqual(value, m_out.constInt64(ValueFalse)); FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), ~SpecCell, valueIsNotBooleanFalse); ValueFromBlock convertedFalse = m_out.anchor(m_out.constDouble(0)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setDouble(m_out.phi(Double, intToDouble, unboxedDouble, convertedUndefined, convertedNull, convertedTrue, convertedFalse)); return; } m_out.appendTo(nonDoubleCase, continuation); FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecBytecodeNumber, m_out.booleanTrue); m_out.unreachable(); m_out.appendTo(continuation, lastNext); setDouble(m_out.phi(Double, intToDouble, unboxedDouble)); return; } case Int52RepUse: { setDouble(strictInt52ToDouble(lowStrictInt52(m_node->child1()))); return; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); } } void compileDoubleAsInt32() { LValue integerValue = convertDoubleToInt32(lowDouble(m_node->child1()), shouldCheckNegativeZero(m_node->arithMode())); setInt32(integerValue); } void compileValueRep() { switch (m_node->child1().useKind()) { case DoubleRepUse: { LValue value = lowDouble(m_node->child1()); if (m_interpreter.needsTypeCheck(m_node->child1(), ~SpecDoubleImpureNaN)) { value = m_out.select( m_out.doubleEqual(value, value), value, m_out.constDouble(PNaN)); } setJSValue(boxDouble(value)); return; } case Int52RepUse: { setJSValue(strictInt52ToJSValue(lowStrictInt52(m_node->child1()))); return; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); } } void compileInt52Rep() { switch (m_node->child1().useKind()) { case Int32Use: setStrictInt52(m_out.signExt32To64(lowInt32(m_node->child1()))); return; case AnyIntUse: setStrictInt52( jsValueToStrictInt52( m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation))); return; case DoubleRepAnyIntUse: setStrictInt52( doubleToStrictInt52( m_node->child1(), lowDouble(m_node->child1()))); return; default: RELEASE_ASSERT_NOT_REACHED(); } } void compileValueToInt32() { switch (m_node->child1().useKind()) { case Int52RepUse: setInt32(m_out.castToInt32(lowStrictInt52(m_node->child1()))); break; case DoubleRepUse: setInt32(doubleToInt32(lowDouble(m_node->child1()))); break; case NumberUse: case NotCellUse: { LoweredNodeValue value = m_int32Values.get(m_node->child1().node()); if (isValid(value)) { setInt32(value.value()); break; } value = m_jsValueValues.get(m_node->child1().node()); if (isValid(value)) { setInt32(numberOrNotCellToInt32(m_node->child1(), value.value())); break; } // We'll basically just get here for constants. But it's good to have this // catch-all since we often add new representations into the mix. setInt32( numberOrNotCellToInt32( m_node->child1(), lowJSValue(m_node->child1(), ManualOperandSpeculation))); break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } } void compileBooleanToNumber() { switch (m_node->child1().useKind()) { case BooleanUse: { setInt32(m_out.zeroExt(lowBoolean(m_node->child1()), Int32)); return; } case UntypedUse: { LValue value = lowJSValue(m_node->child1()); if (!m_interpreter.needsTypeCheck(m_node->child1(), SpecBoolInt32 | SpecBoolean)) { setInt32(m_out.bitAnd(m_out.castToInt32(value), m_out.int32One)); return; } LBasicBlock booleanCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock notBooleanResult = m_out.anchor(value); m_out.branch( isBoolean(value, provenType(m_node->child1())), unsure(booleanCase), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(booleanCase, continuation); ValueFromBlock booleanResult = m_out.anchor(m_out.bitOr( m_out.zeroExt(unboxBoolean(value), Int64), m_tagTypeNumber)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, booleanResult, notBooleanResult)); return; } default: RELEASE_ASSERT_NOT_REACHED(); return; } } void compileExtractOSREntryLocal() { EncodedJSValue* buffer = static_cast( m_ftlState.jitCode->ftlForOSREntry()->entryBuffer()->dataBuffer()); setJSValue(m_out.load64(m_out.absolute(buffer + m_node->unlinkedLocal().toLocal()))); } void compileGetStack() { // GetLocals arise only for captured variables and arguments. For arguments, we might have // already loaded it. if (LValue value = m_loadedArgumentValues.get(m_node)) { setJSValue(value); return; } StackAccessData* data = m_node->stackAccessData(); AbstractValue& value = m_state.variables().operand(data->local); DFG_ASSERT(m_graph, m_node, isConcrete(data->format)); DFG_ASSERT(m_graph, m_node, data->format != FlushedDouble); // This just happens to not arise for GetStacks, right now. It would be trivial to support. if (isInt32Speculation(value.m_type)) setInt32(m_out.load32(payloadFor(data->machineLocal))); else setJSValue(m_out.load64(addressFor(data->machineLocal))); } void compilePutStack() { StackAccessData* data = m_node->stackAccessData(); switch (data->format) { case FlushedJSValue: { LValue value = lowJSValue(m_node->child1()); m_out.store64(value, addressFor(data->machineLocal)); break; } case FlushedDouble: { LValue value = lowDouble(m_node->child1()); m_out.storeDouble(value, addressFor(data->machineLocal)); break; } case FlushedInt32: { LValue value = lowInt32(m_node->child1()); m_out.store32(value, payloadFor(data->machineLocal)); break; } case FlushedInt52: { LValue value = lowInt52(m_node->child1()); m_out.store64(value, addressFor(data->machineLocal)); break; } case FlushedCell: { LValue value = lowCell(m_node->child1()); m_out.store64(value, addressFor(data->machineLocal)); break; } case FlushedBoolean: { speculateBoolean(m_node->child1()); m_out.store64( lowJSValue(m_node->child1(), ManualOperandSpeculation), addressFor(data->machineLocal)); break; } default: DFG_CRASH(m_graph, m_node, "Bad flush format"); break; } } void compileNoOp() { DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, speculate); } void compileCallObjectConstructor() { JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); LValue value = lowJSValue(m_node->child1()); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase)); LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase); ValueFromBlock fastResult = m_out.anchor(value); m_out.branch(isObject(value), usually(continuation), rarely(slowCase)); m_out.appendTo(slowCase, continuation); ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, m_out.operation(operationObjectConstructor), m_callFrame, weakPointer(globalObject), value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, fastResult, slowResult)); } void compileToThis() { LValue value = lowJSValue(m_node->child1()); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( isCell(value, provenType(m_node->child1())), usually(isCellCase), rarely(slowCase)); LBasicBlock lastNext = m_out.appendTo(isCellCase, slowCase); ValueFromBlock fastResult = m_out.anchor(value); m_out.branch( m_out.testIsZero32( m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoFlags), m_out.constInt32(OverridesToThis)), usually(continuation), rarely(slowCase)); m_out.appendTo(slowCase, continuation); J_JITOperation_EJ function; if (m_graph.isStrictModeFor(m_node->origin.semantic)) function = operationToThisStrict; else function = operationToThis; ValueFromBlock slowResult = m_out.anchor( vmCall(Int64, m_out.operation(function), m_callFrame, value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, fastResult, slowResult)); } void compileValueAdd() { ArithProfile* arithProfile = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic)->arithProfileForBytecodeOffset(m_node->origin.semantic.bytecodeIndex); JITAddIC* addIC = codeBlock()->addJITAddIC(arithProfile); auto repatchingFunction = operationValueAddOptimize; auto nonRepatchingFunction = operationValueAdd; compileMathIC(addIC, repatchingFunction, nonRepatchingFunction); } template void compileMathIC(JITUnaryMathIC* mathIC, FunctionPtr repatchingFunction, FunctionPtr nonRepatchingFunction) { Node* node = m_node; LValue operand = lowJSValue(node->child1()); PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendSomeRegister(operand); patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister)); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->numGPScratchRegisters = 1; patchpoint->clobber(RegisterSet::macroScratchRegisters()); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); #if ENABLE(MATH_IC_STATS) auto inlineStart = jit.label(); #endif Box mathICGenerationState = Box::create(); mathIC->m_generator = Generator(JSValueRegs(params[0].gpr()), JSValueRegs(params[1].gpr()), params.gpScratch(0)); bool shouldEmitProfiling = false; bool generatedInline = mathIC->generateInline(jit, *mathICGenerationState, shouldEmitProfiling); if (generatedInline) { ASSERT(!mathICGenerationState->slowPathJumps.empty()); auto done = jit.label(); params.addLatePath([=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); mathICGenerationState->slowPathJumps.link(&jit); mathICGenerationState->slowPathStart = jit.label(); #if ENABLE(MATH_IC_STATS) auto slowPathStart = jit.label(); #endif if (mathICGenerationState->shouldSlowPathRepatch) { SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), repatchingFunction, params[0].gpr(), params[1].gpr(), CCallHelpers::TrustedImmPtr(mathIC)); mathICGenerationState->slowPathCall = call.call(); } else { SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), nonRepatchingFunction, params[0].gpr(), params[1].gpr()); mathICGenerationState->slowPathCall = call.call(); } jit.jump().linkTo(done, &jit); jit.addLinkTask([=] (LinkBuffer& linkBuffer) { mathIC->finalizeInlineCode(*mathICGenerationState, linkBuffer); }); #if ENABLE(MATH_IC_STATS) auto slowPathEnd = jit.label(); jit.addLinkTask([=] (LinkBuffer& linkBuffer) { size_t size = static_cast(linkBuffer.locationOf(slowPathEnd).executableAddress()) - static_cast(linkBuffer.locationOf(slowPathStart).executableAddress()); mathIC->m_generatedCodeSize += size; }); #endif }); } else { callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), nonRepatchingFunction, params[0].gpr(), params[1].gpr()); } #if ENABLE(MATH_IC_STATS) auto inlineEnd = jit.label(); jit.addLinkTask([=] (LinkBuffer& linkBuffer) { size_t size = static_cast(linkBuffer.locationOf(inlineEnd).executableAddress()) - static_cast(linkBuffer.locationOf(inlineStart).executableAddress()); mathIC->m_generatedCodeSize += size; }); #endif }); setJSValue(patchpoint); } template void compileMathIC(JITBinaryMathIC* mathIC, FunctionPtr repatchingFunction, FunctionPtr nonRepatchingFunction) { Node* node = m_node; LValue left = lowJSValue(node->child1()); LValue right = lowJSValue(node->child2()); SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType()); SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType()); PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendSomeRegister(left); patchpoint->appendSomeRegister(right); patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister)); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->numGPScratchRegisters = 1; patchpoint->numFPScratchRegisters = 2; patchpoint->clobber(RegisterSet::macroScratchRegisters()); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); #if ENABLE(MATH_IC_STATS) auto inlineStart = jit.label(); #endif Box mathICGenerationState = Box::create(); mathIC->m_generator = Generator(leftOperand, rightOperand, JSValueRegs(params[0].gpr()), JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()), params.fpScratch(0), params.fpScratch(1), params.gpScratch(0), InvalidFPRReg); bool shouldEmitProfiling = false; bool generatedInline = mathIC->generateInline(jit, *mathICGenerationState, shouldEmitProfiling); if (generatedInline) { ASSERT(!mathICGenerationState->slowPathJumps.empty()); auto done = jit.label(); params.addLatePath([=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); mathICGenerationState->slowPathJumps.link(&jit); mathICGenerationState->slowPathStart = jit.label(); #if ENABLE(MATH_IC_STATS) auto slowPathStart = jit.label(); #endif if (mathICGenerationState->shouldSlowPathRepatch) { SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), repatchingFunction, params[0].gpr(), params[1].gpr(), params[2].gpr(), CCallHelpers::TrustedImmPtr(mathIC)); mathICGenerationState->slowPathCall = call.call(); } else { SlowPathCall call = callOperation(*state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), nonRepatchingFunction, params[0].gpr(), params[1].gpr(), params[2].gpr()); mathICGenerationState->slowPathCall = call.call(); } jit.jump().linkTo(done, &jit); jit.addLinkTask([=] (LinkBuffer& linkBuffer) { mathIC->finalizeInlineCode(*mathICGenerationState, linkBuffer); }); #if ENABLE(MATH_IC_STATS) auto slowPathEnd = jit.label(); jit.addLinkTask([=] (LinkBuffer& linkBuffer) { size_t size = static_cast(linkBuffer.locationOf(slowPathEnd).executableAddress()) - static_cast(linkBuffer.locationOf(slowPathStart).executableAddress()); mathIC->m_generatedCodeSize += size; }); #endif }); } else { callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), nonRepatchingFunction, params[0].gpr(), params[1].gpr(), params[2].gpr()); } #if ENABLE(MATH_IC_STATS) auto inlineEnd = jit.label(); jit.addLinkTask([=] (LinkBuffer& linkBuffer) { size_t size = static_cast(linkBuffer.locationOf(inlineEnd).executableAddress()) - static_cast(linkBuffer.locationOf(inlineStart).executableAddress()); mathIC->m_generatedCodeSize += size; }); #endif }); setJSValue(patchpoint); } void compileStrCat() { LValue result; if (m_node->child3()) { result = vmCall( Int64, m_out.operation(operationStrCat3), m_callFrame, lowJSValue(m_node->child1(), ManualOperandSpeculation), lowJSValue(m_node->child2(), ManualOperandSpeculation), lowJSValue(m_node->child3(), ManualOperandSpeculation)); } else { result = vmCall( Int64, m_out.operation(operationStrCat2), m_callFrame, lowJSValue(m_node->child1(), ManualOperandSpeculation), lowJSValue(m_node->child2(), ManualOperandSpeculation)); } setJSValue(result); } void compileArithAddOrSub() { bool isSub = m_node->op() == ArithSub; switch (m_node->binaryUseKind()) { case Int32Use: { LValue left = lowInt32(m_node->child1()); LValue right = lowInt32(m_node->child2()); if (!shouldCheckOverflow(m_node->arithMode())) { setInt32(isSub ? m_out.sub(left, right) : m_out.add(left, right)); break; } CheckValue* result = isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right); blessSpeculation(result, Overflow, noValue(), nullptr, m_origin); setInt32(result); break; } case Int52RepUse: { if (!abstractValue(m_node->child1()).couldBeType(SpecInt52Only) && !abstractValue(m_node->child2()).couldBeType(SpecInt52Only)) { Int52Kind kind; LValue left = lowWhicheverInt52(m_node->child1(), kind); LValue right = lowInt52(m_node->child2(), kind); setInt52(isSub ? m_out.sub(left, right) : m_out.add(left, right), kind); break; } LValue left = lowInt52(m_node->child1()); LValue right = lowInt52(m_node->child2()); CheckValue* result = isSub ? m_out.speculateSub(left, right) : m_out.speculateAdd(left, right); blessSpeculation(result, Overflow, noValue(), nullptr, m_origin); setInt52(result); break; } case DoubleRepUse: { LValue C1 = lowDouble(m_node->child1()); LValue C2 = lowDouble(m_node->child2()); setDouble(isSub ? m_out.doubleSub(C1, C2) : m_out.doubleAdd(C1, C2)); break; } case UntypedUse: { if (!isSub) { DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } ArithProfile* arithProfile = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic)->arithProfileForBytecodeOffset(m_node->origin.semantic.bytecodeIndex); JITSubIC* subIC = codeBlock()->addJITSubIC(arithProfile); auto repatchingFunction = operationValueSubOptimize; auto nonRepatchingFunction = operationValueSub; compileMathIC(subIC, repatchingFunction, nonRepatchingFunction); break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } } void compileArithClz32() { if (m_node->child1().useKind() == Int32Use || m_node->child1().useKind() == KnownInt32Use) { LValue operand = lowInt32(m_node->child1()); setInt32(m_out.ctlz32(operand)); return; } DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse); LValue argument = lowJSValue(m_node->child1()); LValue result = vmCall(Int32, m_out.operation(operationArithClz32), m_callFrame, argument); setInt32(result); } void compileArithMul() { switch (m_node->binaryUseKind()) { case Int32Use: { LValue left = lowInt32(m_node->child1()); LValue right = lowInt32(m_node->child2()); LValue result; if (!shouldCheckOverflow(m_node->arithMode())) result = m_out.mul(left, right); else { CheckValue* speculation = m_out.speculateMul(left, right); blessSpeculation(speculation, Overflow, noValue(), nullptr, m_origin); result = speculation; } if (shouldCheckNegativeZero(m_node->arithMode())) { LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.notZero32(result), usually(continuation), rarely(slowCase)); LBasicBlock lastNext = m_out.appendTo(slowCase, continuation); speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int32Zero)); speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int32Zero)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } setInt32(result); break; } case Int52RepUse: { Int52Kind kind; LValue left = lowWhicheverInt52(m_node->child1(), kind); LValue right = lowInt52(m_node->child2(), opposite(kind)); CheckValue* result = m_out.speculateMul(left, right); blessSpeculation(result, Overflow, noValue(), nullptr, m_origin); if (shouldCheckNegativeZero(m_node->arithMode())) { LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.notZero64(result), usually(continuation), rarely(slowCase)); LBasicBlock lastNext = m_out.appendTo(slowCase, continuation); speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(left, m_out.int64Zero)); speculate(NegativeZero, noValue(), nullptr, m_out.lessThan(right, m_out.int64Zero)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } setInt52(result); break; } case DoubleRepUse: { setDouble( m_out.doubleMul(lowDouble(m_node->child1()), lowDouble(m_node->child2()))); break; } case UntypedUse: { ArithProfile* arithProfile = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic)->arithProfileForBytecodeOffset(m_node->origin.semantic.bytecodeIndex); JITMulIC* mulIC = codeBlock()->addJITMulIC(arithProfile); auto repatchingFunction = operationValueMulOptimize; auto nonRepatchingFunction = operationValueMul; compileMathIC(mulIC, repatchingFunction, nonRepatchingFunction); break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } } void compileArithDiv() { switch (m_node->binaryUseKind()) { case Int32Use: { LValue numerator = lowInt32(m_node->child1()); LValue denominator = lowInt32(m_node->child2()); if (shouldCheckNegativeZero(m_node->arithMode())) { LBasicBlock zeroNumerator = m_out.newBlock(); LBasicBlock numeratorContinuation = m_out.newBlock(); m_out.branch( m_out.isZero32(numerator), rarely(zeroNumerator), usually(numeratorContinuation)); LBasicBlock innerLastNext = m_out.appendTo(zeroNumerator, numeratorContinuation); speculate( NegativeZero, noValue(), 0, m_out.lessThan(denominator, m_out.int32Zero)); m_out.jump(numeratorContinuation); m_out.appendTo(numeratorContinuation, innerLastNext); } if (shouldCheckOverflow(m_node->arithMode())) { LBasicBlock unsafeDenominator = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue adjustedDenominator = m_out.add(denominator, m_out.int32One); m_out.branch( m_out.above(adjustedDenominator, m_out.int32One), usually(continuation), rarely(unsafeDenominator)); LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation); LValue neg2ToThe31 = m_out.constInt32(-2147483647-1); speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator)); speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); LValue result = m_out.div(numerator, denominator); speculate( Overflow, noValue(), 0, m_out.notEqual(m_out.mul(result, denominator), numerator)); setInt32(result); } else setInt32(m_out.chillDiv(numerator, denominator)); break; } case DoubleRepUse: { setDouble(m_out.doubleDiv( lowDouble(m_node->child1()), lowDouble(m_node->child2()))); break; } case UntypedUse: { emitBinarySnippet(operationValueDiv); break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } } void compileArithMod() { switch (m_node->binaryUseKind()) { case Int32Use: { LValue numerator = lowInt32(m_node->child1()); LValue denominator = lowInt32(m_node->child2()); LValue remainder; if (shouldCheckOverflow(m_node->arithMode())) { LBasicBlock unsafeDenominator = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue adjustedDenominator = m_out.add(denominator, m_out.int32One); m_out.branch( m_out.above(adjustedDenominator, m_out.int32One), usually(continuation), rarely(unsafeDenominator)); LBasicBlock lastNext = m_out.appendTo(unsafeDenominator, continuation); LValue neg2ToThe31 = m_out.constInt32(-2147483647-1); speculate(Overflow, noValue(), nullptr, m_out.isZero32(denominator)); speculate(Overflow, noValue(), nullptr, m_out.equal(numerator, neg2ToThe31)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); LValue result = m_out.mod(numerator, denominator); remainder = result; } else remainder = m_out.chillMod(numerator, denominator); if (shouldCheckNegativeZero(m_node->arithMode())) { LBasicBlock negativeNumerator = m_out.newBlock(); LBasicBlock numeratorContinuation = m_out.newBlock(); m_out.branch( m_out.lessThan(numerator, m_out.int32Zero), unsure(negativeNumerator), unsure(numeratorContinuation)); LBasicBlock innerLastNext = m_out.appendTo(negativeNumerator, numeratorContinuation); speculate(NegativeZero, noValue(), 0, m_out.isZero32(remainder)); m_out.jump(numeratorContinuation); m_out.appendTo(numeratorContinuation, innerLastNext); } setInt32(remainder); break; } case DoubleRepUse: { setDouble( m_out.doubleMod(lowDouble(m_node->child1()), lowDouble(m_node->child2()))); break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } } void compileArithMinOrMax() { switch (m_node->binaryUseKind()) { case Int32Use: { LValue left = lowInt32(m_node->child1()); LValue right = lowInt32(m_node->child2()); setInt32( m_out.select( m_node->op() == ArithMin ? m_out.lessThan(left, right) : m_out.lessThan(right, left), left, right)); break; } case DoubleRepUse: { LValue left = lowDouble(m_node->child1()); LValue right = lowDouble(m_node->child2()); LBasicBlock notLessThan = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); Vector results; results.append(m_out.anchor(left)); m_out.branch( m_node->op() == ArithMin ? m_out.doubleLessThan(left, right) : m_out.doubleGreaterThan(left, right), unsure(continuation), unsure(notLessThan)); LBasicBlock lastNext = m_out.appendTo(notLessThan, continuation); results.append(m_out.anchor(m_out.select( m_node->op() == ArithMin ? m_out.doubleGreaterThanOrEqual(left, right) : m_out.doubleLessThanOrEqual(left, right), right, m_out.constDouble(PNaN)))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setDouble(m_out.phi(Double, results)); break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } } void compileArithAbs() { switch (m_node->child1().useKind()) { case Int32Use: { LValue value = lowInt32(m_node->child1()); LValue mask = m_out.aShr(value, m_out.constInt32(31)); LValue result = m_out.bitXor(mask, m_out.add(mask, value)); if (shouldCheckOverflow(m_node->arithMode())) speculate(Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero)); setInt32(result); break; } case DoubleRepUse: { setDouble(m_out.doubleAbs(lowDouble(m_node->child1()))); break; } default: { DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse); LValue argument = lowJSValue(m_node->child1()); LValue result = vmCall(Double, m_out.operation(operationArithAbs), m_callFrame, argument); setDouble(result); break; } } } void compileArithSin() { if (m_node->child1().useKind() == DoubleRepUse) { setDouble(m_out.doubleSin(lowDouble(m_node->child1()))); return; } LValue argument = lowJSValue(m_node->child1()); LValue result = vmCall(Double, m_out.operation(operationArithSin), m_callFrame, argument); setDouble(result); } void compileArithCos() { if (m_node->child1().useKind() == DoubleRepUse) { setDouble(m_out.doubleCos(lowDouble(m_node->child1()))); return; } LValue argument = lowJSValue(m_node->child1()); LValue result = vmCall(Double, m_out.operation(operationArithCos), m_callFrame, argument); setDouble(result); } void compileArithTan() { if (m_node->child1().useKind() == DoubleRepUse) { setDouble(m_out.doubleTan(lowDouble(m_node->child1()))); return; } LValue argument = lowJSValue(m_node->child1()); LValue result = vmCall(Double, m_out.operation(operationArithTan), m_callFrame, argument); setDouble(result); } void compileArithPow() { if (m_node->child2().useKind() == Int32Use) setDouble(m_out.doublePowi(lowDouble(m_node->child1()), lowInt32(m_node->child2()))); else { LValue base = lowDouble(m_node->child1()); LValue exponent = lowDouble(m_node->child2()); LBasicBlock integerExponentIsSmallBlock = m_out.newBlock(); LBasicBlock integerExponentPowBlock = m_out.newBlock(); LBasicBlock doubleExponentPowBlockEntry = m_out.newBlock(); LBasicBlock nanExceptionBaseIsOne = m_out.newBlock(); LBasicBlock nanExceptionExponentIsInfinity = m_out.newBlock(); LBasicBlock testExponentIsOneHalf = m_out.newBlock(); LBasicBlock handleBaseZeroExponentIsOneHalf = m_out.newBlock(); LBasicBlock handleInfinityForExponentIsOneHalf = m_out.newBlock(); LBasicBlock exponentIsOneHalfNormal = m_out.newBlock(); LBasicBlock exponentIsOneHalfInfinity = m_out.newBlock(); LBasicBlock testExponentIsNegativeOneHalf = m_out.newBlock(); LBasicBlock testBaseZeroExponentIsNegativeOneHalf = m_out.newBlock(); LBasicBlock handleBaseZeroExponentIsNegativeOneHalf = m_out.newBlock(); LBasicBlock handleInfinityForExponentIsNegativeOneHalf = m_out.newBlock(); LBasicBlock exponentIsNegativeOneHalfNormal = m_out.newBlock(); LBasicBlock exponentIsNegativeOneHalfInfinity = m_out.newBlock(); LBasicBlock powBlock = m_out.newBlock(); LBasicBlock nanExceptionResultIsNaN = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue integerExponent = m_out.doubleToInt(exponent); LValue integerExponentConvertedToDouble = m_out.intToDouble(integerExponent); LValue exponentIsInteger = m_out.doubleEqual(exponent, integerExponentConvertedToDouble); m_out.branch(exponentIsInteger, unsure(integerExponentIsSmallBlock), unsure(doubleExponentPowBlockEntry)); LBasicBlock lastNext = m_out.appendTo(integerExponentIsSmallBlock, integerExponentPowBlock); LValue integerExponentBelowMax = m_out.belowOrEqual(integerExponent, m_out.constInt32(maxExponentForIntegerMathPow)); m_out.branch(integerExponentBelowMax, usually(integerExponentPowBlock), rarely(doubleExponentPowBlockEntry)); m_out.appendTo(integerExponentPowBlock, doubleExponentPowBlockEntry); ValueFromBlock powDoubleIntResult = m_out.anchor(m_out.doublePowi(base, integerExponent)); m_out.jump(continuation); // If y is NaN, the result is NaN. m_out.appendTo(doubleExponentPowBlockEntry, nanExceptionBaseIsOne); LValue exponentIsNaN; if (provenType(m_node->child2()) & SpecDoubleNaN) exponentIsNaN = m_out.doubleNotEqualOrUnordered(exponent, exponent); else exponentIsNaN = m_out.booleanFalse; m_out.branch(exponentIsNaN, rarely(nanExceptionResultIsNaN), usually(nanExceptionBaseIsOne)); // If abs(x) is 1 and y is +infinity, the result is NaN. // If abs(x) is 1 and y is -infinity, the result is NaN. // Test if base == 1. m_out.appendTo(nanExceptionBaseIsOne, nanExceptionExponentIsInfinity); LValue absoluteBase = m_out.doubleAbs(base); LValue absoluteBaseIsOne = m_out.doubleEqual(absoluteBase, m_out.constDouble(1)); m_out.branch(absoluteBaseIsOne, rarely(nanExceptionExponentIsInfinity), usually(testExponentIsOneHalf)); // Test if abs(y) == Infinity. m_out.appendTo(nanExceptionExponentIsInfinity, testExponentIsOneHalf); LValue absoluteExponent = m_out.doubleAbs(exponent); LValue absoluteExponentIsInfinity = m_out.doubleEqual(absoluteExponent, m_out.constDouble(std::numeric_limits::infinity())); m_out.branch(absoluteExponentIsInfinity, rarely(nanExceptionResultIsNaN), usually(testExponentIsOneHalf)); // If y == 0.5 or y == -0.5, handle it through SQRT. // We have be carefuly with -0 and -Infinity. // Test if y == 0.5 m_out.appendTo(testExponentIsOneHalf, handleBaseZeroExponentIsOneHalf); LValue exponentIsOneHalf = m_out.doubleEqual(exponent, m_out.constDouble(0.5)); m_out.branch(exponentIsOneHalf, rarely(handleBaseZeroExponentIsOneHalf), usually(testExponentIsNegativeOneHalf)); // Handle x == -0. m_out.appendTo(handleBaseZeroExponentIsOneHalf, handleInfinityForExponentIsOneHalf); LValue baseIsZeroExponentIsOneHalf = m_out.doubleEqual(base, m_out.doubleZero); ValueFromBlock zeroResultExponentIsOneHalf = m_out.anchor(m_out.doubleZero); m_out.branch(baseIsZeroExponentIsOneHalf, rarely(continuation), usually(handleInfinityForExponentIsOneHalf)); // Test if abs(x) == Infinity. m_out.appendTo(handleInfinityForExponentIsOneHalf, exponentIsOneHalfNormal); LValue absoluteBaseIsInfinityOneHalf = m_out.doubleEqual(absoluteBase, m_out.constDouble(std::numeric_limits::infinity())); m_out.branch(absoluteBaseIsInfinityOneHalf, rarely(exponentIsOneHalfInfinity), usually(exponentIsOneHalfNormal)); // The exponent is 0.5, the base is finite or NaN, we can use SQRT. m_out.appendTo(exponentIsOneHalfNormal, exponentIsOneHalfInfinity); ValueFromBlock sqrtResult = m_out.anchor(m_out.doubleSqrt(base)); m_out.jump(continuation); // The exponent is 0.5, the base is infinite, the result is always infinite. m_out.appendTo(exponentIsOneHalfInfinity, testExponentIsNegativeOneHalf); ValueFromBlock sqrtInfinityResult = m_out.anchor(m_out.constDouble(std::numeric_limits::infinity())); m_out.jump(continuation); // Test if y == -0.5 m_out.appendTo(testExponentIsNegativeOneHalf, testBaseZeroExponentIsNegativeOneHalf); LValue exponentIsNegativeOneHalf = m_out.doubleEqual(exponent, m_out.constDouble(-0.5)); m_out.branch(exponentIsNegativeOneHalf, rarely(testBaseZeroExponentIsNegativeOneHalf), usually(powBlock)); // Handle x == -0. m_out.appendTo(testBaseZeroExponentIsNegativeOneHalf, handleBaseZeroExponentIsNegativeOneHalf); LValue baseIsZeroExponentIsNegativeOneHalf = m_out.doubleEqual(base, m_out.doubleZero); m_out.branch(baseIsZeroExponentIsNegativeOneHalf, rarely(handleBaseZeroExponentIsNegativeOneHalf), usually(handleInfinityForExponentIsNegativeOneHalf)); m_out.appendTo(handleBaseZeroExponentIsNegativeOneHalf, handleInfinityForExponentIsNegativeOneHalf); ValueFromBlock oneOverSqrtZeroResult = m_out.anchor(m_out.constDouble(std::numeric_limits::infinity())); m_out.jump(continuation); // Test if abs(x) == Infinity. m_out.appendTo(handleInfinityForExponentIsNegativeOneHalf, exponentIsNegativeOneHalfNormal); LValue absoluteBaseIsInfinityNegativeOneHalf = m_out.doubleEqual(absoluteBase, m_out.constDouble(std::numeric_limits::infinity())); m_out.branch(absoluteBaseIsInfinityNegativeOneHalf, rarely(exponentIsNegativeOneHalfInfinity), usually(exponentIsNegativeOneHalfNormal)); // The exponent is -0.5, the base is finite or NaN, we can use 1/SQRT. m_out.appendTo(exponentIsNegativeOneHalfNormal, exponentIsNegativeOneHalfInfinity); LValue sqrtBase = m_out.doubleSqrt(base); ValueFromBlock oneOverSqrtResult = m_out.anchor(m_out.div(m_out.constDouble(1.), sqrtBase)); m_out.jump(continuation); // The exponent is -0.5, the base is infinite, the result is always zero. m_out.appendTo(exponentIsNegativeOneHalfInfinity, powBlock); ValueFromBlock oneOverSqrtInfinityResult = m_out.anchor(m_out.doubleZero); m_out.jump(continuation); m_out.appendTo(powBlock, nanExceptionResultIsNaN); ValueFromBlock powResult = m_out.anchor(m_out.doublePow(base, exponent)); m_out.jump(continuation); m_out.appendTo(nanExceptionResultIsNaN, continuation); ValueFromBlock pureNan = m_out.anchor(m_out.constDouble(PNaN)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setDouble(m_out.phi(Double, powDoubleIntResult, zeroResultExponentIsOneHalf, sqrtResult, sqrtInfinityResult, oneOverSqrtZeroResult, oneOverSqrtResult, oneOverSqrtInfinityResult, powResult, pureNan)); } } void compileArithRandom() { JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); // Inlined WeakRandom::advance(). // uint64_t x = m_low; void* lowAddress = reinterpret_cast(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::lowOffset(); LValue low = m_out.load64(m_out.absolute(lowAddress)); // uint64_t y = m_high; void* highAddress = reinterpret_cast(globalObject) + JSGlobalObject::weakRandomOffset() + WeakRandom::highOffset(); LValue high = m_out.load64(m_out.absolute(highAddress)); // m_low = y; m_out.store64(high, m_out.absolute(lowAddress)); // x ^= x << 23; LValue phase1 = m_out.bitXor(m_out.shl(low, m_out.constInt64(23)), low); // x ^= x >> 17; LValue phase2 = m_out.bitXor(m_out.lShr(phase1, m_out.constInt64(17)), phase1); // x ^= y ^ (y >> 26); LValue phase3 = m_out.bitXor(m_out.bitXor(high, m_out.lShr(high, m_out.constInt64(26))), phase2); // m_high = x; m_out.store64(phase3, m_out.absolute(highAddress)); // return x + y; LValue random64 = m_out.add(phase3, high); // Extract random 53bit. [0, 53] bit is safe integer number ranges in double representation. LValue random53 = m_out.bitAnd(random64, m_out.constInt64((1ULL << 53) - 1)); LValue double53Integer = m_out.intToDouble(random53); // Convert `(53bit double integer value) / (1 << 53)` to `(53bit double integer value) * (1.0 / (1 << 53))`. // In latter case, `1.0 / (1 << 53)` will become a double value represented as (mantissa = 0 & exp = 970, it means 1e-(2**54)). static const double scale = 1.0 / (1ULL << 53); // Multiplying 1e-(2**54) with the double integer does not change anything of the mantissa part of the double integer. // It just reduces the exp part of the given 53bit double integer. // (Except for 0.0. This is specially handled and in this case, exp just becomes 0.) // Now we get 53bit precision random double value in [0, 1). LValue result = m_out.doubleMul(double53Integer, m_out.constDouble(scale)); setDouble(result); } void compileArithRound() { if (m_node->child1().useKind() == DoubleRepUse) { LValue result = nullptr; if (producesInteger(m_node->arithRoundingMode()) && !shouldCheckNegativeZero(m_node->arithRoundingMode())) { LValue value = lowDouble(m_node->child1()); result = m_out.doubleFloor(m_out.doubleAdd(value, m_out.constDouble(0.5))); } else { LBasicBlock realPartIsMoreThanHalf = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue value = lowDouble(m_node->child1()); LValue integerValue = m_out.doubleCeil(value); ValueFromBlock integerValueResult = m_out.anchor(integerValue); LValue realPart = m_out.doubleSub(integerValue, value); m_out.branch(m_out.doubleGreaterThanOrUnordered(realPart, m_out.constDouble(0.5)), unsure(realPartIsMoreThanHalf), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(realPartIsMoreThanHalf, continuation); LValue integerValueRoundedDown = m_out.doubleSub(integerValue, m_out.constDouble(1)); ValueFromBlock integerValueRoundedDownResult = m_out.anchor(integerValueRoundedDown); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); result = m_out.phi(Double, integerValueResult, integerValueRoundedDownResult); } if (producesInteger(m_node->arithRoundingMode())) { LValue integerValue = convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode())); setInt32(integerValue); } else setDouble(result); return; } DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse); LValue argument = lowJSValue(m_node->child1()); setJSValue(vmCall(Int64, m_out.operation(operationArithRound), m_callFrame, argument)); } void compileArithFloor() { if (m_node->child1().useKind() == DoubleRepUse) { LValue value = lowDouble(m_node->child1()); LValue integerValue = m_out.doubleFloor(value); if (producesInteger(m_node->arithRoundingMode())) setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode()))); else setDouble(integerValue); return; } DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse); LValue argument = lowJSValue(m_node->child1()); setJSValue(vmCall(Int64, m_out.operation(operationArithFloor), m_callFrame, argument)); } void compileArithCeil() { if (m_node->child1().useKind() == DoubleRepUse) { LValue value = lowDouble(m_node->child1()); LValue integerValue = m_out.doubleCeil(value); if (producesInteger(m_node->arithRoundingMode())) setInt32(convertDoubleToInt32(integerValue, shouldCheckNegativeZero(m_node->arithRoundingMode()))); else setDouble(integerValue); return; } DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse); LValue argument = lowJSValue(m_node->child1()); setJSValue(vmCall(Int64, m_out.operation(operationArithCeil), m_callFrame, argument)); } void compileArithTrunc() { if (m_node->child1().useKind() == DoubleRepUse) { LValue value = lowDouble(m_node->child1()); LValue result = m_out.doubleTrunc(value); if (producesInteger(m_node->arithRoundingMode())) setInt32(convertDoubleToInt32(result, shouldCheckNegativeZero(m_node->arithRoundingMode()))); else setDouble(result); return; } DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse); LValue argument = lowJSValue(m_node->child1()); setJSValue(vmCall(Int64, m_out.operation(operationArithTrunc), m_callFrame, argument)); } void compileArithSqrt() { if (m_node->child1().useKind() == DoubleRepUse) { setDouble(m_out.doubleSqrt(lowDouble(m_node->child1()))); return; } LValue argument = lowJSValue(m_node->child1()); LValue result = vmCall(Double, m_out.operation(operationArithSqrt), m_callFrame, argument); setDouble(result); } void compileArithLog() { if (m_node->child1().useKind() == DoubleRepUse) { setDouble(m_out.doubleLog(lowDouble(m_node->child1()))); return; } LValue argument = lowJSValue(m_node->child1()); LValue result = vmCall(Double, m_out.operation(operationArithLog), m_callFrame, argument); setDouble(result); } void compileArithFRound() { if (m_node->child1().useKind() == DoubleRepUse) { setDouble(m_out.fround(lowDouble(m_node->child1()))); return; } LValue argument = lowJSValue(m_node->child1()); LValue result = vmCall(Double, m_out.operation(operationArithFRound), m_callFrame, argument); setDouble(result); } void compileArithNegate() { switch (m_node->child1().useKind()) { case Int32Use: { LValue value = lowInt32(m_node->child1()); LValue result; if (!shouldCheckOverflow(m_node->arithMode())) result = m_out.neg(value); else if (!shouldCheckNegativeZero(m_node->arithMode())) { CheckValue* check = m_out.speculateSub(m_out.int32Zero, value); blessSpeculation(check, Overflow, noValue(), nullptr, m_origin); result = check; } else { speculate(Overflow, noValue(), 0, m_out.testIsZero32(value, m_out.constInt32(0x7fffffff))); result = m_out.neg(value); } setInt32(result); break; } case Int52RepUse: { if (!abstractValue(m_node->child1()).couldBeType(SpecInt52Only)) { Int52Kind kind; LValue value = lowWhicheverInt52(m_node->child1(), kind); LValue result = m_out.neg(value); if (shouldCheckNegativeZero(m_node->arithMode())) speculate(NegativeZero, noValue(), 0, m_out.isZero64(result)); setInt52(result, kind); break; } LValue value = lowInt52(m_node->child1()); CheckValue* result = m_out.speculateSub(m_out.int64Zero, value); blessSpeculation(result, Int52Overflow, noValue(), nullptr, m_origin); if (shouldCheckNegativeZero(m_node->arithMode())) speculate(NegativeZero, noValue(), 0, m_out.isZero64(result)); setInt52(result); break; } case DoubleRepUse: { setDouble(m_out.doubleNeg(lowDouble(m_node->child1()))); break; } default: DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == UntypedUse); ArithProfile* arithProfile = m_ftlState.graph.baselineCodeBlockFor(m_node->origin.semantic)->arithProfileForBytecodeOffset(m_node->origin.semantic.bytecodeIndex); JITNegIC* negIC = codeBlock()->addJITNegIC(arithProfile); auto repatchingFunction = operationArithNegateOptimize; auto nonRepatchingFunction = operationArithNegate; compileMathIC(negIC, repatchingFunction, nonRepatchingFunction); break; } } void compileBitAnd() { if (m_node->isBinaryUseKind(UntypedUse)) { emitBinaryBitOpSnippet(operationValueBitAnd); return; } setInt32(m_out.bitAnd(lowInt32(m_node->child1()), lowInt32(m_node->child2()))); } void compileBitOr() { if (m_node->isBinaryUseKind(UntypedUse)) { emitBinaryBitOpSnippet(operationValueBitOr); return; } setInt32(m_out.bitOr(lowInt32(m_node->child1()), lowInt32(m_node->child2()))); } void compileBitXor() { if (m_node->isBinaryUseKind(UntypedUse)) { emitBinaryBitOpSnippet(operationValueBitXor); return; } setInt32(m_out.bitXor(lowInt32(m_node->child1()), lowInt32(m_node->child2()))); } void compileBitRShift() { if (m_node->isBinaryUseKind(UntypedUse)) { emitRightShiftSnippet(JITRightShiftGenerator::SignedShift); return; } setInt32(m_out.aShr( lowInt32(m_node->child1()), m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31)))); } void compileBitLShift() { if (m_node->isBinaryUseKind(UntypedUse)) { emitBinaryBitOpSnippet(operationValueBitLShift); return; } setInt32(m_out.shl( lowInt32(m_node->child1()), m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31)))); } void compileBitURShift() { if (m_node->isBinaryUseKind(UntypedUse)) { emitRightShiftSnippet(JITRightShiftGenerator::UnsignedShift); return; } setInt32(m_out.lShr( lowInt32(m_node->child1()), m_out.bitAnd(lowInt32(m_node->child2()), m_out.constInt32(31)))); } void compileUInt32ToNumber() { LValue value = lowInt32(m_node->child1()); if (doesOverflow(m_node->arithMode())) { setStrictInt52(m_out.zeroExtPtr(value)); return; } speculate(Overflow, noValue(), 0, m_out.lessThan(value, m_out.int32Zero)); setInt32(value); } void compileCheckStructure() { ExitKind exitKind; if (m_node->child1()->hasConstant()) exitKind = BadConstantCache; else exitKind = BadCache; switch (m_node->child1().useKind()) { case CellUse: case KnownCellUse: { LValue cell = lowCell(m_node->child1()); checkStructure( m_out.load32(cell, m_heaps.JSCell_structureID), jsValueValue(cell), exitKind, m_node->structureSet(), [&] (RegisteredStructure structure) { return weakStructureID(structure); }); return; } case CellOrOtherUse: { LValue value = lowJSValue(m_node->child1(), ManualOperandSpeculation); LBasicBlock cellCase = m_out.newBlock(); LBasicBlock notCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase); checkStructure( m_out.load32(value, m_heaps.JSCell_structureID), jsValueValue(value), exitKind, m_node->structureSet(), [&] (RegisteredStructure structure) { return weakStructureID(structure); }); m_out.jump(continuation); m_out.appendTo(notCellCase, continuation); FTL_TYPE_CHECK(jsValueValue(value), m_node->child1(), SpecCell | SpecOther, isNotOther(value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); return; } } void compileCheckCell() { LValue cell = lowCell(m_node->child1()); speculate( BadCell, jsValueValue(cell), m_node->child1().node(), m_out.notEqual(cell, weakPointer(m_node->cellOperand()->cell()))); } void compileCheckBadCell() { terminate(BadCell); } void compileCheckNotEmpty() { speculate(TDZFailure, noValue(), nullptr, m_out.isZero64(lowJSValue(m_node->child1()))); } void compileCheckStringIdent() { UniquedStringImpl* uid = m_node->uidOperand(); LValue stringImpl = lowStringIdent(m_node->child1()); speculate(BadIdent, noValue(), nullptr, m_out.notEqual(stringImpl, m_out.constIntPtr(uid))); } void compileGetExecutable() { LValue cell = lowCell(m_node->child1()); speculateFunction(m_node->child1(), cell); setJSValue(m_out.loadPtr(cell, m_heaps.JSFunction_executable)); } void compileArrayifyToStructure() { LValue cell = lowCell(m_node->child1()); LValue property = !!m_node->child2() ? lowInt32(m_node->child2()) : 0; LBasicBlock unexpectedStructure = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID); m_out.branch( m_out.notEqual(structureID, weakStructureID(m_node->structure())), rarely(unexpectedStructure), usually(continuation)); LBasicBlock lastNext = m_out.appendTo(unexpectedStructure, continuation); if (property) { switch (m_node->arrayMode().type()) { case Array::Int32: case Array::Double: case Array::Contiguous: speculate( Uncountable, noValue(), 0, m_out.aboveOrEqual(property, m_out.constInt32(MIN_SPARSE_ARRAY_INDEX))); break; default: break; } } switch (m_node->arrayMode().type()) { case Array::Int32: vmCall(Void, m_out.operation(operationEnsureInt32), m_callFrame, cell); break; case Array::Double: vmCall(Void, m_out.operation(operationEnsureDouble), m_callFrame, cell); break; case Array::Contiguous: vmCall(Void, m_out.operation(operationEnsureContiguous), m_callFrame, cell); break; case Array::ArrayStorage: case Array::SlowPutArrayStorage: vmCall(Void, m_out.operation(operationEnsureArrayStorage), m_callFrame, cell); break; default: DFG_CRASH(m_graph, m_node, "Bad array type"); break; } structureID = m_out.load32(cell, m_heaps.JSCell_structureID); speculate( BadIndexingType, jsValueValue(cell), 0, m_out.notEqual(structureID, weakStructureID(m_node->structure()))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void compilePutStructure() { m_ftlState.jitCode->common.notifyCompilingStructureTransition(m_graph.m_plan, codeBlock(), m_node); RegisteredStructure oldStructure = m_node->transition()->previous; RegisteredStructure newStructure = m_node->transition()->next; ASSERT_UNUSED(oldStructure, oldStructure->indexingType() == newStructure->indexingType()); ASSERT(oldStructure->typeInfo().inlineTypeFlags() == newStructure->typeInfo().inlineTypeFlags()); ASSERT(oldStructure->typeInfo().type() == newStructure->typeInfo().type()); LValue cell = lowCell(m_node->child1()); m_out.store32( weakStructureID(newStructure), cell, m_heaps.JSCell_structureID); } void compileGetById(AccessType type) { ASSERT(type == AccessType::Get || type == AccessType::TryGet); switch (m_node->child1().useKind()) { case CellUse: { setJSValue(getById(lowCell(m_node->child1()), type)); return; } case UntypedUse: { // This is pretty weird, since we duplicate the slow path both here and in the // code generated by the IC. We should investigate making this less bad. // https://bugs.webkit.org/show_bug.cgi?id=127830 LValue value = lowJSValue(m_node->child1()); LBasicBlock cellCase = m_out.newBlock(); LBasicBlock notCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(notCellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase); ValueFromBlock cellResult = m_out.anchor(getById(value, type)); m_out.jump(continuation); J_JITOperation_EJI getByIdFunction; if (type == AccessType::Get) getByIdFunction = operationGetByIdGeneric; else getByIdFunction = operationTryGetByIdGeneric; m_out.appendTo(notCellCase, continuation); ValueFromBlock notCellResult = m_out.anchor(vmCall( Int64, m_out.operation(getByIdFunction), m_callFrame, value, m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()]))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, cellResult, notCellResult)); return; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); return; } } void compileGetByIdWithThis() { LValue base = lowJSValue(m_node->child1()); LValue thisValue = lowJSValue(m_node->child2()); LValue result = vmCall(Int64, m_out.operation(operationGetByIdWithThis), m_callFrame, base, thisValue, m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])); setJSValue(result); } void compileGetByValWithThis() { LValue base = lowJSValue(m_node->child1()); LValue thisValue = lowJSValue(m_node->child2()); LValue subscript = lowJSValue(m_node->child3()); LValue result = vmCall(Int64, m_out.operation(operationGetByValWithThis), m_callFrame, base, thisValue, subscript); setJSValue(result); } void compilePutByIdWithThis() { LValue base = lowJSValue(m_node->child1()); LValue thisValue = lowJSValue(m_node->child2()); LValue value = lowJSValue(m_node->child3()); vmCall(Void, m_out.operation(m_graph.isStrictModeFor(m_node->origin.semantic) ? operationPutByIdWithThisStrict : operationPutByIdWithThis), m_callFrame, base, thisValue, value, m_out.constIntPtr(m_graph.identifiers()[m_node->identifierNumber()])); } void compilePutByValWithThis() { LValue base = lowJSValue(m_graph.varArgChild(m_node, 0)); LValue thisValue = lowJSValue(m_graph.varArgChild(m_node, 1)); LValue property = lowJSValue(m_graph.varArgChild(m_node, 2)); LValue value = lowJSValue(m_graph.varArgChild(m_node, 3)); vmCall(Void, m_out.operation(m_graph.isStrictModeFor(m_node->origin.semantic) ? operationPutByValWithThisStrict : operationPutByValWithThis), m_callFrame, base, thisValue, property, value); } void compileDefineDataProperty() { LValue base = lowCell(m_graph.varArgChild(m_node, 0)); LValue value = lowJSValue(m_graph.varArgChild(m_node, 2)); LValue attributes = lowInt32(m_graph.varArgChild(m_node, 3)); Edge& propertyEdge = m_graph.varArgChild(m_node, 1); switch (propertyEdge.useKind()) { case StringUse: { LValue property = lowString(propertyEdge); vmCall(Void, m_out.operation(operationDefineDataPropertyString), m_callFrame, base, property, value, attributes); break; } case StringIdentUse: { LValue property = lowStringIdent(propertyEdge); vmCall(Void, m_out.operation(operationDefineDataPropertyStringIdent), m_callFrame, base, property, value, attributes); break; } case SymbolUse: { LValue property = lowSymbol(propertyEdge); vmCall(Void, m_out.operation(operationDefineDataPropertySymbol), m_callFrame, base, property, value, attributes); break; } case UntypedUse: { LValue property = lowJSValue(propertyEdge); vmCall(Void, m_out.operation(operationDefineDataProperty), m_callFrame, base, property, value, attributes); break; } default: RELEASE_ASSERT_NOT_REACHED(); } } void compileDefineAccessorProperty() { LValue base = lowCell(m_graph.varArgChild(m_node, 0)); LValue getter = lowCell(m_graph.varArgChild(m_node, 2)); LValue setter = lowCell(m_graph.varArgChild(m_node, 3)); LValue attributes = lowInt32(m_graph.varArgChild(m_node, 4)); Edge& propertyEdge = m_graph.varArgChild(m_node, 1); switch (propertyEdge.useKind()) { case StringUse: { LValue property = lowString(propertyEdge); vmCall(Void, m_out.operation(operationDefineAccessorPropertyString), m_callFrame, base, property, getter, setter, attributes); break; } case StringIdentUse: { LValue property = lowStringIdent(propertyEdge); vmCall(Void, m_out.operation(operationDefineAccessorPropertyStringIdent), m_callFrame, base, property, getter, setter, attributes); break; } case SymbolUse: { LValue property = lowSymbol(propertyEdge); vmCall(Void, m_out.operation(operationDefineAccessorPropertySymbol), m_callFrame, base, property, getter, setter, attributes); break; } case UntypedUse: { LValue property = lowJSValue(propertyEdge); vmCall(Void, m_out.operation(operationDefineAccessorProperty), m_callFrame, base, property, getter, setter, attributes); break; } default: RELEASE_ASSERT_NOT_REACHED(); } } void compilePutById() { DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == CellUse); Node* node = m_node; LValue base = lowCell(node->child1()); LValue value = lowJSValue(node->child2()); auto uid = m_graph.identifiers()[node->identifierNumber()]; B3::PatchpointValue* patchpoint = m_out.patchpoint(Void); patchpoint->appendSomeRegister(base); patchpoint->appendSomeRegister(value); patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister)); patchpoint->clobber(RegisterSet::macroScratchRegisters()); // FIXME: If this is a PutByIdFlush, we might want to late-clobber volatile registers. // https://bugs.webkit.org/show_bug.cgi?id=152848 RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); State* state = &m_ftlState; ECMAMode ecmaMode = m_graph.executableFor(node->origin.semantic)->ecmaMode(); patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); // JS setter call ICs generated by the PutById IC will need this. exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex); auto generator = Box::create( jit.codeBlock(), node->origin.semantic, callSiteIndex, params.unavailableRegisters(), JSValueRegs(params[0].gpr()), JSValueRegs(params[1].gpr()), GPRInfo::patchpointScratchRegister, ecmaMode, node->op() == PutByIdDirect ? Direct : NotDirect); generator->generateFastPath(jit); CCallHelpers::Label done = jit.label(); params.addLatePath( [=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); generator->slowPathJump().link(&jit); CCallHelpers::Label slowPathBegin = jit.label(); CCallHelpers::Call slowPathCall = callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), generator->slowPathFunction(), InvalidGPRReg, CCallHelpers::TrustedImmPtr(generator->stubInfo()), params[1].gpr(), params[0].gpr(), CCallHelpers::TrustedImmPtr(uid)).call(); jit.jump().linkTo(done, &jit); generator->reportSlowPathCall(slowPathBegin, slowPathCall); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { generator->finalize(linkBuffer); }); }); }); } void compileGetButterfly() { setStorage(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSObject_butterfly)); } void compileConstantStoragePointer() { setStorage(m_out.constIntPtr(m_node->storagePointer())); } void compileGetIndexedPropertyStorage() { LValue cell = lowCell(m_node->child1()); if (m_node->arrayMode().type() == Array::String) { LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue fastResultValue = m_out.loadPtr(cell, m_heaps.JSString_value); ValueFromBlock fastResult = m_out.anchor(fastResultValue); m_out.branch( m_out.notNull(fastResultValue), usually(continuation), rarely(slowPath)); LBasicBlock lastNext = m_out.appendTo(slowPath, continuation); ValueFromBlock slowResult = m_out.anchor( vmCall(pointerType(), m_out.operation(operationResolveRope), m_callFrame, cell)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setStorage(m_out.loadPtr(m_out.phi(pointerType(), fastResult, slowResult), m_heaps.StringImpl_data)); return; } DFG_ASSERT(m_graph, m_node, isTypedView(m_node->arrayMode().typedArrayType())); setStorage(m_out.loadPtr(cell, m_heaps.JSArrayBufferView_vector)); } void compileCheckArray() { Edge edge = m_node->child1(); LValue cell = lowCell(edge); if (m_node->arrayMode().alreadyChecked(m_graph, m_node, abstractValue(edge))) return; speculate( BadIndexingType, jsValueValue(cell), 0, m_out.logicalNot(isArrayType(cell, m_node->arrayMode()))); } void compileGetTypedArrayByteOffset() { LValue basePtr = lowCell(m_node->child1()); LBasicBlock simpleCase = m_out.newBlock(); LBasicBlock wastefulCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue mode = m_out.load32(basePtr, m_heaps.JSArrayBufferView_mode); m_out.branch( m_out.notEqual(mode, m_out.constInt32(WastefulTypedArray)), unsure(simpleCase), unsure(wastefulCase)); LBasicBlock lastNext = m_out.appendTo(simpleCase, wastefulCase); ValueFromBlock simpleOut = m_out.anchor(m_out.constIntPtr(0)); m_out.jump(continuation); m_out.appendTo(wastefulCase, continuation); LValue vectorPtr = m_out.loadPtr(basePtr, m_heaps.JSArrayBufferView_vector); LValue butterflyPtr = m_out.loadPtr(basePtr, m_heaps.JSObject_butterfly); LValue arrayBufferPtr = m_out.loadPtr(butterflyPtr, m_heaps.Butterfly_arrayBuffer); LValue dataPtr = m_out.loadPtr(arrayBufferPtr, m_heaps.ArrayBuffer_data); ValueFromBlock wastefulOut = m_out.anchor(m_out.sub(vectorPtr, dataPtr)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setInt32(m_out.castToInt32(m_out.phi(pointerType(), simpleOut, wastefulOut))); } void compileGetArrayLength() { switch (m_node->arrayMode().type()) { case Array::Undecided: case Array::Int32: case Array::Double: case Array::Contiguous: { setInt32(m_out.load32NonNegative(lowStorage(m_node->child2()), m_heaps.Butterfly_publicLength)); return; } case Array::String: { LValue string = lowCell(m_node->child1()); setInt32(m_out.load32NonNegative(string, m_heaps.JSString_length)); return; } case Array::DirectArguments: { LValue arguments = lowCell(m_node->child1()); speculate( ExoticObjectMode, noValue(), nullptr, m_out.notNull(m_out.loadPtr(arguments, m_heaps.DirectArguments_mappedArguments))); setInt32(m_out.load32NonNegative(arguments, m_heaps.DirectArguments_length)); return; } case Array::ScopedArguments: { LValue arguments = lowCell(m_node->child1()); speculate( ExoticObjectMode, noValue(), nullptr, m_out.notZero32(m_out.load8ZeroExt32(arguments, m_heaps.ScopedArguments_overrodeThings))); setInt32(m_out.load32NonNegative(arguments, m_heaps.ScopedArguments_totalLength)); return; } default: if (m_node->arrayMode().isSomeTypedArrayView()) { setInt32( m_out.load32NonNegative(lowCell(m_node->child1()), m_heaps.JSArrayBufferView_length)); return; } DFG_CRASH(m_graph, m_node, "Bad array type"); return; } } void compileCheckInBounds() { speculate( OutOfBounds, noValue(), 0, m_out.aboveOrEqual(lowInt32(m_node->child1()), lowInt32(m_node->child2()))); } void compileGetByVal() { switch (m_node->arrayMode().type()) { case Array::Int32: case Array::Contiguous: { LValue index = lowInt32(m_node->child2()); LValue storage = lowStorage(m_node->child3()); IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ? m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties; if (m_node->arrayMode().isInBounds()) { LValue result = m_out.load64(baseIndex(heap, storage, index, m_node->child2())); LValue isHole = m_out.isZero64(result); if (m_node->arrayMode().isSaneChain()) { DFG_ASSERT( m_graph, m_node, m_node->arrayMode().type() == Array::Contiguous); result = m_out.select( isHole, m_out.constInt64(JSValue::encode(jsUndefined())), result); } else speculate(LoadFromHole, noValue(), 0, isHole); setJSValue(result); return; } LValue base = lowCell(m_node->child1()); LBasicBlock fastCase = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.aboveOrEqual( index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)), rarely(slowCase), usually(fastCase)); LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase); LValue fastResultValue = m_out.load64(baseIndex(heap, storage, index, m_node->child2())); ValueFromBlock fastResult = m_out.anchor(fastResultValue); m_out.branch( m_out.isZero64(fastResultValue), rarely(slowCase), usually(continuation)); m_out.appendTo(slowCase, continuation); ValueFromBlock slowResult = m_out.anchor( vmCall(Int64, m_out.operation(operationGetByValArrayInt), m_callFrame, base, index)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, fastResult, slowResult)); return; } case Array::Double: { LValue index = lowInt32(m_node->child2()); LValue storage = lowStorage(m_node->child3()); IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties; if (m_node->arrayMode().isInBounds()) { LValue result = m_out.loadDouble( baseIndex(heap, storage, index, m_node->child2())); if (!m_node->arrayMode().isSaneChain()) { speculate( LoadFromHole, noValue(), 0, m_out.doubleNotEqualOrUnordered(result, result)); } setDouble(result); break; } LValue base = lowCell(m_node->child1()); LBasicBlock inBounds = m_out.newBlock(); LBasicBlock boxPath = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.aboveOrEqual( index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)), rarely(slowCase), usually(inBounds)); LBasicBlock lastNext = m_out.appendTo(inBounds, boxPath); LValue doubleValue = m_out.loadDouble( baseIndex(heap, storage, index, m_node->child2())); m_out.branch( m_out.doubleNotEqualOrUnordered(doubleValue, doubleValue), rarely(slowCase), usually(boxPath)); m_out.appendTo(boxPath, slowCase); ValueFromBlock fastResult = m_out.anchor(boxDouble(doubleValue)); m_out.jump(continuation); m_out.appendTo(slowCase, continuation); ValueFromBlock slowResult = m_out.anchor( vmCall(Int64, m_out.operation(operationGetByValArrayInt), m_callFrame, base, index)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, fastResult, slowResult)); return; } case Array::Undecided: { LValue index = lowInt32(m_node->child2()); speculate(OutOfBounds, noValue(), m_node, m_out.lessThan(index, m_out.int32Zero)); setJSValue(m_out.constInt64(ValueUndefined)); return; } case Array::DirectArguments: { LValue base = lowCell(m_node->child1()); LValue index = lowInt32(m_node->child2()); speculate( ExoticObjectMode, noValue(), nullptr, m_out.notNull(m_out.loadPtr(base, m_heaps.DirectArguments_mappedArguments))); speculate( ExoticObjectMode, noValue(), nullptr, m_out.aboveOrEqual( index, m_out.load32NonNegative(base, m_heaps.DirectArguments_length))); TypedPointer address = m_out.baseIndex( m_heaps.DirectArguments_storage, base, m_out.zeroExtPtr(index)); setJSValue(m_out.load64(address)); return; } case Array::ScopedArguments: { LValue base = lowCell(m_node->child1()); LValue index = lowInt32(m_node->child2()); speculate( ExoticObjectMode, noValue(), nullptr, m_out.aboveOrEqual( index, m_out.load32NonNegative(base, m_heaps.ScopedArguments_totalLength))); LValue table = m_out.loadPtr(base, m_heaps.ScopedArguments_table); LValue namedLength = m_out.load32(table, m_heaps.ScopedArgumentsTable_length); LBasicBlock namedCase = m_out.newBlock(); LBasicBlock overflowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.aboveOrEqual(index, namedLength), unsure(overflowCase), unsure(namedCase)); LBasicBlock lastNext = m_out.appendTo(namedCase, overflowCase); LValue scope = m_out.loadPtr(base, m_heaps.ScopedArguments_scope); LValue arguments = m_out.loadPtr(table, m_heaps.ScopedArgumentsTable_arguments); TypedPointer address = m_out.baseIndex( m_heaps.scopedArgumentsTableArguments, arguments, m_out.zeroExtPtr(index)); LValue scopeOffset = m_out.load32(address); speculate( ExoticObjectMode, noValue(), nullptr, m_out.equal(scopeOffset, m_out.constInt32(ScopeOffset::invalidOffset))); address = m_out.baseIndex( m_heaps.JSEnvironmentRecord_variables, scope, m_out.zeroExtPtr(scopeOffset)); ValueFromBlock namedResult = m_out.anchor(m_out.load64(address)); m_out.jump(continuation); m_out.appendTo(overflowCase, continuation); address = m_out.baseIndex( m_heaps.ScopedArguments_overflowStorage, base, m_out.zeroExtPtr(m_out.sub(index, namedLength))); LValue overflowValue = m_out.load64(address); speculate(ExoticObjectMode, noValue(), nullptr, m_out.isZero64(overflowValue)); ValueFromBlock overflowResult = m_out.anchor(overflowValue); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, namedResult, overflowResult)); return; } case Array::Generic: { setJSValue(vmCall( Int64, m_out.operation(operationGetByVal), m_callFrame, lowJSValue(m_node->child1()), lowJSValue(m_node->child2()))); return; } case Array::String: { compileStringCharAt(); return; } default: { LValue index = lowInt32(m_node->child2()); LValue storage = lowStorage(m_node->child3()); TypedArrayType type = m_node->arrayMode().typedArrayType(); if (isTypedView(type)) { TypedPointer pointer = TypedPointer( m_heaps.typedArrayProperties, m_out.add( storage, m_out.shl( m_out.zeroExtPtr(index), m_out.constIntPtr(logElementSize(type))))); if (isInt(type)) { LValue result; switch (elementSize(type)) { case 1: result = isSigned(type) ? m_out.load8SignExt32(pointer) : m_out.load8ZeroExt32(pointer); break; case 2: result = isSigned(type) ? m_out.load16SignExt32(pointer) : m_out.load16ZeroExt32(pointer); break; case 4: result = m_out.load32(pointer); break; default: DFG_CRASH(m_graph, m_node, "Bad element size"); } if (elementSize(type) < 4 || isSigned(type)) { setInt32(result); return; } if (m_node->shouldSpeculateInt32()) { speculate( Overflow, noValue(), 0, m_out.lessThan(result, m_out.int32Zero)); setInt32(result); return; } if (m_node->shouldSpeculateAnyInt()) { setStrictInt52(m_out.zeroExt(result, Int64)); return; } setDouble(m_out.unsignedToDouble(result)); return; } ASSERT(isFloat(type)); LValue result; switch (type) { case TypeFloat32: result = m_out.floatToDouble(m_out.loadFloat(pointer)); break; case TypeFloat64: result = m_out.loadDouble(pointer); break; default: DFG_CRASH(m_graph, m_node, "Bad typed array type"); } setDouble(result); return; } DFG_CRASH(m_graph, m_node, "Bad array type"); return; } } } void compileGetMyArgumentByVal() { InlineCallFrame* inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame; LValue index = lowInt32(m_node->child2()); if (m_node->numberOfArgumentsToSkip()) index = m_out.add(index, m_out.constInt32(m_node->numberOfArgumentsToSkip())); LValue limit; if (inlineCallFrame && !inlineCallFrame->isVarargs()) limit = m_out.constInt32(inlineCallFrame->arguments.size() - 1); else { VirtualRegister argumentCountRegister = AssemblyHelpers::argumentCount(inlineCallFrame); limit = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One); } LValue isOutOfBounds = m_out.aboveOrEqual(index, limit); LBasicBlock continuation = nullptr; LBasicBlock lastNext = nullptr; ValueFromBlock slowResult; if (m_node->op() == GetMyArgumentByValOutOfBounds) { LBasicBlock normalCase = m_out.newBlock(); continuation = m_out.newBlock(); slowResult = m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))); m_out.branch(isOutOfBounds, unsure(continuation), unsure(normalCase)); lastNext = m_out.appendTo(normalCase, continuation); } else speculate(ExoticObjectMode, noValue(), 0, isOutOfBounds); TypedPointer base; if (inlineCallFrame) { if (inlineCallFrame->arguments.size() > 1) base = addressFor(inlineCallFrame->arguments[1].virtualRegister()); } else base = addressFor(virtualRegisterForArgument(1)); LValue result; if (base) { LValue pointer = m_out.baseIndex( base.value(), m_out.zeroExt(index, pointerType()), ScaleEight); result = m_out.load64(TypedPointer(m_heaps.variables.atAnyIndex(), pointer)); } else result = m_out.constInt64(JSValue::encode(jsUndefined())); if (m_node->op() == GetMyArgumentByValOutOfBounds) { ValueFromBlock normalResult = m_out.anchor(result); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); result = m_out.phi(Int64, slowResult, normalResult); } setJSValue(result); } void compilePutByVal() { Edge child1 = m_graph.varArgChild(m_node, 0); Edge child2 = m_graph.varArgChild(m_node, 1); Edge child3 = m_graph.varArgChild(m_node, 2); Edge child4 = m_graph.varArgChild(m_node, 3); Edge child5 = m_graph.varArgChild(m_node, 4); switch (m_node->arrayMode().type()) { case Array::Generic: { V_JITOperation_EJJJ operation; if (m_node->op() == PutByValDirect) { if (m_graph.isStrictModeFor(m_node->origin.semantic)) operation = operationPutByValDirectStrict; else operation = operationPutByValDirectNonStrict; } else { if (m_graph.isStrictModeFor(m_node->origin.semantic)) operation = operationPutByValStrict; else operation = operationPutByValNonStrict; } vmCall( Void, m_out.operation(operation), m_callFrame, lowJSValue(child1), lowJSValue(child2), lowJSValue(child3)); return; } default: break; } LValue base = lowCell(child1); LValue index = lowInt32(child2); LValue storage = lowStorage(child4); switch (m_node->arrayMode().type()) { case Array::Int32: case Array::Double: case Array::Contiguous: { LBasicBlock continuation = m_out.newBlock(); LBasicBlock outerLastNext = m_out.appendTo(m_out.m_block, continuation); switch (m_node->arrayMode().type()) { case Array::Int32: case Array::Contiguous: { LValue value = lowJSValue(child3, ManualOperandSpeculation); if (m_node->arrayMode().type() == Array::Int32) FTL_TYPE_CHECK(jsValueValue(value), child3, SpecInt32Only, isNotInt32(value)); TypedPointer elementPointer = m_out.baseIndex( m_node->arrayMode().type() == Array::Int32 ? m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties, storage, m_out.zeroExtPtr(index), provenValue(child2)); if (m_node->op() == PutByValAlias) { m_out.store64(value, elementPointer); break; } contiguousPutByValOutOfBounds( codeBlock()->isStrictMode() ? operationPutByValBeyondArrayBoundsStrict : operationPutByValBeyondArrayBoundsNonStrict, base, storage, index, value, continuation); m_out.store64(value, elementPointer); break; } case Array::Double: { LValue value = lowDouble(child3); FTL_TYPE_CHECK( doubleValue(value), child3, SpecDoubleReal, m_out.doubleNotEqualOrUnordered(value, value)); TypedPointer elementPointer = m_out.baseIndex( m_heaps.indexedDoubleProperties, storage, m_out.zeroExtPtr(index), provenValue(child2)); if (m_node->op() == PutByValAlias) { m_out.storeDouble(value, elementPointer); break; } contiguousPutByValOutOfBounds( codeBlock()->isStrictMode() ? operationPutDoubleByValBeyondArrayBoundsStrict : operationPutDoubleByValBeyondArrayBoundsNonStrict, base, storage, index, value, continuation); m_out.storeDouble(value, elementPointer); break; } default: DFG_CRASH(m_graph, m_node, "Bad array type"); } m_out.jump(continuation); m_out.appendTo(continuation, outerLastNext); return; } default: TypedArrayType type = m_node->arrayMode().typedArrayType(); if (isTypedView(type)) { TypedPointer pointer = TypedPointer( m_heaps.typedArrayProperties, m_out.add( storage, m_out.shl( m_out.zeroExt(index, pointerType()), m_out.constIntPtr(logElementSize(type))))); Output::StoreType storeType; LValue valueToStore; if (isInt(type)) { LValue intValue; switch (child3.useKind()) { case Int52RepUse: case Int32Use: { if (child3.useKind() == Int32Use) intValue = lowInt32(child3); else intValue = m_out.castToInt32(lowStrictInt52(child3)); if (isClamped(type)) { ASSERT(elementSize(type) == 1); LBasicBlock atLeastZero = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); Vector intValues; intValues.append(m_out.anchor(m_out.int32Zero)); m_out.branch( m_out.lessThan(intValue, m_out.int32Zero), unsure(continuation), unsure(atLeastZero)); LBasicBlock lastNext = m_out.appendTo(atLeastZero, continuation); intValues.append(m_out.anchor(m_out.select( m_out.greaterThan(intValue, m_out.constInt32(255)), m_out.constInt32(255), intValue))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); intValue = m_out.phi(Int32, intValues); } break; } case DoubleRepUse: { LValue doubleValue = lowDouble(child3); if (isClamped(type)) { ASSERT(elementSize(type) == 1); LBasicBlock atLeastZero = m_out.newBlock(); LBasicBlock withinRange = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); Vector intValues; intValues.append(m_out.anchor(m_out.int32Zero)); m_out.branch( m_out.doubleLessThanOrUnordered(doubleValue, m_out.doubleZero), unsure(continuation), unsure(atLeastZero)); LBasicBlock lastNext = m_out.appendTo(atLeastZero, withinRange); intValues.append(m_out.anchor(m_out.constInt32(255))); m_out.branch( m_out.doubleGreaterThan(doubleValue, m_out.constDouble(255)), unsure(continuation), unsure(withinRange)); m_out.appendTo(withinRange, continuation); intValues.append(m_out.anchor(m_out.doubleToInt(doubleValue))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); intValue = m_out.phi(Int32, intValues); } else intValue = doubleToInt32(doubleValue); break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); } valueToStore = intValue; switch (elementSize(type)) { case 1: storeType = Output::Store32As8; break; case 2: storeType = Output::Store32As16; break; case 4: storeType = Output::Store32; break; default: DFG_CRASH(m_graph, m_node, "Bad element size"); } } else /* !isInt(type) */ { LValue value = lowDouble(child3); switch (type) { case TypeFloat32: valueToStore = m_out.doubleToFloat(value); storeType = Output::StoreFloat; break; case TypeFloat64: valueToStore = value; storeType = Output::StoreDouble; break; default: DFG_CRASH(m_graph, m_node, "Bad typed array type"); } } if (m_node->arrayMode().isInBounds() || m_node->op() == PutByValAlias) m_out.store(valueToStore, pointer, storeType); else { LBasicBlock isInBounds = m_out.newBlock(); LBasicBlock isOutOfBounds = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.aboveOrEqual(index, lowInt32(child5)), unsure(isOutOfBounds), unsure(isInBounds)); LBasicBlock lastNext = m_out.appendTo(isInBounds, isOutOfBounds); m_out.store(valueToStore, pointer, storeType); m_out.jump(continuation); m_out.appendTo(isOutOfBounds, continuation); speculateTypedArrayIsNotNeutered(base); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } return; } DFG_CRASH(m_graph, m_node, "Bad array type"); break; } } void compilePutAccessorById() { LValue base = lowCell(m_node->child1()); LValue accessor = lowCell(m_node->child2()); auto uid = m_graph.identifiers()[m_node->identifierNumber()]; vmCall( Void, m_out.operation(m_node->op() == PutGetterById ? operationPutGetterById : operationPutSetterById), m_callFrame, base, m_out.constIntPtr(uid), m_out.constInt32(m_node->accessorAttributes()), accessor); } void compilePutGetterSetterById() { LValue base = lowCell(m_node->child1()); LValue getter = lowJSValue(m_node->child2()); LValue setter = lowJSValue(m_node->child3()); auto uid = m_graph.identifiers()[m_node->identifierNumber()]; vmCall( Void, m_out.operation(operationPutGetterSetter), m_callFrame, base, m_out.constIntPtr(uid), m_out.constInt32(m_node->accessorAttributes()), getter, setter); } void compilePutAccessorByVal() { LValue base = lowCell(m_node->child1()); LValue subscript = lowJSValue(m_node->child2()); LValue accessor = lowCell(m_node->child3()); vmCall( Void, m_out.operation(m_node->op() == PutGetterByVal ? operationPutGetterByVal : operationPutSetterByVal), m_callFrame, base, subscript, m_out.constInt32(m_node->accessorAttributes()), accessor); } void compileArrayPush() { LValue base = lowCell(m_node->child1()); LValue storage = lowStorage(m_node->child3()); switch (m_node->arrayMode().type()) { case Array::Int32: case Array::Contiguous: case Array::Double: { LValue value; Output::StoreType storeType; if (m_node->arrayMode().type() != Array::Double) { value = lowJSValue(m_node->child2(), ManualOperandSpeculation); if (m_node->arrayMode().type() == Array::Int32) { FTL_TYPE_CHECK( jsValueValue(value), m_node->child2(), SpecInt32Only, isNotInt32(value)); } storeType = Output::Store64; } else { value = lowDouble(m_node->child2()); FTL_TYPE_CHECK( doubleValue(value), m_node->child2(), SpecDoubleReal, m_out.doubleNotEqualOrUnordered(value, value)); storeType = Output::StoreDouble; } IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type()); LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength); LBasicBlock fastPath = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.aboveOrEqual( prevLength, m_out.load32(storage, m_heaps.Butterfly_vectorLength)), unsure(slowPath), unsure(fastPath)); LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath); m_out.store( value, m_out.baseIndex(heap, storage, m_out.zeroExtPtr(prevLength)), storeType); LValue newLength = m_out.add(prevLength, m_out.int32One); m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength); ValueFromBlock fastResult = m_out.anchor(boxInt32(newLength)); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); LValue operation; if (m_node->arrayMode().type() != Array::Double) operation = m_out.operation(operationArrayPush); else operation = m_out.operation(operationArrayPushDouble); ValueFromBlock slowResult = m_out.anchor( vmCall(Int64, operation, m_callFrame, value, base)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, fastResult, slowResult)); return; } default: DFG_CRASH(m_graph, m_node, "Bad array type"); return; } } void compileArraySlice() { JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); LValue sourceStorage = lowStorage(m_node->numChildren() == 3 ? m_graph.varArgChild(m_node, 2) : m_graph.varArgChild(m_node, 3)); LValue inputLength = m_out.load32(sourceStorage, m_heaps.Butterfly_publicLength); LValue endBoundary; if (m_node->numChildren() == 3) endBoundary = m_out.load32(sourceStorage, m_heaps.Butterfly_publicLength); else { endBoundary = lowInt32(m_graph.varArgChild(m_node, 2)); endBoundary = m_out.select(m_out.greaterThanOrEqual(endBoundary, m_out.constInt32(0)), m_out.select(m_out.above(endBoundary, inputLength), inputLength, endBoundary), m_out.select(m_out.lessThan(m_out.add(inputLength, endBoundary), m_out.constInt32(0)), m_out.constInt32(0), m_out.add(inputLength, endBoundary))); } LValue startIndex = lowInt32(m_graph.varArgChild(m_node, 1)); startIndex = m_out.select(m_out.greaterThanOrEqual(startIndex, m_out.constInt32(0)), m_out.select(m_out.above(startIndex, inputLength), inputLength, startIndex), m_out.select(m_out.lessThan(m_out.add(inputLength, startIndex), m_out.constInt32(0)), m_out.constInt32(0), m_out.add(inputLength, startIndex))); LValue resultLength = m_out.select(m_out.below(startIndex, endBoundary), m_out.sub(endBoundary, startIndex), m_out.constInt32(0)); ArrayValues arrayResult; { LValue indexingType = m_out.load8ZeroExt32(lowCell(m_graph.varArgChild(m_node, 0)), m_heaps.JSCell_indexingTypeAndMisc); indexingType = m_out.bitAnd(indexingType, m_out.constInt32(AllArrayTypesAndHistory)); // When we emit an ArraySlice, we dominate the use of the array by a CheckStructure // to ensure the incoming array is one to be one of the original array structures // with one of the following indexing shapes: Int32, Contiguous, Double. LValue structure = m_out.select( m_out.equal(indexingType, m_out.constInt32(ArrayWithInt32)), weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithInt32))), m_out.select(m_out.equal(indexingType, m_out.constInt32(ArrayWithContiguous)), weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithContiguous))), weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithDouble))))); arrayResult = allocateJSArray(resultLength, structure, indexingType, false, false); } LBasicBlock loop = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); resultLength = m_out.zeroExtPtr(resultLength); ValueFromBlock startLoadIndex = m_out.anchor(m_out.zeroExtPtr(startIndex)); ValueFromBlock startStoreIndex = m_out.anchor(m_out.constIntPtr(0)); m_out.branch( m_out.below(m_out.constIntPtr(0), resultLength), unsure(loop), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(loop, continuation); LValue storeIndex = m_out.phi(pointerType(), startStoreIndex); LValue loadIndex = m_out.phi(pointerType(), startLoadIndex); LValue value = m_out.load64(m_out.baseIndex(m_heaps.root, sourceStorage, loadIndex, ScaleEight)); m_out.store64(value, m_out.baseIndex(m_heaps.root, arrayResult.butterfly, storeIndex, ScaleEight)); LValue nextStoreIndex = m_out.add(storeIndex, m_out.constIntPtr(1)); m_out.addIncomingToPhi(storeIndex, m_out.anchor(nextStoreIndex)); m_out.addIncomingToPhi(loadIndex, m_out.anchor(m_out.add(loadIndex, m_out.constIntPtr(1)))); m_out.branch( m_out.below(nextStoreIndex, resultLength), unsure(loop), unsure(continuation)); m_out.appendTo(continuation, lastNext); mutatorFence(); setJSValue(arrayResult.array); } void compileArrayPop() { LValue base = lowCell(m_node->child1()); LValue storage = lowStorage(m_node->child2()); switch (m_node->arrayMode().type()) { case Array::Int32: case Array::Double: case Array::Contiguous: { IndexedAbstractHeap& heap = m_heaps.forArrayType(m_node->arrayMode().type()); LBasicBlock fastCase = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue prevLength = m_out.load32(storage, m_heaps.Butterfly_publicLength); Vector results; results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined())))); m_out.branch( m_out.isZero32(prevLength), rarely(continuation), usually(fastCase)); LBasicBlock lastNext = m_out.appendTo(fastCase, slowCase); LValue newLength = m_out.sub(prevLength, m_out.int32One); m_out.store32(newLength, storage, m_heaps.Butterfly_publicLength); TypedPointer pointer = m_out.baseIndex(heap, storage, m_out.zeroExtPtr(newLength)); if (m_node->arrayMode().type() != Array::Double) { LValue result = m_out.load64(pointer); m_out.store64(m_out.int64Zero, pointer); results.append(m_out.anchor(result)); m_out.branch( m_out.notZero64(result), usually(continuation), rarely(slowCase)); } else { LValue result = m_out.loadDouble(pointer); m_out.store64(m_out.constInt64(bitwise_cast(PNaN)), pointer); results.append(m_out.anchor(boxDouble(result))); m_out.branch( m_out.doubleEqual(result, result), usually(continuation), rarely(slowCase)); } m_out.appendTo(slowCase, continuation); results.append(m_out.anchor(vmCall( Int64, m_out.operation(operationArrayPopAndRecoverLength), m_callFrame, base))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, results)); return; } default: DFG_CRASH(m_graph, m_node, "Bad array type"); return; } } void compileCreateActivation() { LValue scope = lowCell(m_node->child1()); SymbolTable* table = m_node->castOperand(); RegisteredStructure structure = m_graph.registerStructure(m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure()); JSValue initializationValue = m_node->initializationValueForActivation(); ASSERT(initializationValue.isUndefined() || initializationValue == jsTDZValue()); if (table->singletonScope()->isStillValid()) { LValue callResult = vmCall( Int64, m_out.operation(operationCreateActivationDirect), m_callFrame, weakStructure(structure), scope, weakPointer(table), m_out.constInt64(JSValue::encode(initializationValue))); setJSValue(callResult); return; } LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); LValue fastObject = allocateObject( JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath); // We don't need memory barriers since we just fast-created the activation, so the // activation must be young. m_out.storePtr(scope, fastObject, m_heaps.JSScope_next); m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable); for (unsigned i = 0; i < table->scopeSize(); ++i) { m_out.store64( m_out.constInt64(JSValue::encode(initializationValue)), fastObject, m_heaps.JSEnvironmentRecord_variables[i]); } mutatorFence(); ValueFromBlock fastResult = m_out.anchor(fastObject); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); LValue callResult = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationCreateActivationDirect, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR(), CCallHelpers::TrustedImmPtr(table), CCallHelpers::TrustedImm64(JSValue::encode(initializationValue))); }, scope); ValueFromBlock slowResult = m_out.anchor(callResult); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(pointerType(), fastResult, slowResult)); } void compileNewFunction() { ASSERT(m_node->op() == NewFunction || m_node->op() == NewGeneratorFunction || m_node->op() == NewAsyncFunction); bool isGeneratorFunction = m_node->op() == NewGeneratorFunction; bool isAsyncFunction = m_node->op() == NewAsyncFunction; LValue scope = lowCell(m_node->child1()); FunctionExecutable* executable = m_node->castOperand(); if (executable->singletonFunction()->isStillValid()) { LValue callResult = isGeneratorFunction ? vmCall(Int64, m_out.operation(operationNewGeneratorFunction), m_callFrame, scope, weakPointer(executable)) : isAsyncFunction ? vmCall(Int64, m_out.operation(operationNewAsyncFunction), m_callFrame, scope, weakPointer(executable)) : vmCall(Int64, m_out.operation(operationNewFunction), m_callFrame, scope, weakPointer(executable)); setJSValue(callResult); return; } RegisteredStructure structure = m_graph.registerStructure( isGeneratorFunction ? m_graph.globalObjectFor(m_node->origin.semantic)->generatorFunctionStructure() : isAsyncFunction ? m_graph.globalObjectFor(m_node->origin.semantic)->asyncFunctionStructure() : m_graph.globalObjectFor(m_node->origin.semantic)->functionStructure()); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); LValue fastObject = isGeneratorFunction ? allocateObject(structure, m_out.intPtrZero, slowPath) : allocateObject(structure, m_out.intPtrZero, slowPath); // We don't need memory barriers since we just fast-created the function, so it // must be young. m_out.storePtr(scope, fastObject, m_heaps.JSFunction_scope); m_out.storePtr(weakPointer(executable), fastObject, m_heaps.JSFunction_executable); m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.JSFunction_rareData); mutatorFence(); ValueFromBlock fastResult = m_out.anchor(fastObject); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); Vector slowPathArguments; slowPathArguments.append(scope); LValue callResult = lazySlowPath( [=] (const Vector& locations) -> RefPtr { if (isGeneratorFunction) { return createLazyCallGenerator( operationNewGeneratorFunctionWithInvalidatedReallocationWatchpoint, locations[0].directGPR(), locations[1].directGPR(), CCallHelpers::TrustedImmPtr(executable)); } if (isAsyncFunction) { return createLazyCallGenerator( operationNewAsyncFunctionWithInvalidatedReallocationWatchpoint, locations[0].directGPR(), locations[1].directGPR(), CCallHelpers::TrustedImmPtr(executable)); } return createLazyCallGenerator( operationNewFunctionWithInvalidatedReallocationWatchpoint, locations[0].directGPR(), locations[1].directGPR(), CCallHelpers::TrustedImmPtr(executable)); }, slowPathArguments); ValueFromBlock slowResult = m_out.anchor(callResult); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(pointerType(), fastResult, slowResult)); } void compileCreateDirectArguments() { // FIXME: A more effective way of dealing with the argument count and callee is to have // them be explicit arguments to this node. // https://bugs.webkit.org/show_bug.cgi?id=142207 RegisteredStructure structure = m_graph.registerStructure(m_graph.globalObjectFor(m_node->origin.semantic)->directArgumentsStructure()); unsigned minCapacity = m_graph.baselineCodeBlockFor(m_node->origin.semantic)->numParameters() - 1; LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); ArgumentsLength length = getArgumentsLength(); LValue fastObject; if (length.isKnown) { fastObject = allocateObject( DirectArguments::allocationSize(std::max(length.known, minCapacity)), structure, m_out.intPtrZero, slowPath); } else { LValue size = m_out.add( m_out.shl(length.value, m_out.constInt32(3)), m_out.constInt32(DirectArguments::storageOffset())); size = m_out.select( m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)), size, m_out.constInt32(DirectArguments::allocationSize(minCapacity))); fastObject = allocateVariableSizedObject( m_out.zeroExtPtr(size), structure, m_out.intPtrZero, slowPath); } m_out.store32(length.value, fastObject, m_heaps.DirectArguments_length); m_out.store32(m_out.constInt32(minCapacity), fastObject, m_heaps.DirectArguments_minCapacity); m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.DirectArguments_mappedArguments); m_out.storePtr(m_out.intPtrZero, fastObject, m_heaps.DirectArguments_modifiedArgumentsDescriptor); ValueFromBlock fastResult = m_out.anchor(fastObject); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); LValue callResult = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationCreateDirectArguments, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR(), CCallHelpers::TrustedImm32(minCapacity)); }, length.value); ValueFromBlock slowResult = m_out.anchor(callResult); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); LValue result = m_out.phi(pointerType(), fastResult, slowResult); m_out.storePtr(getCurrentCallee(), result, m_heaps.DirectArguments_callee); if (length.isKnown) { VirtualRegister start = AssemblyHelpers::argumentsStart(m_node->origin.semantic); for (unsigned i = 0; i < std::max(length.known, minCapacity); ++i) { m_out.store64( m_out.load64(addressFor(start + i)), result, m_heaps.DirectArguments_storage[i]); } } else { LValue stackBase = getArgumentsStart(); LBasicBlock loop = m_out.newBlock(); LBasicBlock end = m_out.newBlock(); ValueFromBlock originalLength; if (minCapacity) { LValue capacity = m_out.select( m_out.aboveOrEqual(length.value, m_out.constInt32(minCapacity)), length.value, m_out.constInt32(minCapacity)); LValue originalLengthValue = m_out.zeroExtPtr(capacity); originalLength = m_out.anchor(originalLengthValue); m_out.jump(loop); } else { LValue originalLengthValue = m_out.zeroExtPtr(length.value); originalLength = m_out.anchor(originalLengthValue); m_out.branch(m_out.isNull(originalLengthValue), unsure(end), unsure(loop)); } lastNext = m_out.appendTo(loop, end); LValue previousIndex = m_out.phi(pointerType(), originalLength); LValue index = m_out.sub(previousIndex, m_out.intPtrOne); m_out.store64( m_out.load64(m_out.baseIndex(m_heaps.variables, stackBase, index)), m_out.baseIndex(m_heaps.DirectArguments_storage, result, index)); ValueFromBlock nextIndex = m_out.anchor(index); m_out.addIncomingToPhi(previousIndex, nextIndex); m_out.branch(m_out.isNull(index), unsure(end), unsure(loop)); m_out.appendTo(end, lastNext); } mutatorFence(); setJSValue(result); } void compileCreateScopedArguments() { LValue scope = lowCell(m_node->child1()); LValue result = vmCall( Int64, m_out.operation(operationCreateScopedArguments), m_callFrame, weakPointer( m_graph.globalObjectFor(m_node->origin.semantic)->scopedArgumentsStructure()), getArgumentsStart(), getArgumentsLength().value, getCurrentCallee(), scope); setJSValue(result); } void compileCreateClonedArguments() { LValue result = vmCall( Int64, m_out.operation(operationCreateClonedArguments), m_callFrame, weakPointer( m_graph.globalObjectFor(m_node->origin.semantic)->clonedArgumentsStructure()), getArgumentsStart(), getArgumentsLength().value, getCurrentCallee()); setJSValue(result); } void compileCreateRest() { if (m_graph.isWatchingHavingABadTimeWatchpoint(m_node)) { LBasicBlock continuation = m_out.newBlock(); LValue arrayLength = lowInt32(m_node->child1()); LBasicBlock loopStart = m_out.newBlock(); JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); RegisteredStructure structure = m_graph.registerStructure(globalObject->restParameterStructure()); ArrayValues arrayValues = allocateUninitializedContiguousJSArray(arrayLength, structure); LValue array = arrayValues.array; LValue butterfly = arrayValues.butterfly; ValueFromBlock startLength = m_out.anchor(arrayLength); LValue argumentRegion = m_out.add(getArgumentsStart(), m_out.constInt64(sizeof(Register) * m_node->numberOfArgumentsToSkip())); m_out.branch(m_out.equal(arrayLength, m_out.constInt32(0)), unsure(continuation), unsure(loopStart)); LBasicBlock lastNext = m_out.appendTo(loopStart, continuation); LValue phiOffset = m_out.phi(Int32, startLength); LValue currentOffset = m_out.sub(phiOffset, m_out.int32One); m_out.addIncomingToPhi(phiOffset, m_out.anchor(currentOffset)); LValue loadedValue = m_out.load64(m_out.baseIndex(m_heaps.variables, argumentRegion, m_out.zeroExtPtr(currentOffset))); IndexedAbstractHeap& heap = m_heaps.indexedContiguousProperties; m_out.store64(loadedValue, m_out.baseIndex(heap, butterfly, m_out.zeroExtPtr(currentOffset))); m_out.branch(m_out.equal(currentOffset, m_out.constInt32(0)), unsure(continuation), unsure(loopStart)); m_out.appendTo(continuation, lastNext); mutatorFence(); setJSValue(array); return; } LValue arrayLength = lowInt32(m_node->child1()); LValue argumentStart = getArgumentsStart(); LValue numberOfArgumentsToSkip = m_out.constInt32(m_node->numberOfArgumentsToSkip()); setJSValue(vmCall( Int64, m_out.operation(operationCreateRest), m_callFrame, argumentStart, numberOfArgumentsToSkip, arrayLength)); } void compileGetRestLength() { LBasicBlock nonZeroLength = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock zeroLengthResult = m_out.anchor(m_out.constInt32(0)); LValue numberOfArgumentsToSkip = m_out.constInt32(m_node->numberOfArgumentsToSkip()); LValue argumentsLength = getArgumentsLength().value; m_out.branch(m_out.above(argumentsLength, numberOfArgumentsToSkip), unsure(nonZeroLength), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(nonZeroLength, continuation); ValueFromBlock nonZeroLengthResult = m_out.anchor(m_out.sub(argumentsLength, numberOfArgumentsToSkip)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setInt32(m_out.phi(Int32, zeroLengthResult, nonZeroLengthResult)); } void compileNewObject() { setJSValue(allocateObject(m_node->structure())); mutatorFence(); } void compileNewArray() { // First speculate appropriately on all of the children. Do this unconditionally up here // because some of the slow paths may otherwise forget to do it. It's sort of arguable // that doing the speculations up here might be unprofitable for RA - so we can consider // sinking this to below the allocation fast path if we find that this has a lot of // register pressure. for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) speculate(m_graph.varArgChild(m_node, operandIndex)); JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); RegisteredStructure structure = m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation( m_node->indexingType())); if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) { unsigned numElements = m_node->numChildren(); ArrayValues arrayValues = allocateUninitializedContiguousJSArray(m_out.constInt32(numElements), structure); for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) { Edge edge = m_graph.varArgChild(m_node, operandIndex); switch (m_node->indexingType()) { case ALL_BLANK_INDEXING_TYPES: case ALL_UNDECIDED_INDEXING_TYPES: DFG_CRASH(m_graph, m_node, "Bad indexing type"); break; case ALL_DOUBLE_INDEXING_TYPES: m_out.storeDouble( lowDouble(edge), arrayValues.butterfly, m_heaps.indexedDoubleProperties[operandIndex]); break; case ALL_INT32_INDEXING_TYPES: case ALL_CONTIGUOUS_INDEXING_TYPES: m_out.store64( lowJSValue(edge, ManualOperandSpeculation), arrayValues.butterfly, m_heaps.forIndexingType(m_node->indexingType())->at(operandIndex)); break; default: DFG_CRASH(m_graph, m_node, "Corrupt indexing type"); break; } } setJSValue(arrayValues.array); mutatorFence(); return; } if (!m_node->numChildren()) { setJSValue(vmCall( Int64, m_out.operation(operationNewEmptyArray), m_callFrame, weakStructure(structure))); return; } size_t scratchSize = sizeof(EncodedJSValue) * m_node->numChildren(); ASSERT(scratchSize); ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize); EncodedJSValue* buffer = static_cast(scratchBuffer->dataBuffer()); for (unsigned operandIndex = 0; operandIndex < m_node->numChildren(); ++operandIndex) { Edge edge = m_graph.varArgChild(m_node, operandIndex); m_out.store64( lowJSValue(edge, ManualOperandSpeculation), m_out.absolute(buffer + operandIndex)); } m_out.storePtr( m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->activeLengthPtr())); LValue result = vmCall( Int64, m_out.operation(operationNewArray), m_callFrame, weakStructure(structure), m_out.constIntPtr(buffer), m_out.constIntPtr(m_node->numChildren())); m_out.storePtr(m_out.intPtrZero, m_out.absolute(scratchBuffer->activeLengthPtr())); setJSValue(result); } void compileNewArrayWithSpread() { if (m_graph.isWatchingHavingABadTimeWatchpoint(m_node)) { unsigned startLength = 0; BitVector* bitVector = m_node->bitVector(); HashMap::Hash, WTF::NullableHashTraits> cachedSpreadLengths; for (unsigned i = 0; i < m_node->numChildren(); ++i) { if (!bitVector->get(i)) ++startLength; } LValue length = m_out.constInt32(startLength); for (unsigned i = 0; i < m_node->numChildren(); ++i) { if (bitVector->get(i)) { Edge use = m_graph.varArgChild(m_node, i); CheckValue* lengthCheck = nullptr; if (use->op() == PhantomSpread) { RELEASE_ASSERT(use->child1()->op() == PhantomCreateRest); InlineCallFrame* inlineCallFrame = use->child1()->origin.semantic.inlineCallFrame; unsigned numberOfArgumentsToSkip = use->child1()->numberOfArgumentsToSkip(); LValue spreadLength = cachedSpreadLengths.ensure(inlineCallFrame, [&] () { return getSpreadLengthFromInlineCallFrame(inlineCallFrame, numberOfArgumentsToSkip); }).iterator->value; lengthCheck = m_out.speculateAdd(length, spreadLength); } else { LValue fixedArray = lowCell(use); lengthCheck = m_out.speculateAdd(length, m_out.load32(fixedArray, m_heaps.JSFixedArray_size)); } blessSpeculation(lengthCheck, Overflow, noValue(), nullptr, m_origin); length = lengthCheck; } } RegisteredStructure structure = m_graph.registerStructure(m_graph.globalObjectFor(m_node->origin.semantic)->originalArrayStructureForIndexingType(ArrayWithContiguous)); ArrayValues arrayValues = allocateUninitializedContiguousJSArray(length, structure); LValue result = arrayValues.array; LValue storage = arrayValues.butterfly; LValue index = m_out.constIntPtr(0); for (unsigned i = 0; i < m_node->numChildren(); ++i) { Edge use = m_graph.varArgChild(m_node, i); if (bitVector->get(i)) { if (use->op() == PhantomSpread) { RELEASE_ASSERT(use->child1()->op() == PhantomCreateRest); InlineCallFrame* inlineCallFrame = use->child1()->origin.semantic.inlineCallFrame; unsigned numberOfArgumentsToSkip = use->child1()->numberOfArgumentsToSkip(); LValue length = m_out.zeroExtPtr(cachedSpreadLengths.get(inlineCallFrame)); LValue sourceStart = getArgumentsStart(inlineCallFrame, numberOfArgumentsToSkip); LBasicBlock loopStart = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock loadIndexStart = m_out.anchor(m_out.constIntPtr(0)); ValueFromBlock arrayIndexStart = m_out.anchor(index); ValueFromBlock arrayIndexStartForFinish = m_out.anchor(index); m_out.branch( m_out.isZero64(length), unsure(continuation), unsure(loopStart)); LBasicBlock lastNext = m_out.appendTo(loopStart, continuation); LValue arrayIndex = m_out.phi(pointerType(), arrayIndexStart); LValue loadIndex = m_out.phi(pointerType(), loadIndexStart); LValue item = m_out.load64(m_out.baseIndex(m_heaps.variables, sourceStart, loadIndex)); m_out.store64(item, m_out.baseIndex(m_heaps.indexedContiguousProperties, storage, arrayIndex)); LValue nextArrayIndex = m_out.add(arrayIndex, m_out.constIntPtr(1)); LValue nextLoadIndex = m_out.add(loadIndex, m_out.constIntPtr(1)); ValueFromBlock arrayIndexLoopForFinish = m_out.anchor(nextArrayIndex); m_out.addIncomingToPhi(loadIndex, m_out.anchor(nextLoadIndex)); m_out.addIncomingToPhi(arrayIndex, m_out.anchor(nextArrayIndex)); m_out.branch( m_out.below(nextLoadIndex, length), unsure(loopStart), unsure(continuation)); m_out.appendTo(continuation, lastNext); index = m_out.phi(pointerType(), arrayIndexStartForFinish, arrayIndexLoopForFinish); } else { LBasicBlock loopStart = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue fixedArray = lowCell(use); ValueFromBlock fixedIndexStart = m_out.anchor(m_out.constIntPtr(0)); ValueFromBlock arrayIndexStart = m_out.anchor(index); ValueFromBlock arrayIndexStartForFinish = m_out.anchor(index); LValue fixedArraySize = m_out.zeroExtPtr(m_out.load32(fixedArray, m_heaps.JSFixedArray_size)); m_out.branch( m_out.isZero64(fixedArraySize), unsure(continuation), unsure(loopStart)); LBasicBlock lastNext = m_out.appendTo(loopStart, continuation); LValue arrayIndex = m_out.phi(pointerType(), arrayIndexStart); LValue fixedArrayIndex = m_out.phi(pointerType(), fixedIndexStart); LValue item = m_out.load64(m_out.baseIndex(m_heaps.JSFixedArray_buffer, fixedArray, fixedArrayIndex)); m_out.store64(item, m_out.baseIndex(m_heaps.indexedContiguousProperties, storage, arrayIndex)); LValue nextArrayIndex = m_out.add(arrayIndex, m_out.constIntPtr(1)); LValue nextFixedArrayIndex = m_out.add(fixedArrayIndex, m_out.constIntPtr(1)); ValueFromBlock arrayIndexLoopForFinish = m_out.anchor(nextArrayIndex); m_out.addIncomingToPhi(fixedArrayIndex, m_out.anchor(nextFixedArrayIndex)); m_out.addIncomingToPhi(arrayIndex, m_out.anchor(nextArrayIndex)); m_out.branch( m_out.below(nextFixedArrayIndex, fixedArraySize), unsure(loopStart), unsure(continuation)); m_out.appendTo(continuation, lastNext); index = m_out.phi(pointerType(), arrayIndexStartForFinish, arrayIndexLoopForFinish); } } else { IndexedAbstractHeap& heap = m_heaps.indexedContiguousProperties; LValue item = lowJSValue(use); m_out.store64(item, m_out.baseIndex(heap, storage, index)); index = m_out.add(index, m_out.constIntPtr(1)); } } mutatorFence(); setJSValue(result); return; } ASSERT(m_node->numChildren()); size_t scratchSize = sizeof(EncodedJSValue) * m_node->numChildren(); ScratchBuffer* scratchBuffer = vm().scratchBufferForSize(scratchSize); EncodedJSValue* buffer = static_cast(scratchBuffer->dataBuffer()); BitVector* bitVector = m_node->bitVector(); for (unsigned i = 0; i < m_node->numChildren(); ++i) { Edge use = m_graph.m_varArgChildren[m_node->firstChild() + i]; LValue value; if (bitVector->get(i)) value = lowCell(use); else value = lowJSValue(use); m_out.store64(value, m_out.absolute(&buffer[i])); } m_out.storePtr(m_out.constIntPtr(scratchSize), m_out.absolute(scratchBuffer->activeLengthPtr())); LValue result = vmCall(Int64, m_out.operation(operationNewArrayWithSpreadSlow), m_callFrame, m_out.constIntPtr(buffer), m_out.constInt32(m_node->numChildren())); m_out.storePtr(m_out.constIntPtr(0), m_out.absolute(scratchBuffer->activeLengthPtr())); setJSValue(result); } void compileSpread() { // It would be trivial to support this, but for now, we never create // IR that would necessitate this. The reason is that Spread is only // consumed by NewArrayWithSpread and never anything else. Also, any // Spread(PhantomCreateRest) will turn into PhantomSpread(PhantomCreateRest). RELEASE_ASSERT(m_node->child1()->op() != PhantomCreateRest); LValue argument = lowCell(m_node->child1()); LValue result; if (m_node->child1().useKind() == ArrayUse) { speculateArray(m_node->child1()); LBasicBlock preLoop = m_out.newBlock(); LBasicBlock loopSelection = m_out.newBlock(); LBasicBlock contiguousLoopStart = m_out.newBlock(); LBasicBlock doubleLoopStart = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue indexingShape = m_out.load8ZeroExt32(argument, m_heaps.JSCell_indexingTypeAndMisc); indexingShape = m_out.bitAnd(indexingShape, m_out.constInt32(IndexingShapeMask)); LValue isOKIndexingType = m_out.belowOrEqual( m_out.sub(indexingShape, m_out.constInt32(Int32Shape)), m_out.constInt32(ContiguousShape - Int32Shape)); m_out.branch(isOKIndexingType, unsure(preLoop), unsure(slowPath)); LBasicBlock lastNext = m_out.appendTo(preLoop, loopSelection); LValue butterfly = m_out.loadPtr(argument, m_heaps.JSObject_butterfly); LValue length = m_out.load32NonNegative(butterfly, m_heaps.Butterfly_publicLength); static_assert(sizeof(JSValue) == 8 && 1 << 3 == 8, "Assumed in the code below."); LValue size = m_out.add( m_out.shl(m_out.zeroExtPtr(length), m_out.constInt32(3)), m_out.constIntPtr(JSFixedArray::offsetOfData())); LValue fastAllocation = allocateVariableSizedCell(size, m_graph.m_vm.fixedArrayStructure.get(), slowPath); ValueFromBlock fastResult = m_out.anchor(fastAllocation); m_out.store32(length, fastAllocation, m_heaps.JSFixedArray_size); ValueFromBlock startIndexForContiguous = m_out.anchor(m_out.constIntPtr(0)); ValueFromBlock startIndexForDouble = m_out.anchor(m_out.constIntPtr(0)); m_out.branch(m_out.isZero32(length), unsure(continuation), unsure(loopSelection)); m_out.appendTo(loopSelection, contiguousLoopStart); m_out.branch(m_out.equal(indexingShape, m_out.constInt32(DoubleShape)), unsure(doubleLoopStart), unsure(contiguousLoopStart)); { m_out.appendTo(contiguousLoopStart, doubleLoopStart); LValue index = m_out.phi(pointerType(), startIndexForContiguous); TypedPointer loadSite = m_out.baseIndex(m_heaps.root, butterfly, index, ScaleEight); // We read TOP here since we can be reading either int32 or contiguous properties. LValue value = m_out.load64(loadSite); value = m_out.select(m_out.isZero64(value), m_out.constInt64(JSValue::encode(jsUndefined())), value); m_out.store64(value, m_out.baseIndex(m_heaps.JSFixedArray_buffer, fastAllocation, index)); LValue nextIndex = m_out.add(index, m_out.constIntPtr(1)); m_out.addIncomingToPhi(index, m_out.anchor(nextIndex)); m_out.branch(m_out.below(nextIndex, m_out.zeroExtPtr(length)), unsure(contiguousLoopStart), unsure(continuation)); } { m_out.appendTo(doubleLoopStart, slowPath); LValue index = m_out.phi(pointerType(), startIndexForDouble); LValue value = m_out.loadDouble(m_out.baseIndex(m_heaps.indexedDoubleProperties, butterfly, index)); LValue isNaN = m_out.doubleNotEqualOrUnordered(value, value); LValue holeResult = m_out.constInt64(JSValue::encode(jsUndefined())); LValue normalResult = boxDouble(value); value = m_out.select(isNaN, holeResult, normalResult); m_out.store64(value, m_out.baseIndex(m_heaps.JSFixedArray_buffer, fastAllocation, index)); LValue nextIndex = m_out.add(index, m_out.constIntPtr(1)); m_out.addIncomingToPhi(index, m_out.anchor(nextIndex)); m_out.branch(m_out.below(nextIndex, m_out.zeroExtPtr(length)), unsure(doubleLoopStart), unsure(continuation)); } m_out.appendTo(slowPath, continuation); ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, m_out.operation(operationSpreadFastArray), m_callFrame, argument)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); result = m_out.phi(Int64, fastResult, slowResult); mutatorFence(); } else result = vmCall(Int64, m_out.operation(operationSpreadGeneric), m_callFrame, argument); setJSValue(result); } void compileNewArrayBuffer() { JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); RegisteredStructure structure = m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation( m_node->indexingType())); if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) { unsigned numElements = m_node->numConstants(); ArrayValues arrayValues = allocateUninitializedContiguousJSArray(m_out.constInt32(numElements), structure); JSValue* data = codeBlock()->constantBuffer(m_node->startConstant()); for (unsigned index = 0; index < m_node->numConstants(); ++index) { int64_t value; if (hasDouble(m_node->indexingType())) value = bitwise_cast(data[index].asNumber()); else value = JSValue::encode(data[index]); m_out.store64( m_out.constInt64(value), arrayValues.butterfly, m_heaps.forIndexingType(m_node->indexingType())->at(index)); } mutatorFence(); setJSValue(arrayValues.array); return; } setJSValue(vmCall( Int64, m_out.operation(operationNewArrayBuffer), m_callFrame, weakStructure(structure), m_out.constIntPtr(m_node->startConstant()), m_out.constIntPtr(m_node->numConstants()))); } void compileNewArrayWithSize() { LValue publicLength = lowInt32(m_node->child1()); JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); RegisteredStructure structure = m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation( m_node->indexingType())); if (!globalObject->isHavingABadTime() && !hasAnyArrayStorage(m_node->indexingType())) { IndexingType indexingType = m_node->indexingType(); setJSValue( allocateJSArray( publicLength, weakPointer(globalObject->arrayStructureForIndexingTypeDuringAllocation(indexingType)), m_out.constInt32(indexingType)).array); mutatorFence(); return; } LValue structureValue = m_out.select( m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)), weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage))), weakStructure(structure)); setJSValue(vmCall(Int64, m_out.operation(operationNewArrayWithSize), m_callFrame, structureValue, publicLength, m_out.intPtrZero)); } void compileNewTypedArray() { TypedArrayType type = m_node->typedArrayType(); JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); switch (m_node->child1().useKind()) { case Int32Use: { RegisteredStructure structure = m_graph.registerStructure(globalObject->typedArrayStructureConcurrently(type)); LValue size = lowInt32(m_node->child1()); LBasicBlock smallEnoughCase = m_out.newBlock(); LBasicBlock nonZeroCase = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock noStorage = m_out.anchor(m_out.intPtrZero); m_out.branch( m_out.above(size, m_out.constInt32(JSArrayBufferView::fastSizeLimit)), rarely(slowCase), usually(smallEnoughCase)); LBasicBlock lastNext = m_out.appendTo(smallEnoughCase, nonZeroCase); m_out.branch(m_out.notZero32(size), usually(nonZeroCase), rarely(slowCase)); m_out.appendTo(nonZeroCase, slowCase); LValue byteSize = m_out.shl(m_out.zeroExtPtr(size), m_out.constInt32(logElementSize(type))); if (elementSize(type) < 8) { byteSize = m_out.bitAnd( m_out.add(byteSize, m_out.constIntPtr(7)), m_out.constIntPtr(~static_cast(7))); } LValue allocator = allocatorForSize(vm().auxiliarySpace, byteSize, slowCase); LValue storage = allocateHeapCell(allocator, slowCase); splatWords( storage, m_out.int32Zero, m_out.castToInt32(m_out.lShr(byteSize, m_out.constIntPtr(3))), m_out.int64Zero, m_heaps.typedArrayProperties); ValueFromBlock haveStorage = m_out.anchor(storage); LValue fastResultValue = allocateObject(structure, m_out.intPtrZero, slowCase); m_out.storePtr(storage, fastResultValue, m_heaps.JSArrayBufferView_vector); m_out.store32(size, fastResultValue, m_heaps.JSArrayBufferView_length); m_out.store32(m_out.constInt32(FastTypedArray), fastResultValue, m_heaps.JSArrayBufferView_mode); mutatorFence(); ValueFromBlock fastResult = m_out.anchor(fastResultValue); m_out.jump(continuation); m_out.appendTo(slowCase, continuation); LValue storageValue = m_out.phi(pointerType(), noStorage, haveStorage); LValue slowResultValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationNewTypedArrayWithSizeForType(type), locations[0].directGPR(), CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR(), locations[2].directGPR()); }, size, storageValue); ValueFromBlock slowResult = m_out.anchor(slowResultValue); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(pointerType(), fastResult, slowResult)); return; } case UntypedUse: { LValue argument = lowJSValue(m_node->child1()); LValue result = vmCall( pointerType(), m_out.operation(operationNewTypedArrayWithOneArgumentForType(type)), m_callFrame, weakPointer(globalObject->typedArrayStructureConcurrently(type)), argument); setJSValue(result); return; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); return; } } void compileAllocatePropertyStorage() { LValue object = lowCell(m_node->child1()); setStorage(allocatePropertyStorage(object, m_node->transition()->previous.get())); } void compileReallocatePropertyStorage() { Transition* transition = m_node->transition(); LValue object = lowCell(m_node->child1()); LValue oldStorage = lowStorage(m_node->child2()); setStorage( reallocatePropertyStorage( object, oldStorage, transition->previous.get(), transition->next.get())); } void compileNukeStructureAndSetButterfly() { nukeStructureAndSetButterfly(lowStorage(m_node->child2()), lowCell(m_node->child1())); } void compileToNumber() { LValue value = lowJSValue(m_node->child1()); if (!(abstractValue(m_node->child1()).m_type & SpecBytecodeNumber)) setJSValue(vmCall(Int64, m_out.operation(operationToNumber), m_callFrame, value)); else { LBasicBlock notNumber = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock fastResult = m_out.anchor(value); m_out.branch(isNumber(value, provenType(m_node->child1())), unsure(continuation), unsure(notNumber)); // notNumber case. LBasicBlock lastNext = m_out.appendTo(notNumber, continuation); // We have several attempts to remove ToNumber. But ToNumber still exists. // It means that converting non-numbers to numbers by this ToNumber is not rare. // Instead of the lazy slow path generator, we call the operation here. ValueFromBlock slowResult = m_out.anchor(vmCall(Int64, m_out.operation(operationToNumber), m_callFrame, value)); m_out.jump(continuation); // continuation case. m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, fastResult, slowResult)); } } void compileToStringOrCallStringConstructor() { switch (m_node->child1().useKind()) { case StringObjectUse: { LValue cell = lowCell(m_node->child1()); speculateStringObjectForCell(m_node->child1(), cell); m_interpreter.filter(m_node->child1(), SpecStringObject); setJSValue(m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue)); return; } case StringOrStringObjectUse: { LValue cell = lowCell(m_node->child1()); LValue structureID = m_out.load32(cell, m_heaps.JSCell_structureID); LBasicBlock notString = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock simpleResult = m_out.anchor(cell); m_out.branch( m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())), unsure(continuation), unsure(notString)); LBasicBlock lastNext = m_out.appendTo(notString, continuation); speculateStringObjectForStructureID(m_node->child1(), structureID); ValueFromBlock unboxedResult = m_out.anchor( m_out.loadPtr(cell, m_heaps.JSWrapperObject_internalValue)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, simpleResult, unboxedResult)); m_interpreter.filter(m_node->child1(), SpecString | SpecStringObject); return; } case CellUse: case NotCellUse: case UntypedUse: { LValue value; if (m_node->child1().useKind() == CellUse) value = lowCell(m_node->child1()); else if (m_node->child1().useKind() == NotCellUse) value = lowNotCell(m_node->child1()); else value = lowJSValue(m_node->child1()); LBasicBlock isCell = m_out.newBlock(); LBasicBlock notString = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue isCellPredicate; if (m_node->child1().useKind() == CellUse) isCellPredicate = m_out.booleanTrue; else if (m_node->child1().useKind() == NotCellUse) isCellPredicate = m_out.booleanFalse; else isCellPredicate = this->isCell(value, provenType(m_node->child1())); m_out.branch(isCellPredicate, unsure(isCell), unsure(notString)); LBasicBlock lastNext = m_out.appendTo(isCell, notString); ValueFromBlock simpleResult = m_out.anchor(value); LValue isStringPredicate; if (m_node->child1()->prediction() & SpecString) { isStringPredicate = isString(value, provenType(m_node->child1())); } else isStringPredicate = m_out.booleanFalse; m_out.branch(isStringPredicate, unsure(continuation), unsure(notString)); m_out.appendTo(notString, continuation); LValue operation; if (m_node->child1().useKind() == CellUse) operation = m_out.operation(m_node->op() == ToString ? operationToStringOnCell : operationCallStringConstructorOnCell); else operation = m_out.operation(m_node->op() == ToString ? operationToString : operationCallStringConstructor); ValueFromBlock convertedResult = m_out.anchor(vmCall(Int64, operation, m_callFrame, value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, simpleResult, convertedResult)); return; } case Int32Use: setJSValue(vmCall(Int64, m_out.operation(operationInt32ToStringWithValidRadix), m_callFrame, lowInt32(m_node->child1()), m_out.constInt32(10))); return; case Int52RepUse: setJSValue(vmCall(Int64, m_out.operation(operationInt52ToStringWithValidRadix), m_callFrame, lowStrictInt52(m_node->child1()), m_out.constInt32(10))); return; case DoubleRepUse: setJSValue(vmCall(Int64, m_out.operation(operationDoubleToStringWithValidRadix), m_callFrame, lowDouble(m_node->child1()), m_out.constInt32(10))); return; default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } } void compileToPrimitive() { LValue value = lowJSValue(m_node->child1()); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock isObjectCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); Vector results; results.append(m_out.anchor(value)); m_out.branch( isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isCellCase, isObjectCase); results.append(m_out.anchor(value)); m_out.branch( isObject(value, provenType(m_node->child1())), unsure(isObjectCase), unsure(continuation)); m_out.appendTo(isObjectCase, continuation); results.append(m_out.anchor(vmCall( Int64, m_out.operation(operationToPrimitive), m_callFrame, value))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, results)); } void compileMakeRope() { LValue kids[3]; unsigned numKids; kids[0] = lowCell(m_node->child1()); kids[1] = lowCell(m_node->child2()); if (m_node->child3()) { kids[2] = lowCell(m_node->child3()); numKids = 3; } else { kids[2] = 0; numKids = 2; } LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); MarkedAllocator* allocator = subspaceFor(vm())->allocatorFor(sizeof(JSRopeString)); DFG_ASSERT(m_graph, m_node, allocator); LValue result = allocateCell( m_out.constIntPtr(allocator), vm().stringStructure.get(), slowPath); m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSString_value); for (unsigned i = 0; i < numKids; ++i) m_out.storePtr(kids[i], result, m_heaps.JSRopeString_fibers[i]); for (unsigned i = numKids; i < JSRopeString::s_maxInternalRopeLength; ++i) m_out.storePtr(m_out.intPtrZero, result, m_heaps.JSRopeString_fibers[i]); LValue flags = m_out.load32(kids[0], m_heaps.JSString_flags); LValue length = m_out.load32(kids[0], m_heaps.JSString_length); for (unsigned i = 1; i < numKids; ++i) { flags = m_out.bitAnd(flags, m_out.load32(kids[i], m_heaps.JSString_flags)); CheckValue* lengthCheck = m_out.speculateAdd( length, m_out.load32(kids[i], m_heaps.JSString_length)); blessSpeculation(lengthCheck, Uncountable, noValue(), nullptr, m_origin); length = lengthCheck; } m_out.store32( m_out.bitAnd(m_out.constInt32(JSString::Is8Bit), flags), result, m_heaps.JSString_flags); m_out.store32(length, result, m_heaps.JSString_length); mutatorFence(); ValueFromBlock fastResult = m_out.anchor(result); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); LValue slowResultValue; switch (numKids) { case 2: slowResultValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationMakeRope2, locations[0].directGPR(), locations[1].directGPR(), locations[2].directGPR()); }, kids[0], kids[1]); break; case 3: slowResultValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationMakeRope3, locations[0].directGPR(), locations[1].directGPR(), locations[2].directGPR(), locations[3].directGPR()); }, kids[0], kids[1], kids[2]); break; default: DFG_CRASH(m_graph, m_node, "Bad number of children"); break; } ValueFromBlock slowResult = m_out.anchor(slowResultValue); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, fastResult, slowResult)); } void compileStringCharAt() { LValue base = lowCell(m_node->child1()); LValue index = lowInt32(m_node->child2()); LValue storage = lowStorage(m_node->child3()); LBasicBlock fastPath = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.aboveOrEqual( index, m_out.load32NonNegative(base, m_heaps.JSString_length)), rarely(slowPath), usually(fastPath)); LBasicBlock lastNext = m_out.appendTo(fastPath, slowPath); LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value); LBasicBlock is8Bit = m_out.newBlock(); LBasicBlock is16Bit = m_out.newBlock(); LBasicBlock bitsContinuation = m_out.newBlock(); LBasicBlock bigCharacter = m_out.newBlock(); m_out.branch( m_out.testIsZero32( m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), unsure(is16Bit), unsure(is8Bit)); m_out.appendTo(is8Bit, is16Bit); ValueFromBlock char8Bit = m_out.anchor( m_out.load8ZeroExt32(m_out.baseIndex( m_heaps.characters8, storage, m_out.zeroExtPtr(index), provenValue(m_node->child2())))); m_out.jump(bitsContinuation); m_out.appendTo(is16Bit, bigCharacter); LValue char16BitValue = m_out.load16ZeroExt32( m_out.baseIndex( m_heaps.characters16, storage, m_out.zeroExtPtr(index), provenValue(m_node->child2()))); ValueFromBlock char16Bit = m_out.anchor(char16BitValue); m_out.branch( m_out.aboveOrEqual(char16BitValue, m_out.constInt32(0x100)), rarely(bigCharacter), usually(bitsContinuation)); m_out.appendTo(bigCharacter, bitsContinuation); Vector results; results.append(m_out.anchor(vmCall( Int64, m_out.operation(operationSingleCharacterString), m_callFrame, char16BitValue))); m_out.jump(continuation); m_out.appendTo(bitsContinuation, slowPath); LValue character = m_out.phi(Int32, char8Bit, char16Bit); LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings()); results.append(m_out.anchor(m_out.loadPtr(m_out.baseIndex( m_heaps.singleCharacterStrings, smallStrings, m_out.zeroExtPtr(character))))); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); if (m_node->arrayMode().isInBounds()) { speculate(OutOfBounds, noValue(), 0, m_out.booleanTrue); results.append(m_out.anchor(m_out.intPtrZero)); } else { JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); bool prototypeChainIsSane = false; if (globalObject->stringPrototypeChainIsSane()) { // FIXME: This could be captured using a Speculation mode that means // "out-of-bounds loads return a trivial value", something like // SaneChainOutOfBounds. // https://bugs.webkit.org/show_bug.cgi?id=144668 m_graph.watchpoints().addLazily(globalObject->stringPrototype()->structure()->transitionWatchpointSet()); m_graph.watchpoints().addLazily(globalObject->objectPrototype()->structure()->transitionWatchpointSet()); prototypeChainIsSane = globalObject->stringPrototypeChainIsSane(); } if (prototypeChainIsSane) { LBasicBlock negativeIndex = m_out.newBlock(); results.append(m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined())))); m_out.branch( m_out.lessThan(index, m_out.int32Zero), rarely(negativeIndex), usually(continuation)); m_out.appendTo(negativeIndex, continuation); } results.append(m_out.anchor(vmCall( Int64, m_out.operation(operationGetByValStringInt), m_callFrame, base, index))); } m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, results)); } void compileStringCharCodeAt() { LBasicBlock is8Bit = m_out.newBlock(); LBasicBlock is16Bit = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue base = lowCell(m_node->child1()); LValue index = lowInt32(m_node->child2()); LValue storage = lowStorage(m_node->child3()); speculate( Uncountable, noValue(), 0, m_out.aboveOrEqual( index, m_out.load32NonNegative(base, m_heaps.JSString_length))); LValue stringImpl = m_out.loadPtr(base, m_heaps.JSString_value); m_out.branch( m_out.testIsZero32( m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), unsure(is16Bit), unsure(is8Bit)); LBasicBlock lastNext = m_out.appendTo(is8Bit, is16Bit); ValueFromBlock char8Bit = m_out.anchor( m_out.load8ZeroExt32(m_out.baseIndex( m_heaps.characters8, storage, m_out.zeroExtPtr(index), provenValue(m_node->child2())))); m_out.jump(continuation); m_out.appendTo(is16Bit, continuation); ValueFromBlock char16Bit = m_out.anchor( m_out.load16ZeroExt32(m_out.baseIndex( m_heaps.characters16, storage, m_out.zeroExtPtr(index), provenValue(m_node->child2())))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setInt32(m_out.phi(Int32, char8Bit, char16Bit)); } void compileStringFromCharCode() { Edge childEdge = m_node->child1(); if (childEdge.useKind() == UntypedUse) { LValue result = vmCall( Int64, m_out.operation(operationStringFromCharCodeUntyped), m_callFrame, lowJSValue(childEdge)); setJSValue(result); return; } DFG_ASSERT(m_graph, m_node, childEdge.useKind() == Int32Use); LValue value = lowInt32(childEdge); LBasicBlock smallIntCase = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.aboveOrEqual(value, m_out.constInt32(0xff)), rarely(slowCase), usually(smallIntCase)); LBasicBlock lastNext = m_out.appendTo(smallIntCase, slowCase); LValue smallStrings = m_out.constIntPtr(vm().smallStrings.singleCharacterStrings()); LValue fastResultValue = m_out.loadPtr( m_out.baseIndex(m_heaps.singleCharacterStrings, smallStrings, m_out.zeroExtPtr(value))); ValueFromBlock fastResult = m_out.anchor(fastResultValue); m_out.jump(continuation); m_out.appendTo(slowCase, continuation); LValue slowResultValue = vmCall( pointerType(), m_out.operation(operationStringFromCharCode), m_callFrame, value); ValueFromBlock slowResult = m_out.anchor(slowResultValue); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, fastResult, slowResult)); } void compileGetByOffset() { StorageAccessData& data = m_node->storageAccessData(); setJSValue(loadProperty( lowStorage(m_node->child1()), data.identifierNumber, data.offset)); } void compileGetGetter() { setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_getter)); } void compileGetSetter() { setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.GetterSetter_setter)); } void compileMultiGetByOffset() { LValue base = lowCell(m_node->child1()); MultiGetByOffsetData& data = m_node->multiGetByOffsetData(); if (data.cases.isEmpty()) { // Protect against creating a Phi function with zero inputs. LLVM didn't like that. // It's not clear if this is needed anymore. // FIXME: https://bugs.webkit.org/show_bug.cgi?id=154382 terminate(BadCache); return; } Vector blocks(data.cases.size()); for (unsigned i = data.cases.size(); i--;) blocks[i] = m_out.newBlock(); LBasicBlock exit = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); Vector cases; RegisteredStructureSet baseSet; for (unsigned i = data.cases.size(); i--;) { MultiGetByOffsetCase getCase = data.cases[i]; for (unsigned j = getCase.set().size(); j--;) { RegisteredStructure structure = getCase.set()[j]; baseSet.add(structure); cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1))); } } m_out.switchInstruction( m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0)); LBasicBlock lastNext = m_out.m_nextBlock; Vector results; for (unsigned i = data.cases.size(); i--;) { MultiGetByOffsetCase getCase = data.cases[i]; GetByOffsetMethod method = getCase.method(); m_out.appendTo(blocks[i], i + 1 < data.cases.size() ? blocks[i + 1] : exit); LValue result; switch (method.kind()) { case GetByOffsetMethod::Invalid: RELEASE_ASSERT_NOT_REACHED(); break; case GetByOffsetMethod::Constant: result = m_out.constInt64(JSValue::encode(method.constant()->value())); break; case GetByOffsetMethod::Load: case GetByOffsetMethod::LoadFromPrototype: { LValue propertyBase; if (method.kind() == GetByOffsetMethod::Load) propertyBase = base; else propertyBase = weakPointer(method.prototype()->value().asCell()); if (!isInlineOffset(method.offset())) propertyBase = m_out.loadPtr(propertyBase, m_heaps.JSObject_butterfly); result = loadProperty( propertyBase, data.identifierNumber, method.offset()); break; } } results.append(m_out.anchor(result)); m_out.jump(continuation); } m_out.appendTo(exit, continuation); if (!m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet)) speculate(BadCache, noValue(), nullptr, m_out.booleanTrue); m_out.unreachable(); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, results)); } void compilePutByOffset() { StorageAccessData& data = m_node->storageAccessData(); storeProperty( lowJSValue(m_node->child3()), lowStorage(m_node->child1()), data.identifierNumber, data.offset); } void compileMultiPutByOffset() { LValue base = lowCell(m_node->child1()); LValue value = lowJSValue(m_node->child2()); MultiPutByOffsetData& data = m_node->multiPutByOffsetData(); Vector blocks(data.variants.size()); for (unsigned i = data.variants.size(); i--;) blocks[i] = m_out.newBlock(); LBasicBlock exit = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); Vector cases; RegisteredStructureSet baseSet; for (unsigned i = data.variants.size(); i--;) { PutByIdVariant variant = data.variants[i]; for (unsigned j = variant.oldStructure().size(); j--;) { RegisteredStructure structure = m_graph.registerStructure(variant.oldStructure()[j]); baseSet.add(structure); cases.append(SwitchCase(weakStructureID(structure), blocks[i], Weight(1))); } } m_out.switchInstruction( m_out.load32(base, m_heaps.JSCell_structureID), cases, exit, Weight(0)); LBasicBlock lastNext = m_out.m_nextBlock; for (unsigned i = data.variants.size(); i--;) { m_out.appendTo(blocks[i], i + 1 < data.variants.size() ? blocks[i + 1] : exit); PutByIdVariant variant = data.variants[i]; checkInferredType(m_node->child2(), value, variant.requiredType()); LValue storage; if (variant.kind() == PutByIdVariant::Replace) { if (isInlineOffset(variant.offset())) storage = base; else storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly); } else { DFG_ASSERT(m_graph, m_node, variant.kind() == PutByIdVariant::Transition); m_graph.m_plan.transitions.addLazily( codeBlock(), m_node->origin.semantic.codeOriginOwner(), variant.oldStructureForTransition(), variant.newStructure()); storage = storageForTransition( base, variant.offset(), variant.oldStructureForTransition(), variant.newStructure()); } storeProperty(value, storage, data.identifierNumber, variant.offset()); if (variant.kind() == PutByIdVariant::Transition) { ASSERT(variant.oldStructureForTransition()->indexingType() == variant.newStructure()->indexingType()); ASSERT(variant.oldStructureForTransition()->typeInfo().inlineTypeFlags() == variant.newStructure()->typeInfo().inlineTypeFlags()); ASSERT(variant.oldStructureForTransition()->typeInfo().type() == variant.newStructure()->typeInfo().type()); m_out.store32( weakStructureID(m_graph.registerStructure(variant.newStructure())), base, m_heaps.JSCell_structureID); } m_out.jump(continuation); } m_out.appendTo(exit, continuation); if (!m_interpreter.forNode(m_node->child1()).m_structure.isSubsetOf(baseSet)) speculate(BadCache, noValue(), nullptr, m_out.booleanTrue); m_out.unreachable(); m_out.appendTo(continuation, lastNext); } void compileGetGlobalVariable() { setJSValue(m_out.load64(m_out.absolute(m_node->variablePointer()))); } void compilePutGlobalVariable() { m_out.store64( lowJSValue(m_node->child2()), m_out.absolute(m_node->variablePointer())); } void compileNotifyWrite() { WatchpointSet* set = m_node->watchpointSet(); LBasicBlock isNotInvalidated = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue state = m_out.load8ZeroExt32(m_out.absolute(set->addressOfState())); m_out.branch( m_out.equal(state, m_out.constInt32(IsInvalidated)), usually(continuation), rarely(isNotInvalidated)); LBasicBlock lastNext = m_out.appendTo(isNotInvalidated, continuation); lazySlowPath( [=] (const Vector&) -> RefPtr { return createLazyCallGenerator( operationNotifyWrite, InvalidGPRReg, CCallHelpers::TrustedImmPtr(set)); }); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void compileGetCallee() { setJSValue(m_out.loadPtr(addressFor(CallFrameSlot::callee))); } void compileGetArgumentCountIncludingThis() { setInt32(m_out.load32(payloadFor(CallFrameSlot::argumentCount))); } void compileGetScope() { setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSFunction_scope)); } void compileSkipScope() { setJSValue(m_out.loadPtr(lowCell(m_node->child1()), m_heaps.JSScope_next)); } void compileGetGlobalObject() { LValue structure = loadStructure(lowCell(m_node->child1())); setJSValue(m_out.loadPtr(structure, m_heaps.Structure_globalObject)); } void compileGetClosureVar() { setJSValue( m_out.load64( lowCell(m_node->child1()), m_heaps.JSEnvironmentRecord_variables[m_node->scopeOffset().offset()])); } void compilePutClosureVar() { m_out.store64( lowJSValue(m_node->child2()), lowCell(m_node->child1()), m_heaps.JSEnvironmentRecord_variables[m_node->scopeOffset().offset()]); } void compileGetFromArguments() { setJSValue( m_out.load64( lowCell(m_node->child1()), m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()])); } void compilePutToArguments() { m_out.store64( lowJSValue(m_node->child2()), lowCell(m_node->child1()), m_heaps.DirectArguments_storage[m_node->capturedArgumentsOffset().offset()]); } void compileGetArgument() { LValue argumentCount = m_out.load32(payloadFor(AssemblyHelpers::argumentCount(m_node->origin.semantic))); LBasicBlock inBounds = m_out.newBlock(); LBasicBlock outOfBounds = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(m_out.lessThanOrEqual(argumentCount, m_out.constInt32(m_node->argumentIndex())), unsure(outOfBounds), unsure(inBounds)); LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds); VirtualRegister arg = AssemblyHelpers::argumentsStart(m_node->origin.semantic) + m_node->argumentIndex() - 1; ValueFromBlock inBoundsResult = m_out.anchor(m_out.load64(addressFor(arg))); m_out.jump(continuation); m_out.appendTo(outOfBounds, continuation); ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(ValueUndefined)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, inBoundsResult, outOfBoundsResult)); } void compileCompareEq() { if (m_node->isBinaryUseKind(Int32Use) || m_node->isBinaryUseKind(Int52RepUse) || m_node->isBinaryUseKind(DoubleRepUse) || m_node->isBinaryUseKind(ObjectUse) || m_node->isBinaryUseKind(BooleanUse) || m_node->isBinaryUseKind(SymbolUse) || m_node->isBinaryUseKind(StringIdentUse) || m_node->isBinaryUseKind(StringUse)) { compileCompareStrictEq(); return; } if (m_node->isBinaryUseKind(ObjectUse, ObjectOrOtherUse)) { compareEqObjectOrOtherToObject(m_node->child2(), m_node->child1()); return; } if (m_node->isBinaryUseKind(ObjectOrOtherUse, ObjectUse)) { compareEqObjectOrOtherToObject(m_node->child1(), m_node->child2()); return; } if (m_node->child1().useKind() == OtherUse) { ASSERT(!m_interpreter.needsTypeCheck(m_node->child1(), SpecOther)); setBoolean(equalNullOrUndefined(m_node->child2(), AllCellsAreFalse, EqualNullOrUndefined, ManualOperandSpeculation)); return; } if (m_node->child2().useKind() == OtherUse) { ASSERT(!m_interpreter.needsTypeCheck(m_node->child2(), SpecOther)); setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualNullOrUndefined, ManualOperandSpeculation)); return; } DFG_ASSERT(m_graph, m_node, m_node->isBinaryUseKind(UntypedUse)); nonSpeculativeCompare( [&] (LValue left, LValue right) { return m_out.equal(left, right); }, operationCompareEq); } void compileCompareStrictEq() { if (m_node->isBinaryUseKind(Int32Use)) { setBoolean( m_out.equal(lowInt32(m_node->child1()), lowInt32(m_node->child2()))); return; } if (m_node->isBinaryUseKind(Int52RepUse)) { Int52Kind kind; LValue left = lowWhicheverInt52(m_node->child1(), kind); LValue right = lowInt52(m_node->child2(), kind); setBoolean(m_out.equal(left, right)); return; } if (m_node->isBinaryUseKind(DoubleRepUse)) { setBoolean( m_out.doubleEqual(lowDouble(m_node->child1()), lowDouble(m_node->child2()))); return; } if (m_node->isBinaryUseKind(StringIdentUse)) { setBoolean( m_out.equal(lowStringIdent(m_node->child1()), lowStringIdent(m_node->child2()))); return; } if (m_node->isBinaryUseKind(StringUse)) { LValue left = lowCell(m_node->child1()); LValue right = lowCell(m_node->child2()); LBasicBlock notTriviallyEqualCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); speculateString(m_node->child1(), left); ValueFromBlock fastResult = m_out.anchor(m_out.booleanTrue); m_out.branch( m_out.equal(left, right), unsure(continuation), unsure(notTriviallyEqualCase)); LBasicBlock lastNext = m_out.appendTo(notTriviallyEqualCase, continuation); speculateString(m_node->child2(), right); ValueFromBlock slowResult = m_out.anchor(stringsEqual(left, right)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, fastResult, slowResult)); return; } if (m_node->isBinaryUseKind(ObjectUse, UntypedUse)) { setBoolean( m_out.equal( lowNonNullObject(m_node->child1()), lowJSValue(m_node->child2()))); return; } if (m_node->isBinaryUseKind(UntypedUse, ObjectUse)) { setBoolean( m_out.equal( lowNonNullObject(m_node->child2()), lowJSValue(m_node->child1()))); return; } if (m_node->isBinaryUseKind(ObjectUse)) { setBoolean( m_out.equal( lowNonNullObject(m_node->child1()), lowNonNullObject(m_node->child2()))); return; } if (m_node->isBinaryUseKind(BooleanUse)) { setBoolean( m_out.equal(lowBoolean(m_node->child1()), lowBoolean(m_node->child2()))); return; } if (m_node->isBinaryUseKind(SymbolUse)) { LValue leftSymbol = lowSymbol(m_node->child1()); LValue rightSymbol = lowSymbol(m_node->child2()); setBoolean(m_out.equal(leftSymbol, rightSymbol)); return; } if (m_node->isBinaryUseKind(SymbolUse, UntypedUse) || m_node->isBinaryUseKind(UntypedUse, SymbolUse)) { Edge symbolEdge = m_node->child1(); Edge untypedEdge = m_node->child2(); if (symbolEdge.useKind() != SymbolUse) std::swap(symbolEdge, untypedEdge); LValue leftSymbol = lowSymbol(symbolEdge); LValue untypedValue = lowJSValue(untypedEdge); setBoolean(m_out.equal(leftSymbol, untypedValue)); return; } if (m_node->isBinaryUseKind(MiscUse, UntypedUse) || m_node->isBinaryUseKind(UntypedUse, MiscUse)) { speculate(m_node->child1()); speculate(m_node->child2()); LValue left = lowJSValue(m_node->child1(), ManualOperandSpeculation); LValue right = lowJSValue(m_node->child2(), ManualOperandSpeculation); setBoolean(m_out.equal(left, right)); return; } if (m_node->isBinaryUseKind(StringIdentUse, NotStringVarUse) || m_node->isBinaryUseKind(NotStringVarUse, StringIdentUse)) { Edge leftEdge = m_node->childFor(StringIdentUse); Edge rightEdge = m_node->childFor(NotStringVarUse); LValue left = lowStringIdent(leftEdge); LValue rightValue = lowJSValue(rightEdge, ManualOperandSpeculation); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock isStringCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); m_out.branch( isCell(rightValue, provenType(rightEdge)), unsure(isCellCase), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase); ValueFromBlock notStringResult = m_out.anchor(m_out.booleanFalse); m_out.branch( isString(rightValue, provenType(rightEdge)), unsure(isStringCase), unsure(continuation)); m_out.appendTo(isStringCase, continuation); LValue right = m_out.loadPtr(rightValue, m_heaps.JSString_value); speculateStringIdent(rightEdge, rightValue, right); ValueFromBlock isStringResult = m_out.anchor(m_out.equal(left, right)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, notCellResult, notStringResult, isStringResult)); return; } if (m_node->isBinaryUseKind(StringUse, UntypedUse)) { compileStringToUntypedStrictEquality(m_node->child1(), m_node->child2()); return; } if (m_node->isBinaryUseKind(UntypedUse, StringUse)) { compileStringToUntypedStrictEquality(m_node->child2(), m_node->child1()); return; } DFG_ASSERT(m_graph, m_node, m_node->isBinaryUseKind(UntypedUse)); nonSpeculativeCompare( [&] (LValue left, LValue right) { return m_out.equal(left, right); }, operationCompareStrictEq); } void compileStringToUntypedStrictEquality(Edge stringEdge, Edge untypedEdge) { ASSERT(stringEdge.useKind() == StringUse); ASSERT(untypedEdge.useKind() == UntypedUse); LValue leftString = lowCell(stringEdge); LValue rightValue = lowJSValue(untypedEdge); SpeculatedType rightValueType = provenType(untypedEdge); // Verify left is string. speculateString(stringEdge, leftString); LBasicBlock testUntypedEdgeIsCell = m_out.newBlock(); LBasicBlock testUntypedEdgeIsString = m_out.newBlock(); LBasicBlock testStringEquality = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); // Given left is string. If the value are strictly equal, rightValue has to be the same string. ValueFromBlock fastTrue = m_out.anchor(m_out.booleanTrue); m_out.branch(m_out.equal(leftString, rightValue), unsure(continuation), unsure(testUntypedEdgeIsCell)); LBasicBlock lastNext = m_out.appendTo(testUntypedEdgeIsCell, testUntypedEdgeIsString); ValueFromBlock fastFalse = m_out.anchor(m_out.booleanFalse); m_out.branch(isNotCell(rightValue, rightValueType), unsure(continuation), unsure(testUntypedEdgeIsString)); // Check if the untyped edge is a string. m_out.appendTo(testUntypedEdgeIsString, testStringEquality); m_out.branch(isNotString(rightValue, rightValueType), unsure(continuation), unsure(testStringEquality)); // Full String compare. m_out.appendTo(testStringEquality, continuation); ValueFromBlock slowResult = m_out.anchor(stringsEqual(leftString, rightValue)); m_out.jump(continuation); // Continuation. m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, fastTrue, fastFalse, slowResult)); } void compileCompareEqPtr() { setBoolean( m_out.equal( lowJSValue(m_node->child1()), weakPointer(m_node->cellOperand()->cell()))); } void compileCompareLess() { compare( [&] (LValue left, LValue right) { return m_out.lessThan(left, right); }, [&] (LValue left, LValue right) { return m_out.doubleLessThan(left, right); }, operationCompareStringImplLess, operationCompareStringLess, operationCompareLess); } void compileCompareLessEq() { compare( [&] (LValue left, LValue right) { return m_out.lessThanOrEqual(left, right); }, [&] (LValue left, LValue right) { return m_out.doubleLessThanOrEqual(left, right); }, operationCompareStringImplLessEq, operationCompareStringLessEq, operationCompareLessEq); } void compileCompareGreater() { compare( [&] (LValue left, LValue right) { return m_out.greaterThan(left, right); }, [&] (LValue left, LValue right) { return m_out.doubleGreaterThan(left, right); }, operationCompareStringImplGreater, operationCompareStringGreater, operationCompareGreater); } void compileCompareGreaterEq() { compare( [&] (LValue left, LValue right) { return m_out.greaterThanOrEqual(left, right); }, [&] (LValue left, LValue right) { return m_out.doubleGreaterThanOrEqual(left, right); }, operationCompareStringImplGreaterEq, operationCompareStringGreaterEq, operationCompareGreaterEq); } void compileLogicalNot() { setBoolean(m_out.logicalNot(boolify(m_node->child1()))); } void compileCallOrConstruct() { Node* node = m_node; unsigned numArgs = node->numChildren() - 1; LValue jsCallee = lowJSValue(m_graph.varArgChild(node, 0)); unsigned frameSize = (CallFrame::headerSizeInRegisters + numArgs) * sizeof(EncodedJSValue); unsigned alignedFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), frameSize); // JS->JS calling convention requires that the caller allows this much space on top of stack to // get trashed by the callee, even if not all of that space is used to pass arguments. We tell // B3 this explicitly for two reasons: // // - We will only pass frameSize worth of stuff. // - The trashed stack guarantee is logically separate from the act of passing arguments, so we // shouldn't rely on Air to infer the trashed stack property based on the arguments it ends // up seeing. m_proc.requestCallArgAreaSizeInBytes(alignedFrameSize); // Collect the arguments, since this can generate code and we want to generate it before we emit // the call. Vector arguments; // Make sure that the callee goes into GPR0 because that's where the slow path thunks expect the // callee to be. arguments.append(ConstrainedValue(jsCallee, ValueRep::reg(GPRInfo::regT0))); auto addArgument = [&] (LValue value, VirtualRegister reg, int offset) { intptr_t offsetFromSP = (reg.offset() - CallerFrameAndPC::sizeInRegisters) * sizeof(EncodedJSValue) + offset; arguments.append(ConstrainedValue(value, ValueRep::stackArgument(offsetFromSP))); }; addArgument(jsCallee, VirtualRegister(CallFrameSlot::callee), 0); addArgument(m_out.constInt32(numArgs), VirtualRegister(CallFrameSlot::argumentCount), PayloadOffset); for (unsigned i = 0; i < numArgs; ++i) addArgument(lowJSValue(m_graph.varArgChild(node, 1 + i)), virtualRegisterForArgument(i), 0); PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendVector(arguments); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister)); patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall()); patchpoint->resultConstraint = ValueRep::reg(GPRInfo::returnValueGPR); CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite(); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin); exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex); jit.store32( CCallHelpers::TrustedImm32(callSiteIndex.bits()), CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount))); CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo(); CCallHelpers::DataLabelPtr targetToCheck; CCallHelpers::Jump slowPath = jit.branchPtrWithPatch( CCallHelpers::NotEqual, GPRInfo::regT0, targetToCheck, CCallHelpers::TrustedImmPtr(0)); CCallHelpers::Call fastCall = jit.nearCall(); CCallHelpers::Jump done = jit.jump(); slowPath.link(&jit); jit.move(CCallHelpers::TrustedImmPtr(callLinkInfo), GPRInfo::regT2); CCallHelpers::Call slowCall = jit.nearCall(); done.link(&jit); callLinkInfo->setUpCall( node->op() == Construct ? CallLinkInfo::Construct : CallLinkInfo::Call, node->origin.semantic, GPRInfo::regT0); jit.addPtr( CCallHelpers::TrustedImm32(-params.proc().frameSize()), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { MacroAssemblerCodePtr linkCall = linkBuffer.vm().getCTIStub(linkCallThunkGenerator).code(); linkBuffer.link(slowCall, FunctionPtr(linkCall.executableAddress())); callLinkInfo->setCallLocations( CodeLocationLabel(linkBuffer.locationOfNearCall(slowCall)), CodeLocationLabel(linkBuffer.locationOf(targetToCheck)), linkBuffer.locationOfNearCall(fastCall)); }); }); setJSValue(patchpoint); } void compileDirectCallOrConstruct() { Node* node = m_node; bool isTail = node->op() == DirectTailCall; bool isConstruct = node->op() == DirectConstruct; ExecutableBase* executable = node->castOperand(); FunctionExecutable* functionExecutable = jsDynamicCast(vm(), executable); unsigned numPassedArgs = node->numChildren() - 1; unsigned numAllocatedArgs = numPassedArgs; if (functionExecutable) { numAllocatedArgs = std::max( numAllocatedArgs, std::min( static_cast(functionExecutable->parameterCount()) + 1, Options::maximumDirectCallStackSize())); } LValue jsCallee = lowJSValue(m_graph.varArgChild(node, 0)); if (!isTail) { unsigned frameSize = (CallFrame::headerSizeInRegisters + numAllocatedArgs) * sizeof(EncodedJSValue); unsigned alignedFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), frameSize); m_proc.requestCallArgAreaSizeInBytes(alignedFrameSize); } Vector arguments; arguments.append(ConstrainedValue(jsCallee, ValueRep::SomeRegister)); if (!isTail) { auto addArgument = [&] (LValue value, VirtualRegister reg, int offset) { intptr_t offsetFromSP = (reg.offset() - CallerFrameAndPC::sizeInRegisters) * sizeof(EncodedJSValue) + offset; arguments.append(ConstrainedValue(value, ValueRep::stackArgument(offsetFromSP))); }; addArgument(jsCallee, VirtualRegister(CallFrameSlot::callee), 0); addArgument(m_out.constInt32(numPassedArgs), VirtualRegister(CallFrameSlot::argumentCount), PayloadOffset); for (unsigned i = 0; i < numPassedArgs; ++i) addArgument(lowJSValue(m_graph.varArgChild(node, 1 + i)), virtualRegisterForArgument(i), 0); for (unsigned i = numPassedArgs; i < numAllocatedArgs; ++i) addArgument(m_out.constInt64(JSValue::encode(jsUndefined())), virtualRegisterForArgument(i), 0); } else { for (unsigned i = 0; i < numPassedArgs; ++i) arguments.append(ConstrainedValue(lowJSValue(m_graph.varArgChild(node, 1 + i)), ValueRep::WarmAny)); } PatchpointValue* patchpoint = m_out.patchpoint(isTail ? Void : Int64); patchpoint->appendVector(arguments); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); if (isTail) { // The shuffler needs tags. patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister)); } patchpoint->clobber(RegisterSet::macroScratchRegisters()); if (!isTail) { patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall()); patchpoint->resultConstraint = ValueRep::reg(GPRInfo::returnValueGPR); } CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite(); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin); GPRReg calleeGPR = params[!isTail].gpr(); exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); if (isTail) { CallFrameShuffleData shuffleData; shuffleData.numLocals = state->jitCode->common.frameRegisterCount; RegisterSet toSave = params.unavailableRegisters(); shuffleData.callee = ValueRecovery::inGPR(calleeGPR, DataFormatCell); toSave.set(calleeGPR); for (unsigned i = 0; i < numPassedArgs; ++i) { ValueRecovery recovery = params[1 + i].recoveryForJSValue(); shuffleData.args.append(recovery); recovery.forEachReg( [&] (Reg reg) { toSave.set(reg); }); } for (unsigned i = numPassedArgs; i < numAllocatedArgs; ++i) shuffleData.args.append(ValueRecovery::constant(jsUndefined())); shuffleData.numPassedArgs = numPassedArgs; shuffleData.setupCalleeSaveRegisters(jit.codeBlock()); CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo(); CCallHelpers::PatchableJump patchableJump = jit.patchableJump(); CCallHelpers::Label mainPath = jit.label(); jit.store32( CCallHelpers::TrustedImm32(callSiteIndex.bits()), CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount))); callLinkInfo->setFrameShuffleData(shuffleData); CallFrameShuffler(jit, shuffleData).prepareForTailCall(); CCallHelpers::Call call = jit.nearTailCall(); jit.abortWithReason(JITDidReturnFromTailCall); CCallHelpers::Label slowPath = jit.label(); patchableJump.m_jump.linkTo(slowPath, &jit); callOperation( *state, toSave, jit, node->origin.semantic, exceptions.get(), operationLinkDirectCall, InvalidGPRReg, CCallHelpers::TrustedImmPtr(callLinkInfo), calleeGPR).call(); jit.jump().linkTo(mainPath, &jit); callLinkInfo->setUpCall( CallLinkInfo::DirectTailCall, node->origin.semantic, InvalidGPRReg); callLinkInfo->setExecutableDuringCompilation(executable); if (numAllocatedArgs > numPassedArgs) callLinkInfo->setMaxNumArguments(numAllocatedArgs); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { CodeLocationLabel patchableJumpLocation = linkBuffer.locationOf(patchableJump); CodeLocationNearCall callLocation = linkBuffer.locationOfNearCall(call); CodeLocationLabel slowPathLocation = linkBuffer.locationOf(slowPath); callLinkInfo->setCallLocations( patchableJumpLocation, slowPathLocation, callLocation); }); return; } CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo(); CCallHelpers::Label mainPath = jit.label(); jit.store32( CCallHelpers::TrustedImm32(callSiteIndex.bits()), CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount))); CCallHelpers::Call call = jit.nearCall(); jit.addPtr( CCallHelpers::TrustedImm32(-params.proc().frameSize()), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister); callLinkInfo->setUpCall( isConstruct ? CallLinkInfo::DirectConstruct : CallLinkInfo::DirectCall, node->origin.semantic, InvalidGPRReg); callLinkInfo->setExecutableDuringCompilation(executable); if (numAllocatedArgs > numPassedArgs) callLinkInfo->setMaxNumArguments(numAllocatedArgs); params.addLatePath( [=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); CCallHelpers::Label slowPath = jit.label(); if (isX86()) jit.pop(CCallHelpers::selectScratchGPR(calleeGPR)); callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), operationLinkDirectCall, InvalidGPRReg, CCallHelpers::TrustedImmPtr(callLinkInfo), calleeGPR).call(); jit.jump().linkTo(mainPath, &jit); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { CodeLocationNearCall callLocation = linkBuffer.locationOfNearCall(call); CodeLocationLabel slowPathLocation = linkBuffer.locationOf(slowPath); linkBuffer.link(call, slowPathLocation); callLinkInfo->setCallLocations( CodeLocationLabel(), slowPathLocation, callLocation); }); }); }); if (isTail) patchpoint->effects.terminal = true; else setJSValue(patchpoint); } void compileTailCall() { Node* node = m_node; unsigned numArgs = node->numChildren() - 1; LValue jsCallee = lowJSValue(m_graph.varArgChild(node, 0)); // We want B3 to give us all of the arguments using whatever mechanism it thinks is // convenient. The generator then shuffles those arguments into our own call frame, // destroying our frame in the process. // Note that we don't have to do anything special for exceptions. A tail call is only a // tail call if it is not inside a try block. Vector arguments; arguments.append(ConstrainedValue(jsCallee, ValueRep::reg(GPRInfo::regT0))); for (unsigned i = 0; i < numArgs; ++i) { // Note: we could let the shuffler do boxing for us, but it's not super clear that this // would be better. Also, if we wanted to do that, then we'd have to teach the shuffler // that 32-bit values could land at 4-byte alignment but not 8-byte alignment. ConstrainedValue constrainedValue( lowJSValue(m_graph.varArgChild(node, 1 + i)), ValueRep::WarmAny); arguments.append(constrainedValue); } PatchpointValue* patchpoint = m_out.patchpoint(Void); patchpoint->appendVector(arguments); patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister)); // Prevent any of the arguments from using the scratch register. patchpoint->clobberEarly(RegisterSet::macroScratchRegisters()); patchpoint->effects.terminal = true; // We don't have to tell the patchpoint that we will clobber registers, since we won't return // anyway. CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite(); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin); // Yes, this is really necessary. You could throw an exception in a host call on the // slow path. That'll route us to lookupExceptionHandler(), which unwinds starting // with the call site index of our frame. Bad things happen if it's not set. jit.store32( CCallHelpers::TrustedImm32(callSiteIndex.bits()), CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount))); CallFrameShuffleData shuffleData; shuffleData.numLocals = state->jitCode->common.frameRegisterCount; shuffleData.callee = ValueRecovery::inGPR(GPRInfo::regT0, DataFormatJS); for (unsigned i = 0; i < numArgs; ++i) shuffleData.args.append(params[1 + i].recoveryForJSValue()); shuffleData.numPassedArgs = numArgs; shuffleData.setupCalleeSaveRegisters(jit.codeBlock()); CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo(); CCallHelpers::DataLabelPtr targetToCheck; CCallHelpers::Jump slowPath = jit.branchPtrWithPatch( CCallHelpers::NotEqual, GPRInfo::regT0, targetToCheck, CCallHelpers::TrustedImmPtr(0)); callLinkInfo->setFrameShuffleData(shuffleData); CallFrameShuffler(jit, shuffleData).prepareForTailCall(); CCallHelpers::Call fastCall = jit.nearTailCall(); slowPath.link(&jit); CallFrameShuffler slowPathShuffler(jit, shuffleData); slowPathShuffler.setCalleeJSValueRegs(JSValueRegs(GPRInfo::regT0)); slowPathShuffler.prepareForSlowPath(); jit.move(CCallHelpers::TrustedImmPtr(callLinkInfo), GPRInfo::regT2); CCallHelpers::Call slowCall = jit.nearCall(); jit.abortWithReason(JITDidReturnFromTailCall); callLinkInfo->setUpCall(CallLinkInfo::TailCall, codeOrigin, GPRInfo::regT0); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { MacroAssemblerCodePtr linkCall = linkBuffer.vm().getCTIStub(linkCallThunkGenerator).code(); linkBuffer.link(slowCall, FunctionPtr(linkCall.executableAddress())); callLinkInfo->setCallLocations( CodeLocationLabel(linkBuffer.locationOfNearCall(slowCall)), CodeLocationLabel(linkBuffer.locationOf(targetToCheck)), linkBuffer.locationOfNearCall(fastCall)); }); }); } void compileCallOrConstructVarargsSpread() { Node* node = m_node; LValue jsCallee = lowJSValue(m_node->child1()); LValue thisArg = lowJSValue(m_node->child2()); RELEASE_ASSERT(node->child3()->op() == PhantomNewArrayWithSpread); Node* arrayWithSpread = node->child3().node(); BitVector* bitVector = arrayWithSpread->bitVector(); unsigned numNonSpreadParameters = 0; Vector spreadLengths; Vector patchpointArguments; HashMap::Hash, WTF::NullableHashTraits> cachedSpreadLengths; for (unsigned i = 0; i < arrayWithSpread->numChildren(); i++) { if (bitVector->get(i)) { Node* spread = m_graph.varArgChild(arrayWithSpread, i).node(); RELEASE_ASSERT(spread->op() == PhantomSpread); RELEASE_ASSERT(spread->child1()->op() == PhantomCreateRest); InlineCallFrame* inlineCallFrame = spread->child1()->origin.semantic.inlineCallFrame; unsigned numberOfArgumentsToSkip = spread->child1()->numberOfArgumentsToSkip(); LValue length = cachedSpreadLengths.ensure(inlineCallFrame, [&] () { return m_out.zeroExtPtr(getSpreadLengthFromInlineCallFrame(inlineCallFrame, numberOfArgumentsToSkip)); }).iterator->value; patchpointArguments.append(length); spreadLengths.append(length); } else { ++numNonSpreadParameters; LValue argument = lowJSValue(m_graph.varArgChild(arrayWithSpread, i)); patchpointArguments.append(argument); } } LValue argumentCountIncludingThis = m_out.constIntPtr(numNonSpreadParameters + 1); for (LValue length : spreadLengths) argumentCountIncludingThis = m_out.add(length, argumentCountIncludingThis); PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->append(jsCallee, ValueRep::reg(GPRInfo::regT0)); patchpoint->append(thisArg, ValueRep::WarmAny); patchpoint->append(argumentCountIncludingThis, ValueRep::WarmAny); patchpoint->appendVectorWithRep(patchpointArguments, ValueRep::WarmAny); patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister)); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->clobber(RegisterSet::volatileRegistersForJSCall()); // No inputs will be in a volatile register. patchpoint->resultConstraint = ValueRep::reg(GPRInfo::returnValueGPR); patchpoint->numGPScratchRegisters = 0; // This is the minimum amount of call arg area stack space that all JS->JS calls always have. unsigned minimumJSCallAreaSize = sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(EncodedJSValue)); m_proc.requestCallArgAreaSizeInBytes(minimumJSCallAreaSize); CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite(); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex); jit.store32( CCallHelpers::TrustedImm32(callSiteIndex.bits()), CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount))); CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo(); RegisterSet usedRegisters = RegisterSet::allRegisters(); usedRegisters.exclude(RegisterSet::volatileRegistersForJSCall()); GPRReg calleeGPR = params[1].gpr(); usedRegisters.set(calleeGPR); ScratchRegisterAllocator allocator(usedRegisters); GPRReg scratchGPR1 = allocator.allocateScratchGPR(); GPRReg scratchGPR2 = allocator.allocateScratchGPR(); GPRReg scratchGPR3 = allocator.allocateScratchGPR(); GPRReg scratchGPR4 = allocator.allocateScratchGPR(); RELEASE_ASSERT(!allocator.numberOfReusedRegisters()); auto getValueFromRep = [&] (B3::ValueRep rep, GPRReg result) { ASSERT(!usedRegisters.get(result)); if (rep.isConstant()) { jit.move(CCallHelpers::Imm64(rep.value()), result); return; } // Note: in this function, we only request 64 bit values. if (rep.isStack()) { jit.load64( CCallHelpers::Address(GPRInfo::callFrameRegister, rep.offsetFromFP()), result); return; } RELEASE_ASSERT(rep.isGPR()); ASSERT(usedRegisters.get(rep.gpr())); jit.move(rep.gpr(), result); }; auto callWithExceptionCheck = [&] (void* callee) { jit.move(CCallHelpers::TrustedImmPtr(callee), GPRInfo::nonPreservedNonArgumentGPR); jit.call(GPRInfo::nonPreservedNonArgumentGPR); exceptions->append(jit.emitExceptionCheck(AssemblyHelpers::NormalExceptionCheck, AssemblyHelpers::FarJumpWidth)); }; CCallHelpers::JumpList slowCase; unsigned originalStackHeight = params.proc().frameSize(); { unsigned numUsedSlots = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), originalStackHeight / sizeof(EncodedJSValue)); B3::ValueRep argumentCountIncludingThisRep = params[3]; getValueFromRep(argumentCountIncludingThisRep, scratchGPR2); slowCase.append(jit.branch32(CCallHelpers::Above, scratchGPR2, CCallHelpers::TrustedImm32(JSC::maxArguments + 1))); jit.move(scratchGPR2, scratchGPR1); jit.addPtr(CCallHelpers::TrustedImmPtr(static_cast(numUsedSlots + CallFrame::headerSizeInRegisters)), scratchGPR1); // scratchGPR1 now has the required frame size in Register units // Round scratchGPR1 to next multiple of stackAlignmentRegisters() jit.addPtr(CCallHelpers::TrustedImm32(stackAlignmentRegisters() - 1), scratchGPR1); jit.andPtr(CCallHelpers::TrustedImm32(~(stackAlignmentRegisters() - 1)), scratchGPR1); jit.negPtr(scratchGPR1); jit.lshiftPtr(CCallHelpers::Imm32(3), scratchGPR1); jit.addPtr(GPRInfo::callFrameRegister, scratchGPR1); // Before touching stack values, we should update the stack pointer to protect them from signal stack. jit.addPtr(CCallHelpers::TrustedImm32(sizeof(CallerFrameAndPC)), scratchGPR1, CCallHelpers::stackPointerRegister); jit.store32(scratchGPR2, CCallHelpers::Address(scratchGPR1, CallFrameSlot::argumentCount * static_cast(sizeof(Register)) + PayloadOffset)); int storeOffset = CallFrame::thisArgumentOffset() * static_cast(sizeof(Register)); for (unsigned i = arrayWithSpread->numChildren(); i--; ) { unsigned paramsOffset = 4; if (bitVector->get(i)) { Node* spread = state->graph.varArgChild(arrayWithSpread, i).node(); RELEASE_ASSERT(spread->op() == PhantomSpread); RELEASE_ASSERT(spread->child1()->op() == PhantomCreateRest); InlineCallFrame* inlineCallFrame = spread->child1()->origin.semantic.inlineCallFrame; unsigned numberOfArgumentsToSkip = spread->child1()->numberOfArgumentsToSkip(); B3::ValueRep numArgumentsToCopy = params[paramsOffset + i]; getValueFromRep(numArgumentsToCopy, scratchGPR3); int loadOffset = (AssemblyHelpers::argumentsStart(inlineCallFrame).offset() + numberOfArgumentsToSkip) * static_cast(sizeof(Register)); auto done = jit.branchTestPtr(MacroAssembler::Zero, scratchGPR3); auto loopStart = jit.label(); jit.subPtr(CCallHelpers::TrustedImmPtr(static_cast(1)), scratchGPR3); jit.subPtr(CCallHelpers::TrustedImmPtr(static_cast(1)), scratchGPR2); jit.load64(CCallHelpers::BaseIndex(GPRInfo::callFrameRegister, scratchGPR3, CCallHelpers::TimesEight, loadOffset), scratchGPR4); jit.store64(scratchGPR4, CCallHelpers::BaseIndex(scratchGPR1, scratchGPR2, CCallHelpers::TimesEight, storeOffset)); jit.branchTestPtr(CCallHelpers::NonZero, scratchGPR3).linkTo(loopStart, &jit); done.link(&jit); } else { jit.subPtr(CCallHelpers::TrustedImmPtr(static_cast(1)), scratchGPR2); getValueFromRep(params[paramsOffset + i], scratchGPR3); jit.store64(scratchGPR3, CCallHelpers::BaseIndex(scratchGPR1, scratchGPR2, CCallHelpers::TimesEight, storeOffset)); } } } { CCallHelpers::Jump dontThrow = jit.jump(); slowCase.link(&jit); jit.setupArgumentsExecState(); callWithExceptionCheck(bitwise_cast(operationThrowStackOverflowForVarargs)); jit.abortWithReason(DFGVarargsThrowingPathDidNotThrow); dontThrow.link(&jit); } ASSERT(calleeGPR == GPRInfo::regT0); jit.store64(calleeGPR, CCallHelpers::calleeFrameSlot(CallFrameSlot::callee)); getValueFromRep(params[2], scratchGPR3); jit.store64(scratchGPR3, CCallHelpers::calleeArgumentSlot(0)); CallLinkInfo::CallType callType; if (node->op() == ConstructVarargs || node->op() == ConstructForwardVarargs) callType = CallLinkInfo::ConstructVarargs; else if (node->op() == TailCallVarargs || node->op() == TailCallForwardVarargs) callType = CallLinkInfo::TailCallVarargs; else callType = CallLinkInfo::CallVarargs; bool isTailCall = CallLinkInfo::callModeFor(callType) == CallMode::Tail; CCallHelpers::DataLabelPtr targetToCheck; CCallHelpers::Jump slowPath = jit.branchPtrWithPatch( CCallHelpers::NotEqual, GPRInfo::regT0, targetToCheck, CCallHelpers::TrustedImmPtr(nullptr)); CCallHelpers::Call fastCall; CCallHelpers::Jump done; if (isTailCall) { jit.emitRestoreCalleeSaves(); jit.prepareForTailCallSlow(); fastCall = jit.nearTailCall(); } else { fastCall = jit.nearCall(); done = jit.jump(); } slowPath.link(&jit); if (isTailCall) jit.emitRestoreCalleeSaves(); ASSERT(!usedRegisters.get(GPRInfo::regT2)); jit.move(CCallHelpers::TrustedImmPtr(callLinkInfo), GPRInfo::regT2); CCallHelpers::Call slowCall = jit.nearCall(); if (isTailCall) jit.abortWithReason(JITDidReturnFromTailCall); else done.link(&jit); callLinkInfo->setUpCall(callType, node->origin.semantic, GPRInfo::regT0); jit.addPtr( CCallHelpers::TrustedImm32(-originalStackHeight), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { MacroAssemblerCodePtr linkCall = linkBuffer.vm().getCTIStub(linkCallThunkGenerator).code(); linkBuffer.link(slowCall, FunctionPtr(linkCall.executableAddress())); callLinkInfo->setCallLocations( CodeLocationLabel(linkBuffer.locationOfNearCall(slowCall)), CodeLocationLabel(linkBuffer.locationOf(targetToCheck)), linkBuffer.locationOfNearCall(fastCall)); }); }); switch (node->op()) { case TailCallForwardVarargs: m_out.unreachable(); break; default: setJSValue(patchpoint); break; } } void compileCallOrConstructVarargs() { Node* node = m_node; LValue jsCallee = lowJSValue(m_node->child1()); LValue thisArg = lowJSValue(m_node->child2()); LValue jsArguments = nullptr; bool forwarding = false; switch (node->op()) { case CallVarargs: case TailCallVarargs: case TailCallVarargsInlinedCaller: case ConstructVarargs: jsArguments = lowJSValue(node->child3()); break; case CallForwardVarargs: case TailCallForwardVarargs: case TailCallForwardVarargsInlinedCaller: case ConstructForwardVarargs: forwarding = true; break; default: DFG_CRASH(m_graph, node, "bad node type"); break; } if (forwarding && m_node->child3() && m_node->child3()->op() == PhantomNewArrayWithSpread) { compileCallOrConstructVarargsSpread(); return; } PatchpointValue* patchpoint = m_out.patchpoint(Int64); // Append the forms of the arguments that we will use before any clobbering happens. patchpoint->append(jsCallee, ValueRep::reg(GPRInfo::regT0)); if (jsArguments) patchpoint->appendSomeRegister(jsArguments); patchpoint->appendSomeRegister(thisArg); if (!forwarding) { // Now append them again for after clobbering. Note that the compiler may ask us to use a // different register for the late for the post-clobbering version of the value. This gives // the compiler a chance to spill these values without having to burn any callee-saves. patchpoint->append(jsCallee, ValueRep::LateColdAny); patchpoint->append(jsArguments, ValueRep::LateColdAny); patchpoint->append(thisArg, ValueRep::LateColdAny); } RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister)); patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall()); patchpoint->resultConstraint = ValueRep::reg(GPRInfo::returnValueGPR); // This is the minimum amount of call arg area stack space that all JS->JS calls always have. unsigned minimumJSCallAreaSize = sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(EncodedJSValue)); m_proc.requestCallArgAreaSizeInBytes(minimumJSCallAreaSize); CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite(); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex); jit.store32( CCallHelpers::TrustedImm32(callSiteIndex.bits()), CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount))); CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo(); CallVarargsData* data = node->callVarargsData(); unsigned argIndex = 1; GPRReg calleeGPR = params[argIndex++].gpr(); ASSERT(calleeGPR == GPRInfo::regT0); GPRReg argumentsGPR = jsArguments ? params[argIndex++].gpr() : InvalidGPRReg; GPRReg thisGPR = params[argIndex++].gpr(); B3::ValueRep calleeLateRep; B3::ValueRep argumentsLateRep; B3::ValueRep thisLateRep; if (!forwarding) { // If we're not forwarding then we'll need callee, arguments, and this after we // have potentially clobbered calleeGPR, argumentsGPR, and thisGPR. Our technique // for this is to supply all of those operands as late uses in addition to // specifying them as early uses. It's possible that the late use uses a spill // while the early use uses a register, and it's possible for the late and early // uses to use different registers. We do know that the late uses interfere with // all volatile registers and so won't use those, but the early uses may use // volatile registers and in the case of calleeGPR, it's pinned to regT0 so it // definitely will. // // Note that we have to be super careful with these. It's possible that these // use a shuffling of the registers used for calleeGPR, argumentsGPR, and // thisGPR. If that happens and we do for example: // // calleeLateRep.emitRestore(jit, calleeGPR); // argumentsLateRep.emitRestore(jit, calleeGPR); // // Then we might end up with garbage if calleeLateRep.gpr() == argumentsGPR and // argumentsLateRep.gpr() == calleeGPR. // // We do a variety of things to prevent this from happening. For example, we use // argumentsLateRep before needing the other two and after we've already stopped // using the *GPRs. Also, we pin calleeGPR to regT0, and rely on the fact that // the *LateReps cannot use volatile registers (so they cannot be regT0, so // calleeGPR != argumentsLateRep.gpr() and calleeGPR != thisLateRep.gpr()). // // An alternative would have been to just use early uses and early-clobber all // volatile registers. But that would force callee, arguments, and this into // callee-save registers even if we have to spill them. We don't want spilling to // use up three callee-saves. // // TL;DR: The way we use LateReps here is dangerous and barely works but achieves // some desirable performance properties, so don't mistake the cleverness for // elegance. calleeLateRep = params[argIndex++]; argumentsLateRep = params[argIndex++]; thisLateRep = params[argIndex++]; } // Get some scratch registers. RegisterSet usedRegisters; usedRegisters.merge(RegisterSet::stackRegisters()); usedRegisters.merge(RegisterSet::reservedHardwareRegisters()); usedRegisters.merge(RegisterSet::calleeSaveRegisters()); usedRegisters.set(calleeGPR); if (argumentsGPR != InvalidGPRReg) usedRegisters.set(argumentsGPR); usedRegisters.set(thisGPR); if (calleeLateRep.isReg()) usedRegisters.set(calleeLateRep.reg()); if (argumentsLateRep.isReg()) usedRegisters.set(argumentsLateRep.reg()); if (thisLateRep.isReg()) usedRegisters.set(thisLateRep.reg()); ScratchRegisterAllocator allocator(usedRegisters); GPRReg scratchGPR1 = allocator.allocateScratchGPR(); GPRReg scratchGPR2 = allocator.allocateScratchGPR(); GPRReg scratchGPR3 = forwarding ? allocator.allocateScratchGPR() : InvalidGPRReg; RELEASE_ASSERT(!allocator.numberOfReusedRegisters()); auto callWithExceptionCheck = [&] (void* callee) { jit.move(CCallHelpers::TrustedImmPtr(callee), GPRInfo::nonPreservedNonArgumentGPR); jit.call(GPRInfo::nonPreservedNonArgumentGPR); exceptions->append(jit.emitExceptionCheck(AssemblyHelpers::NormalExceptionCheck, AssemblyHelpers::FarJumpWidth)); }; unsigned originalStackHeight = params.proc().frameSize(); if (forwarding) { jit.move(CCallHelpers::TrustedImm32(originalStackHeight / sizeof(EncodedJSValue)), scratchGPR2); CCallHelpers::JumpList slowCase; InlineCallFrame* inlineCallFrame; if (node->child3()) inlineCallFrame = node->child3()->origin.semantic.inlineCallFrame; else inlineCallFrame = node->origin.semantic.inlineCallFrame; // emitSetupVarargsFrameFastCase modifies the stack pointer if it succeeds. emitSetupVarargsFrameFastCase(jit, scratchGPR2, scratchGPR1, scratchGPR2, scratchGPR3, inlineCallFrame, data->firstVarArgOffset, slowCase); CCallHelpers::Jump done = jit.jump(); slowCase.link(&jit); jit.setupArgumentsExecState(); callWithExceptionCheck(bitwise_cast(operationThrowStackOverflowForVarargs)); jit.abortWithReason(DFGVarargsThrowingPathDidNotThrow); done.link(&jit); } else { jit.move(CCallHelpers::TrustedImm32(originalStackHeight / sizeof(EncodedJSValue)), scratchGPR1); jit.setupArgumentsWithExecState(argumentsGPR, scratchGPR1, CCallHelpers::TrustedImm32(data->firstVarArgOffset)); callWithExceptionCheck(bitwise_cast(operationSizeFrameForVarargs)); jit.move(GPRInfo::returnValueGPR, scratchGPR1); jit.move(CCallHelpers::TrustedImm32(originalStackHeight / sizeof(EncodedJSValue)), scratchGPR2); argumentsLateRep.emitRestore(jit, argumentsGPR); emitSetVarargsFrame(jit, scratchGPR1, false, scratchGPR2, scratchGPR2); jit.addPtr(CCallHelpers::TrustedImm32(-minimumJSCallAreaSize), scratchGPR2, CCallHelpers::stackPointerRegister); jit.setupArgumentsWithExecState(scratchGPR2, argumentsGPR, CCallHelpers::TrustedImm32(data->firstVarArgOffset), scratchGPR1); callWithExceptionCheck(bitwise_cast(operationSetupVarargsFrame)); jit.addPtr(CCallHelpers::TrustedImm32(sizeof(CallerFrameAndPC)), GPRInfo::returnValueGPR, CCallHelpers::stackPointerRegister); calleeLateRep.emitRestore(jit, GPRInfo::regT0); // This may not emit code if thisGPR got a callee-save. Also, we're guaranteed // that thisGPR != GPRInfo::regT0 because regT0 interferes with it. thisLateRep.emitRestore(jit, thisGPR); } jit.store64(GPRInfo::regT0, CCallHelpers::calleeFrameSlot(CallFrameSlot::callee)); jit.store64(thisGPR, CCallHelpers::calleeArgumentSlot(0)); CallLinkInfo::CallType callType; if (node->op() == ConstructVarargs || node->op() == ConstructForwardVarargs) callType = CallLinkInfo::ConstructVarargs; else if (node->op() == TailCallVarargs || node->op() == TailCallForwardVarargs) callType = CallLinkInfo::TailCallVarargs; else callType = CallLinkInfo::CallVarargs; bool isTailCall = CallLinkInfo::callModeFor(callType) == CallMode::Tail; CCallHelpers::DataLabelPtr targetToCheck; CCallHelpers::Jump slowPath = jit.branchPtrWithPatch( CCallHelpers::NotEqual, GPRInfo::regT0, targetToCheck, CCallHelpers::TrustedImmPtr(0)); CCallHelpers::Call fastCall; CCallHelpers::Jump done; if (isTailCall) { jit.emitRestoreCalleeSaves(); jit.prepareForTailCallSlow(); fastCall = jit.nearTailCall(); } else { fastCall = jit.nearCall(); done = jit.jump(); } slowPath.link(&jit); if (isTailCall) jit.emitRestoreCalleeSaves(); jit.move(CCallHelpers::TrustedImmPtr(callLinkInfo), GPRInfo::regT2); CCallHelpers::Call slowCall = jit.nearCall(); if (isTailCall) jit.abortWithReason(JITDidReturnFromTailCall); else done.link(&jit); callLinkInfo->setUpCall(callType, node->origin.semantic, GPRInfo::regT0); jit.addPtr( CCallHelpers::TrustedImm32(-originalStackHeight), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { MacroAssemblerCodePtr linkCall = linkBuffer.vm().getCTIStub(linkCallThunkGenerator).code(); linkBuffer.link(slowCall, FunctionPtr(linkCall.executableAddress())); callLinkInfo->setCallLocations( CodeLocationLabel(linkBuffer.locationOfNearCall(slowCall)), CodeLocationLabel(linkBuffer.locationOf(targetToCheck)), linkBuffer.locationOfNearCall(fastCall)); }); }); switch (node->op()) { case TailCallVarargs: case TailCallForwardVarargs: m_out.unreachable(); break; default: setJSValue(patchpoint); break; } } void compileCallEval() { Node* node = m_node; unsigned numArgs = node->numChildren() - 1; LValue jsCallee = lowJSValue(m_graph.varArgChild(node, 0)); unsigned frameSize = (CallFrame::headerSizeInRegisters + numArgs) * sizeof(EncodedJSValue); unsigned alignedFrameSize = WTF::roundUpToMultipleOf(stackAlignmentBytes(), frameSize); m_proc.requestCallArgAreaSizeInBytes(alignedFrameSize); Vector arguments; arguments.append(ConstrainedValue(jsCallee, ValueRep::reg(GPRInfo::regT0))); auto addArgument = [&] (LValue value, VirtualRegister reg, int offset) { intptr_t offsetFromSP = (reg.offset() - CallerFrameAndPC::sizeInRegisters) * sizeof(EncodedJSValue) + offset; arguments.append(ConstrainedValue(value, ValueRep::stackArgument(offsetFromSP))); }; addArgument(jsCallee, VirtualRegister(CallFrameSlot::callee), 0); addArgument(m_out.constInt32(numArgs), VirtualRegister(CallFrameSlot::argumentCount), PayloadOffset); for (unsigned i = 0; i < numArgs; ++i) addArgument(lowJSValue(m_graph.varArgChild(node, 1 + i)), virtualRegisterForArgument(i), 0); PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendVector(arguments); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister)); patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->clobberLate(RegisterSet::volatileRegistersForJSCall()); patchpoint->resultConstraint = ValueRep::reg(GPRInfo::returnValueGPR); CodeOrigin codeOrigin = codeOriginDescriptionOfCallSite(); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(codeOrigin); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex); jit.store32( CCallHelpers::TrustedImm32(callSiteIndex.bits()), CCallHelpers::tagFor(VirtualRegister(CallFrameSlot::argumentCount))); CallLinkInfo* callLinkInfo = jit.codeBlock()->addCallLinkInfo(); callLinkInfo->setUpCall(CallLinkInfo::Call, node->origin.semantic, GPRInfo::regT0); jit.addPtr(CCallHelpers::TrustedImm32(-static_cast(sizeof(CallerFrameAndPC))), CCallHelpers::stackPointerRegister, GPRInfo::regT1); jit.storePtr(GPRInfo::callFrameRegister, CCallHelpers::Address(GPRInfo::regT1, CallFrame::callerFrameOffset())); // Now we need to make room for: // - The caller frame and PC for a call to operationCallEval. // - Potentially two arguments on the stack. unsigned requiredBytes = sizeof(CallerFrameAndPC) + sizeof(ExecState*) * 2; requiredBytes = WTF::roundUpToMultipleOf(stackAlignmentBytes(), requiredBytes); jit.subPtr(CCallHelpers::TrustedImm32(requiredBytes), CCallHelpers::stackPointerRegister); jit.setupArgumentsWithExecState(GPRInfo::regT1); jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast(operationCallEval)), GPRInfo::nonPreservedNonArgumentGPR); jit.call(GPRInfo::nonPreservedNonArgumentGPR); exceptions->append(jit.emitExceptionCheck(AssemblyHelpers::NormalExceptionCheck, AssemblyHelpers::FarJumpWidth)); CCallHelpers::Jump done = jit.branchTest64(CCallHelpers::NonZero, GPRInfo::returnValueGPR); jit.addPtr(CCallHelpers::TrustedImm32(requiredBytes), CCallHelpers::stackPointerRegister); jit.load64(CCallHelpers::calleeFrameSlot(CallFrameSlot::callee), GPRInfo::regT0); jit.emitDumbVirtualCall(callLinkInfo); done.link(&jit); jit.addPtr( CCallHelpers::TrustedImm32(-params.proc().frameSize()), GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister); }); setJSValue(patchpoint); } void compileLoadVarargs() { LoadVarargsData* data = m_node->loadVarargsData(); LValue jsArguments = lowJSValue(m_node->child1()); LValue length = vmCall( Int32, m_out.operation(operationSizeOfVarargs), m_callFrame, jsArguments, m_out.constInt32(data->offset)); // FIXME: There is a chance that we will call an effectful length property twice. This is safe // from the standpoint of the VM's integrity, but it's subtly wrong from a spec compliance // standpoint. The best solution would be one where we can exit *into* the op_call_varargs right // past the sizing. // https://bugs.webkit.org/show_bug.cgi?id=141448 LValue lengthIncludingThis = m_out.add(length, m_out.int32One); speculate( VarargsOverflow, noValue(), nullptr, m_out.above(lengthIncludingThis, m_out.constInt32(data->limit))); m_out.store32(lengthIncludingThis, payloadFor(data->machineCount)); // FIXME: This computation is rather silly. If operationLaodVarargs just took a pointer instead // of a VirtualRegister, we wouldn't have to do this. // https://bugs.webkit.org/show_bug.cgi?id=141660 LValue machineStart = m_out.lShr( m_out.sub(addressFor(data->machineStart.offset()).value(), m_callFrame), m_out.constIntPtr(3)); vmCall( Void, m_out.operation(operationLoadVarargs), m_callFrame, m_out.castToInt32(machineStart), jsArguments, m_out.constInt32(data->offset), length, m_out.constInt32(data->mandatoryMinimum)); } void compileForwardVarargs() { if (m_node->child1() && m_node->child1()->op() == PhantomNewArrayWithSpread) { compileForwardVarargsWithSpread(); return; } LoadVarargsData* data = m_node->loadVarargsData(); InlineCallFrame* inlineCallFrame; if (m_node->child1()) inlineCallFrame = m_node->child1()->origin.semantic.inlineCallFrame; else inlineCallFrame = m_node->origin.semantic.inlineCallFrame; LValue length = nullptr; LValue lengthIncludingThis = nullptr; ArgumentsLength argumentsLength = getArgumentsLength(inlineCallFrame); if (argumentsLength.isKnown) { unsigned knownLength = argumentsLength.known; if (knownLength >= data->offset) knownLength = knownLength - data->offset; else knownLength = 0; length = m_out.constInt32(knownLength); lengthIncludingThis = m_out.constInt32(knownLength + 1); } else { // We need to perform the same logical operation as the code above, but through dynamic operations. if (!data->offset) length = argumentsLength.value; else { LBasicBlock isLarger = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock smallerOrEqualLengthResult = m_out.anchor(m_out.constInt32(0)); m_out.branch( m_out.above(argumentsLength.value, m_out.constInt32(data->offset)), unsure(isLarger), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isLarger, continuation); ValueFromBlock largerLengthResult = m_out.anchor(m_out.sub(argumentsLength.value, m_out.constInt32(data->offset))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); length = m_out.phi(Int32, smallerOrEqualLengthResult, largerLengthResult); } lengthIncludingThis = m_out.add(length, m_out.constInt32(1)); } speculate( VarargsOverflow, noValue(), nullptr, m_out.above(lengthIncludingThis, m_out.constInt32(data->limit))); m_out.store32(lengthIncludingThis, payloadFor(data->machineCount)); unsigned numberOfArgumentsToSkip = data->offset; LValue sourceStart = getArgumentsStart(inlineCallFrame, numberOfArgumentsToSkip); LValue targetStart = addressFor(data->machineStart).value(); LBasicBlock undefinedLoop = m_out.newBlock(); LBasicBlock mainLoopEntry = m_out.newBlock(); LBasicBlock mainLoop = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue lengthAsPtr = m_out.zeroExtPtr(length); LValue loopBoundValue = m_out.constIntPtr(data->mandatoryMinimum); ValueFromBlock loopBound = m_out.anchor(loopBoundValue); m_out.branch( m_out.above(loopBoundValue, lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry)); LBasicBlock lastNext = m_out.appendTo(undefinedLoop, mainLoopEntry); LValue previousIndex = m_out.phi(pointerType(), loopBound); LValue currentIndex = m_out.sub(previousIndex, m_out.intPtrOne); m_out.store64( m_out.constInt64(JSValue::encode(jsUndefined())), m_out.baseIndex(m_heaps.variables, targetStart, currentIndex)); ValueFromBlock nextIndex = m_out.anchor(currentIndex); m_out.addIncomingToPhi(previousIndex, nextIndex); m_out.branch( m_out.above(currentIndex, lengthAsPtr), unsure(undefinedLoop), unsure(mainLoopEntry)); m_out.appendTo(mainLoopEntry, mainLoop); loopBound = m_out.anchor(lengthAsPtr); m_out.branch(m_out.notNull(lengthAsPtr), unsure(mainLoop), unsure(continuation)); m_out.appendTo(mainLoop, continuation); previousIndex = m_out.phi(pointerType(), loopBound); currentIndex = m_out.sub(previousIndex, m_out.intPtrOne); LValue value = m_out.load64( m_out.baseIndex(m_heaps.variables, sourceStart, currentIndex)); m_out.store64(value, m_out.baseIndex(m_heaps.variables, targetStart, currentIndex)); nextIndex = m_out.anchor(currentIndex); m_out.addIncomingToPhi(previousIndex, nextIndex); m_out.branch(m_out.isNull(currentIndex), unsure(continuation), unsure(mainLoop)); m_out.appendTo(continuation, lastNext); } LValue getSpreadLengthFromInlineCallFrame(InlineCallFrame* inlineCallFrame, unsigned numberOfArgumentsToSkip) { ArgumentsLength argumentsLength = getArgumentsLength(inlineCallFrame); if (argumentsLength.isKnown) { unsigned knownLength = argumentsLength.known; if (knownLength >= numberOfArgumentsToSkip) knownLength = knownLength - numberOfArgumentsToSkip; else knownLength = 0; return m_out.constInt32(knownLength); } // We need to perform the same logical operation as the code above, but through dynamic operations. if (!numberOfArgumentsToSkip) return argumentsLength.value; LBasicBlock isLarger = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock smallerOrEqualLengthResult = m_out.anchor(m_out.constInt32(0)); m_out.branch( m_out.above(argumentsLength.value, m_out.constInt32(numberOfArgumentsToSkip)), unsure(isLarger), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isLarger, continuation); ValueFromBlock largerLengthResult = m_out.anchor(m_out.sub(argumentsLength.value, m_out.constInt32(numberOfArgumentsToSkip))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int32, smallerOrEqualLengthResult, largerLengthResult); } void compileForwardVarargsWithSpread() { HashMap::Hash, WTF::NullableHashTraits> cachedSpreadLengths; Node* arrayWithSpread = m_node->child1().node(); RELEASE_ASSERT(arrayWithSpread->op() == PhantomNewArrayWithSpread); BitVector* bitVector = arrayWithSpread->bitVector(); unsigned numberOfStaticArguments = 0; Vector spreadLengths; for (unsigned i = 0; i < arrayWithSpread->numChildren(); i++) { if (bitVector->get(i)) { Node* child = m_graph.varArgChild(arrayWithSpread, i).node(); ASSERT(child->op() == PhantomSpread); ASSERT(child->child1()->op() == PhantomCreateRest); InlineCallFrame* inlineCallFrame = child->child1()->origin.semantic.inlineCallFrame; LValue length = cachedSpreadLengths.ensure(inlineCallFrame, [&] () { return getSpreadLengthFromInlineCallFrame(inlineCallFrame, child->child1()->numberOfArgumentsToSkip()); }).iterator->value; spreadLengths.append(length); } else ++numberOfStaticArguments; } LValue lengthIncludingThis = m_out.constInt32(1 + numberOfStaticArguments); for (LValue length : spreadLengths) lengthIncludingThis = m_out.add(lengthIncludingThis, length); LoadVarargsData* data = m_node->loadVarargsData(); speculate( VarargsOverflow, noValue(), nullptr, m_out.above(lengthIncludingThis, m_out.constInt32(data->limit))); m_out.store32(lengthIncludingThis, payloadFor(data->machineCount)); LValue targetStart = addressFor(data->machineStart).value(); LValue storeIndex = m_out.constIntPtr(0); for (unsigned i = 0; i < arrayWithSpread->numChildren(); i++) { if (bitVector->get(i)) { Node* child = m_graph.varArgChild(arrayWithSpread, i).node(); RELEASE_ASSERT(child->op() == PhantomSpread); RELEASE_ASSERT(child->child1()->op() == PhantomCreateRest); InlineCallFrame* inlineCallFrame = child->child1()->origin.semantic.inlineCallFrame; LValue sourceStart = getArgumentsStart(inlineCallFrame, child->child1()->numberOfArgumentsToSkip()); LValue spreadLength = m_out.zeroExtPtr(cachedSpreadLengths.get(inlineCallFrame)); LBasicBlock loop = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock startLoadIndex = m_out.anchor(m_out.constIntPtr(0)); ValueFromBlock startStoreIndex = m_out.anchor(storeIndex); ValueFromBlock startStoreIndexForEnd = m_out.anchor(storeIndex); m_out.branch(m_out.isZero64(spreadLength), unsure(continuation), unsure(loop)); LBasicBlock lastNext = m_out.appendTo(loop, continuation); LValue loopStoreIndex = m_out.phi(Int64, startStoreIndex); LValue loadIndex = m_out.phi(Int64, startLoadIndex); LValue value = m_out.load64( m_out.baseIndex(m_heaps.variables, sourceStart, loadIndex)); m_out.store64(value, m_out.baseIndex(m_heaps.variables, targetStart, loopStoreIndex)); LValue nextLoadIndex = m_out.add(m_out.constIntPtr(1), loadIndex); m_out.addIncomingToPhi(loadIndex, m_out.anchor(nextLoadIndex)); LValue nextStoreIndex = m_out.add(m_out.constIntPtr(1), loopStoreIndex); m_out.addIncomingToPhi(loopStoreIndex, m_out.anchor(nextStoreIndex)); ValueFromBlock loopStoreIndexForEnd = m_out.anchor(nextStoreIndex); m_out.branch(m_out.below(nextLoadIndex, spreadLength), unsure(loop), unsure(continuation)); m_out.appendTo(continuation, lastNext); storeIndex = m_out.phi(Int64, startStoreIndexForEnd, loopStoreIndexForEnd); } else { LValue value = lowJSValue(m_graph.varArgChild(arrayWithSpread, i)); m_out.store64(value, m_out.baseIndex(m_heaps.variables, targetStart, storeIndex)); storeIndex = m_out.add(m_out.constIntPtr(1), storeIndex); } } LBasicBlock undefinedLoop = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock startStoreIndex = m_out.anchor(storeIndex); LValue loopBoundValue = m_out.constIntPtr(data->mandatoryMinimum); m_out.branch(m_out.below(storeIndex, loopBoundValue), unsure(undefinedLoop), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(undefinedLoop, continuation); LValue loopStoreIndex = m_out.phi(Int64, startStoreIndex); m_out.store64( m_out.constInt64(JSValue::encode(jsUndefined())), m_out.baseIndex(m_heaps.variables, targetStart, loopStoreIndex)); LValue nextIndex = m_out.add(loopStoreIndex, m_out.constIntPtr(1)); m_out.addIncomingToPhi(loopStoreIndex, m_out.anchor(nextIndex)); m_out.branch( m_out.below(nextIndex, loopBoundValue), unsure(undefinedLoop), unsure(continuation)); m_out.appendTo(continuation, lastNext); } void compileJump() { m_out.jump(lowBlock(m_node->targetBlock())); } void compileBranch() { m_out.branch( boolify(m_node->child1()), WeightedTarget( lowBlock(m_node->branchData()->taken.block), m_node->branchData()->taken.count), WeightedTarget( lowBlock(m_node->branchData()->notTaken.block), m_node->branchData()->notTaken.count)); } void compileSwitch() { SwitchData* data = m_node->switchData(); switch (data->kind) { case SwitchImm: { Vector intValues; LBasicBlock switchOnInts = m_out.newBlock(); LBasicBlock lastNext = m_out.appendTo(m_out.m_block, switchOnInts); switch (m_node->child1().useKind()) { case Int32Use: { intValues.append(m_out.anchor(lowInt32(m_node->child1()))); m_out.jump(switchOnInts); break; } case UntypedUse: { LBasicBlock isInt = m_out.newBlock(); LBasicBlock isNotInt = m_out.newBlock(); LBasicBlock isDouble = m_out.newBlock(); LValue boxedValue = lowJSValue(m_node->child1()); m_out.branch(isNotInt32(boxedValue), unsure(isNotInt), unsure(isInt)); LBasicBlock innerLastNext = m_out.appendTo(isInt, isNotInt); intValues.append(m_out.anchor(unboxInt32(boxedValue))); m_out.jump(switchOnInts); m_out.appendTo(isNotInt, isDouble); m_out.branch( isCellOrMisc(boxedValue, provenType(m_node->child1())), usually(lowBlock(data->fallThrough.block)), rarely(isDouble)); m_out.appendTo(isDouble, innerLastNext); LValue doubleValue = unboxDouble(boxedValue); LValue intInDouble = m_out.doubleToInt(doubleValue); intValues.append(m_out.anchor(intInDouble)); m_out.branch( m_out.doubleEqual(m_out.intToDouble(intInDouble), doubleValue), unsure(switchOnInts), unsure(lowBlock(data->fallThrough.block))); break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } m_out.appendTo(switchOnInts, lastNext); buildSwitch(data, Int32, m_out.phi(Int32, intValues)); return; } case SwitchChar: { LValue stringValue; // FIXME: We should use something other than unsure() for the branch weight // of the fallThrough block. The main challenge is just that we have multiple // branches to fallThrough but a single count, so we would need to divvy it up // among the different lowered branches. // https://bugs.webkit.org/show_bug.cgi?id=129082 switch (m_node->child1().useKind()) { case StringUse: { stringValue = lowString(m_node->child1()); break; } case UntypedUse: { LValue unboxedValue = lowJSValue(m_node->child1()); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock isStringCase = m_out.newBlock(); m_out.branch( isNotCell(unboxedValue, provenType(m_node->child1())), unsure(lowBlock(data->fallThrough.block)), unsure(isCellCase)); LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase); LValue cellValue = unboxedValue; m_out.branch( isNotString(cellValue, provenType(m_node->child1())), unsure(lowBlock(data->fallThrough.block)), unsure(isStringCase)); m_out.appendTo(isStringCase, lastNext); stringValue = cellValue; break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); break; } LBasicBlock lengthIs1 = m_out.newBlock(); LBasicBlock needResolution = m_out.newBlock(); LBasicBlock resolved = m_out.newBlock(); LBasicBlock is8Bit = m_out.newBlock(); LBasicBlock is16Bit = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.notEqual( m_out.load32NonNegative(stringValue, m_heaps.JSString_length), m_out.int32One), unsure(lowBlock(data->fallThrough.block)), unsure(lengthIs1)); LBasicBlock lastNext = m_out.appendTo(lengthIs1, needResolution); Vector values; LValue fastValue = m_out.loadPtr(stringValue, m_heaps.JSString_value); values.append(m_out.anchor(fastValue)); m_out.branch(m_out.isNull(fastValue), rarely(needResolution), usually(resolved)); m_out.appendTo(needResolution, resolved); values.append(m_out.anchor( vmCall(pointerType(), m_out.operation(operationResolveRope), m_callFrame, stringValue))); m_out.jump(resolved); m_out.appendTo(resolved, is8Bit); LValue value = m_out.phi(pointerType(), values); LValue characterData = m_out.loadPtr(value, m_heaps.StringImpl_data); m_out.branch( m_out.testNonZero32( m_out.load32(value, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), unsure(is8Bit), unsure(is16Bit)); Vector characters; m_out.appendTo(is8Bit, is16Bit); characters.append(m_out.anchor(m_out.load8ZeroExt32(characterData, m_heaps.characters8[0]))); m_out.jump(continuation); m_out.appendTo(is16Bit, continuation); characters.append(m_out.anchor(m_out.load16ZeroExt32(characterData, m_heaps.characters16[0]))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); buildSwitch(data, Int32, m_out.phi(Int32, characters)); return; } case SwitchString: { switch (m_node->child1().useKind()) { case StringIdentUse: { LValue stringImpl = lowStringIdent(m_node->child1()); Vector cases; for (unsigned i = 0; i < data->cases.size(); ++i) { LValue value = m_out.constIntPtr(data->cases[i].value.stringImpl()); LBasicBlock block = lowBlock(data->cases[i].target.block); Weight weight = Weight(data->cases[i].target.count); cases.append(SwitchCase(value, block, weight)); } m_out.switchInstruction( stringImpl, cases, lowBlock(data->fallThrough.block), Weight(data->fallThrough.count)); return; } case StringUse: { switchString(data, lowString(m_node->child1())); return; } case UntypedUse: { LValue value = lowJSValue(m_node->child1()); LBasicBlock isCellBlock = m_out.newBlock(); LBasicBlock isStringBlock = m_out.newBlock(); m_out.branch( isCell(value, provenType(m_node->child1())), unsure(isCellBlock), unsure(lowBlock(data->fallThrough.block))); LBasicBlock lastNext = m_out.appendTo(isCellBlock, isStringBlock); m_out.branch( isString(value, provenType(m_node->child1())), unsure(isStringBlock), unsure(lowBlock(data->fallThrough.block))); m_out.appendTo(isStringBlock, lastNext); switchString(data, value); return; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); return; } return; } case SwitchCell: { LValue cell; switch (m_node->child1().useKind()) { case CellUse: { cell = lowCell(m_node->child1()); break; } case UntypedUse: { LValue value = lowJSValue(m_node->child1()); LBasicBlock cellCase = m_out.newBlock(); m_out.branch( isCell(value, provenType(m_node->child1())), unsure(cellCase), unsure(lowBlock(data->fallThrough.block))); m_out.appendTo(cellCase); cell = value; break; } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); return; } buildSwitch(m_node->switchData(), pointerType(), cell); return; } } DFG_CRASH(m_graph, m_node, "Bad switch kind"); } void compileReturn() { m_out.ret(lowJSValue(m_node->child1())); } void compileForceOSRExit() { terminate(InadequateCoverage); } void compileThrow() { terminate(Uncountable); } void compileInvalidationPoint() { if (verboseCompilationEnabled()) dataLog(" Invalidation point with availability: ", availabilityMap(), "\n"); DFG_ASSERT(m_graph, m_node, m_origin.exitOK); PatchpointValue* patchpoint = m_out.patchpoint(Void); OSRExitDescriptor* descriptor = appendOSRExitDescriptor(noValue(), nullptr); NodeOrigin origin = m_origin; patchpoint->appendColdAnys(buildExitArguments(descriptor, origin.forExit, noValue())); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { // The MacroAssembler knows more about this than B3 does. The watchpointLabel() method // will ensure that this is followed by a nop shadow but only when this is actually // necessary. CCallHelpers::Label label = jit.watchpointLabel(); RefPtr handle = descriptor->emitOSRExitLater( *state, UncountableInvalidation, origin, params); RefPtr jitCode = state->jitCode.get(); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { JumpReplacement jumpReplacement( linkBuffer.locationOf(label), linkBuffer.locationOf(handle->label)); jitCode->common.jumpReplacements.append(jumpReplacement); }); }); // Set some obvious things. patchpoint->effects.terminal = false; patchpoint->effects.writesLocalState = false; patchpoint->effects.readsLocalState = false; // This is how we tell B3 about the possibility of jump replacement. patchpoint->effects.exitsSideways = true; // It's not possible for some prior branch to determine the safety of this operation. It's always // fine to execute this on some path that wouldn't have originally executed it before // optimization. patchpoint->effects.controlDependent = false; // If this falls through then it won't write anything. patchpoint->effects.writes = HeapRange(); // When this abruptly terminates, it could read any heap location. patchpoint->effects.reads = HeapRange::top(); } void compileIsEmpty() { setBoolean(m_out.isZero64(lowJSValue(m_node->child1()))); } void compileIsUndefined() { setBoolean(equalNullOrUndefined(m_node->child1(), AllCellsAreFalse, EqualUndefined)); } void compileIsBoolean() { setBoolean(isBoolean(lowJSValue(m_node->child1()), provenType(m_node->child1()))); } void compileIsNumber() { setBoolean(isNumber(lowJSValue(m_node->child1()), provenType(m_node->child1()))); } void compileIsCellWithType() { if (m_node->child1().useKind() == UntypedUse) { LValue value = lowJSValue(m_node->child1()); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); m_out.branch( isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation); ValueFromBlock cellResult = m_out.anchor(isCellWithType(value, m_node->queriedType(), m_node->speculatedTypeForQuery(), provenType(m_node->child1()))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, notCellResult, cellResult)); } else { ASSERT(m_node->child1().useKind() == CellUse); setBoolean(isCellWithType(lowCell(m_node->child1()), m_node->queriedType(), m_node->speculatedTypeForQuery(), provenType(m_node->child1()))); } } void compileIsObject() { LValue value = lowJSValue(m_node->child1()); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); m_out.branch( isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation); ValueFromBlock cellResult = m_out.anchor(isObject(value, provenType(m_node->child1()))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, notCellResult, cellResult)); } LValue wangsInt64Hash(LValue input) { // key += ~(key << 32); LValue key = input; LValue temp = key; temp = m_out.shl(temp, m_out.constInt32(32)); temp = m_out.bitNot(temp); key = m_out.add(key, temp); // key ^= (key >> 22); temp = key; temp = m_out.lShr(temp, m_out.constInt32(22)); key = m_out.bitXor(key, temp); // key += ~(key << 13); temp = key; temp = m_out.shl(temp, m_out.constInt32(13)); temp = m_out.bitNot(temp); key = m_out.add(key, temp); // key ^= (key >> 8); temp = key; temp = m_out.lShr(temp, m_out.constInt32(8)); key = m_out.bitXor(key, temp); // key += (key << 3); temp = key; temp = m_out.shl(temp, m_out.constInt32(3)); key = m_out.add(key, temp); // key ^= (key >> 15); temp = key; temp = m_out.lShr(temp, m_out.constInt32(15)); key = m_out.bitXor(key, temp); // key += ~(key << 27); temp = key; temp = m_out.shl(temp, m_out.constInt32(27)); temp = m_out.bitNot(temp); key = m_out.add(key, temp); // key ^= (key >> 31); temp = key; temp = m_out.lShr(temp, m_out.constInt32(31)); key = m_out.bitXor(key, temp); key = m_out.castToInt32(key); return key; } LValue mapHashString(LValue string) { LBasicBlock nonEmptyStringCase = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value); m_out.branch( m_out.equal(stringImpl, m_out.constIntPtr(0)), unsure(slowCase), unsure(nonEmptyStringCase)); LBasicBlock lastNext = m_out.appendTo(nonEmptyStringCase, slowCase); LValue hash = m_out.lShr(m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::s_flagCount)); ValueFromBlock nonEmptyStringHashResult = m_out.anchor(hash); m_out.branch(m_out.equal(hash, m_out.constInt32(0)), unsure(slowCase), unsure(continuation)); m_out.appendTo(slowCase, continuation); ValueFromBlock slowResult = m_out.anchor( vmCall(Int32, m_out.operation(operationMapHash), m_callFrame, string)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int32, slowResult, nonEmptyStringHashResult); } void compileMapHash() { switch (m_node->child1().useKind()) { case BooleanUse: case Int32Use: case SymbolUse: case ObjectUse: { LValue key = lowJSValue(m_node->child1(), ManualOperandSpeculation); speculate(m_node->child1()); setInt32(wangsInt64Hash(key)); return; } case CellUse: { LBasicBlock isString = m_out.newBlock(); LBasicBlock notString = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue value = lowCell(m_node->child1()); LValue isStringValue = m_out.equal(m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoType), m_out.constInt32(StringType)); m_out.branch( isStringValue, unsure(isString), unsure(notString)); LBasicBlock lastNext = m_out.appendTo(isString, notString); ValueFromBlock stringResult = m_out.anchor(mapHashString(value)); m_out.jump(continuation); m_out.appendTo(notString, continuation); ValueFromBlock notStringResult = m_out.anchor(wangsInt64Hash(value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setInt32(m_out.phi(Int32, stringResult, notStringResult)); return; } case StringUse: { LValue string = lowString(m_node->child1()); setInt32(mapHashString(string)); return; } default: RELEASE_ASSERT(m_node->child1().useKind() == UntypedUse); break; } LValue value = lowJSValue(m_node->child1()); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock notCell = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock straightHash = m_out.newBlock(); LBasicBlock isNumberCase = m_out.newBlock(); LBasicBlock isStringCase = m_out.newBlock(); LBasicBlock nonEmptyStringCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(notCell)); LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase); LValue isString = m_out.equal(m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoType), m_out.constInt32(StringType)); m_out.branch( isString, unsure(isStringCase), unsure(straightHash)); m_out.appendTo(isStringCase, nonEmptyStringCase); LValue stringImpl = m_out.loadPtr(value, m_heaps.JSString_value); m_out.branch( m_out.equal(stringImpl, m_out.constIntPtr(0)), rarely(slowCase), usually(nonEmptyStringCase)); m_out.appendTo(nonEmptyStringCase, notCell); LValue hash = m_out.lShr(m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::s_flagCount)); ValueFromBlock nonEmptyStringHashResult = m_out.anchor(hash); m_out.branch(m_out.equal(hash, m_out.constInt32(0)), unsure(slowCase), unsure(continuation)); m_out.appendTo(notCell, isNumberCase); m_out.branch( isNumber(value), unsure(isNumberCase), unsure(straightHash)); m_out.appendTo(isNumberCase, straightHash); m_out.branch( isInt32(value), unsure(straightHash), unsure(slowCase)); m_out.appendTo(straightHash, slowCase); ValueFromBlock fastResult = m_out.anchor(wangsInt64Hash(value)); m_out.jump(continuation); m_out.appendTo(slowCase, continuation); ValueFromBlock slowResult = m_out.anchor( vmCall(Int32, m_out.operation(operationMapHash), m_callFrame, value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setInt32(m_out.phi(Int32, fastResult, slowResult, nonEmptyStringHashResult)); } void compileGetMapBucket() { LBasicBlock loopStart = m_out.newBlock(); LBasicBlock loopAround = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock notPresentInTable = m_out.newBlock(); LBasicBlock notEmptyValue = m_out.newBlock(); LBasicBlock notDeletedValue = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(loopStart); LValue map; if (m_node->child1().useKind() == MapObjectUse) map = lowMapObject(m_node->child1()); else if (m_node->child1().useKind() == SetObjectUse) map = lowSetObject(m_node->child1()); else RELEASE_ASSERT_NOT_REACHED(); LValue key = lowJSValue(m_node->child2(), ManualOperandSpeculation); if (m_node->child2().useKind() != UntypedUse) speculate(m_node->child2()); LValue hash = lowInt32(m_node->child3()); LValue hashMapImpl = m_out.loadPtr(map, m_node->child1().useKind() == MapObjectUse ? m_heaps.JSMap_hashMapImpl : m_heaps.JSSet_hashMapImpl); LValue buffer = m_out.loadPtr(hashMapImpl, m_heaps.HashMapImpl_buffer); LValue mask = m_out.sub(m_out.load32(hashMapImpl, m_heaps.HashMapImpl_capacity), m_out.int32One); ValueFromBlock indexStart = m_out.anchor(hash); m_out.jump(loopStart); m_out.appendTo(loopStart, notEmptyValue); LValue unmaskedIndex = m_out.phi(Int32, indexStart); LValue index = m_out.bitAnd(mask, unmaskedIndex); LValue hashMapBucket = m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), buffer, m_out.zeroExt(index, Int64), ScaleEight)); ValueFromBlock bucketResult = m_out.anchor(hashMapBucket); m_out.branch(m_out.equal(hashMapBucket, m_out.constIntPtr(bitwise_cast(HashMapImpl>::emptyValue()))), unsure(notPresentInTable), unsure(notEmptyValue)); m_out.appendTo(notEmptyValue, notDeletedValue); m_out.branch(m_out.equal(hashMapBucket, m_out.constIntPtr(bitwise_cast(HashMapImpl>::deletedValue()))), unsure(loopAround), unsure(notDeletedValue)); m_out.appendTo(notDeletedValue, loopAround); LValue bucketKey = m_out.load64(hashMapBucket, m_heaps.HashMapBucket_key); // Perform Object.is() switch (m_node->child2().useKind()) { case BooleanUse: case Int32Use: case SymbolUse: case ObjectUse: { m_out.branch(m_out.equal(key, bucketKey), unsure(continuation), unsure(loopAround)); break; } case StringUse: { LBasicBlock notBitEqual = m_out.newBlock(); LBasicBlock bucketKeyIsCell = m_out.newBlock(); m_out.branch(m_out.equal(key, bucketKey), unsure(continuation), unsure(notBitEqual)); m_out.appendTo(notBitEqual, bucketKeyIsCell); m_out.branch(isCell(bucketKey), unsure(bucketKeyIsCell), unsure(loopAround)); m_out.appendTo(bucketKeyIsCell, loopAround); m_out.branch(isString(bucketKey), unsure(slowPath), unsure(loopAround)); break; } case CellUse: { LBasicBlock notBitEqual = m_out.newBlock(); LBasicBlock bucketKeyIsCell = m_out.newBlock(); LBasicBlock bucketKeyIsString = m_out.newBlock(); m_out.branch(m_out.equal(key, bucketKey), unsure(continuation), unsure(notBitEqual)); m_out.appendTo(notBitEqual, bucketKeyIsCell); m_out.branch(isCell(bucketKey), unsure(bucketKeyIsCell), unsure(loopAround)); m_out.appendTo(bucketKeyIsCell, bucketKeyIsString); m_out.branch(isString(bucketKey), unsure(bucketKeyIsString), unsure(loopAround)); m_out.appendTo(bucketKeyIsString, loopAround); m_out.branch(isString(key), unsure(slowPath), unsure(loopAround)); break; } case UntypedUse: { LBasicBlock notBitEqual = m_out.newBlock(); LBasicBlock bucketKeyIsCell = m_out.newBlock(); LBasicBlock bothAreCells = m_out.newBlock(); LBasicBlock bucketKeyIsString = m_out.newBlock(); LBasicBlock bucketKeyNotCell = m_out.newBlock(); LBasicBlock bucketKeyIsNumber = m_out.newBlock(); LBasicBlock bothAreNumbers = m_out.newBlock(); LBasicBlock bucketKeyIsInt32 = m_out.newBlock(); m_out.branch(m_out.equal(key, bucketKey), unsure(continuation), unsure(notBitEqual)); m_out.appendTo(notBitEqual, bucketKeyIsCell); m_out.branch(isCell(bucketKey), unsure(bucketKeyIsCell), unsure(bucketKeyNotCell)); m_out.appendTo(bucketKeyIsCell, bothAreCells); m_out.branch(isCell(key), unsure(bothAreCells), unsure(loopAround)); m_out.appendTo(bothAreCells, bucketKeyIsString); m_out.branch(isString(bucketKey), unsure(bucketKeyIsString), unsure(loopAround)); m_out.appendTo(bucketKeyIsString, bucketKeyNotCell); m_out.branch(isString(key), unsure(slowPath), unsure(loopAround)); m_out.appendTo(bucketKeyNotCell, bucketKeyIsNumber); m_out.branch(isNotNumber(bucketKey), unsure(loopAround), unsure(bucketKeyIsNumber)); m_out.appendTo(bucketKeyIsNumber, bothAreNumbers); m_out.branch(isNotNumber(key), unsure(loopAround), unsure(bothAreNumbers)); m_out.appendTo(bothAreNumbers, bucketKeyIsInt32); m_out.branch(isNotInt32(bucketKey), unsure(slowPath), unsure(bucketKeyIsInt32)); m_out.appendTo(bucketKeyIsInt32, loopAround); m_out.branch(isNotInt32(key), unsure(slowPath), unsure(loopAround)); break; } default: RELEASE_ASSERT_NOT_REACHED(); } m_out.appendTo(loopAround, slowPath); m_out.addIncomingToPhi(unmaskedIndex, m_out.anchor(m_out.add(index, m_out.int32One))); m_out.jump(loopStart); m_out.appendTo(slowPath, notPresentInTable); ValueFromBlock slowPathResult = m_out.anchor(vmCall(pointerType(), m_out.operation(m_node->child1().useKind() == MapObjectUse ? operationJSMapFindBucket : operationJSSetFindBucket), m_callFrame, map, key, hash)); m_out.jump(continuation); m_out.appendTo(notPresentInTable, continuation); ValueFromBlock notPresentResult = m_out.anchor(m_out.constIntPtr(0)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setMapBucket(m_out.phi(pointerType(), bucketResult, slowPathResult, notPresentResult)); } void compileLoadFromJSMapBucket() { LValue mapBucket = lowMapBucket(m_node->child1()); LBasicBlock continuation = m_out.newBlock(); LBasicBlock hasBucket = m_out.newBlock(); ValueFromBlock noBucketResult = m_out.anchor(m_out.constInt64(JSValue::encode(jsUndefined()))); m_out.branch(m_out.equal(mapBucket, m_out.constIntPtr(0)), unsure(continuation), unsure(hasBucket)); LBasicBlock lastNext = m_out.appendTo(hasBucket, continuation); ValueFromBlock bucketResult = m_out.anchor(m_out.load64(mapBucket, m_heaps.HashMapBucket_value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, noBucketResult, bucketResult)); } void compileIsNonEmptyMapBucket() { LValue bucket = lowMapBucket(m_node->child1()); LValue result = m_out.notEqual(bucket, m_out.constIntPtr(0)); setBoolean(result); } void compileIsObjectOrNull() { JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); Edge child = m_node->child1(); LValue value = lowJSValue(child); LBasicBlock cellCase = m_out.newBlock(); LBasicBlock notFunctionCase = m_out.newBlock(); LBasicBlock objectCase = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock notCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase); ValueFromBlock isFunctionResult = m_out.anchor(m_out.booleanFalse); m_out.branch( isFunction(value, provenType(child)), unsure(continuation), unsure(notFunctionCase)); m_out.appendTo(notFunctionCase, objectCase); ValueFromBlock notObjectResult = m_out.anchor(m_out.booleanFalse); m_out.branch( isObject(value, provenType(child)), unsure(objectCase), unsure(continuation)); m_out.appendTo(objectCase, slowPath); ValueFromBlock objectResult = m_out.anchor(m_out.booleanTrue); m_out.branch( isExoticForTypeof(value, provenType(child)), rarely(slowPath), usually(continuation)); m_out.appendTo(slowPath, notCellCase); LValue slowResultValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationObjectIsObject, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(globalObject), locations[1].directGPR()); }, value); ValueFromBlock slowResult = m_out.anchor(m_out.notZero64(slowResultValue)); m_out.jump(continuation); m_out.appendTo(notCellCase, continuation); LValue notCellResultValue = m_out.equal(value, m_out.constInt64(JSValue::encode(jsNull()))); ValueFromBlock notCellResult = m_out.anchor(notCellResultValue); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); LValue result = m_out.phi( Int32, isFunctionResult, notObjectResult, objectResult, slowResult, notCellResult); setBoolean(result); } void compileIsFunction() { JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); Edge child = m_node->child1(); LValue value = lowJSValue(child); LBasicBlock cellCase = m_out.newBlock(); LBasicBlock notFunctionCase = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); m_out.branch( isCell(value, provenType(child)), unsure(cellCase), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(cellCase, notFunctionCase); ValueFromBlock functionResult = m_out.anchor(m_out.booleanTrue); m_out.branch( isFunction(value, provenType(child)), unsure(continuation), unsure(notFunctionCase)); m_out.appendTo(notFunctionCase, slowPath); ValueFromBlock objectResult = m_out.anchor(m_out.booleanFalse); m_out.branch( isExoticForTypeof(value, provenType(child)), rarely(slowPath), usually(continuation)); m_out.appendTo(slowPath, continuation); LValue slowResultValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationObjectIsFunction, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(globalObject), locations[1].directGPR()); }, value); ValueFromBlock slowResult = m_out.anchor(m_out.notNull(slowResultValue)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); LValue result = m_out.phi( Int32, notCellResult, functionResult, objectResult, slowResult); setBoolean(result); } void compileIsTypedArrayView() { LValue value = lowJSValue(m_node->child1()); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); m_out.branch(isCell(value, provenType(m_node->child1())), unsure(isCellCase), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isCellCase, continuation); ValueFromBlock cellResult = m_out.anchor(isTypedArrayView(value, provenType(m_node->child1()))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, notCellResult, cellResult)); } void compileTypeOf() { Edge child = m_node->child1(); LValue value = lowJSValue(child); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); Vector results; buildTypeOf( child, value, [&] (TypeofType type) { results.append(m_out.anchor(weakPointer(vm().smallStrings.typeString(type)))); m_out.jump(continuation); }); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, results)); } void compileIn() { DFG_ASSERT(m_graph, m_node, m_node->child1().useKind() == CellUse); Node* node = m_node; Edge base = node->child1(); LValue cell = lowCell(base); if (JSString* string = node->child2()->dynamicCastConstant(vm())) { if (string->tryGetValueImpl() && string->tryGetValueImpl()->isAtomic()) { UniquedStringImpl* str = bitwise_cast(string->tryGetValueImpl()); B3::PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendSomeRegister(cell); patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister)); patchpoint->clobber(RegisterSet::macroScratchRegisters()); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); // This is the direct exit target for operation calls. We don't need a JS exceptionHandle because we don't // cache Proxy objects. Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); GPRReg baseGPR = params[1].gpr(); GPRReg resultGPR = params[0].gpr(); StructureStubInfo* stubInfo = jit.codeBlock()->addStubInfo(AccessType::In); stubInfo->callSiteIndex = state->jitCode->common.addCodeOrigin(node->origin.semantic); stubInfo->codeOrigin = node->origin.semantic; stubInfo->patch.baseGPR = static_cast(baseGPR); stubInfo->patch.valueGPR = static_cast(resultGPR); stubInfo->patch.usedRegisters = params.unavailableRegisters(); CCallHelpers::PatchableJump jump = jit.patchableJump(); CCallHelpers::Label done = jit.label(); params.addLatePath( [=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); jump.m_jump.link(&jit); CCallHelpers::Label slowPathBegin = jit.label(); CCallHelpers::Call slowPathCall = callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), operationInOptimize, resultGPR, CCallHelpers::TrustedImmPtr(stubInfo), baseGPR, CCallHelpers::TrustedImmPtr(str)).call(); jit.jump().linkTo(done, &jit); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { CodeLocationLabel start = linkBuffer.locationOf(jump); stubInfo->patch.start = start; ptrdiff_t inlineSize = MacroAssembler::differenceBetweenCodePtr( start, linkBuffer.locationOf(done)); RELEASE_ASSERT(inlineSize >= 0); stubInfo->patch.inlineSize = inlineSize; stubInfo->patch.deltaFromStartToSlowPathCallLocation = MacroAssembler::differenceBetweenCodePtr( start, linkBuffer.locationOf(slowPathCall)); stubInfo->patch.deltaFromStartToSlowPathStart = MacroAssembler::differenceBetweenCodePtr( start, linkBuffer.locationOf(slowPathBegin)); }); }); }); setJSValue(patchpoint); return; } } setJSValue(vmCall(Int64, m_out.operation(operationGenericIn), m_callFrame, cell, lowJSValue(m_node->child2()))); } void compileHasOwnProperty() { LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = nullptr; LValue object = lowObject(m_node->child1()); LValue uniquedStringImpl; LValue keyAsValue = nullptr; switch (m_node->child2().useKind()) { case StringUse: { LBasicBlock isNonEmptyString = m_out.newBlock(); LBasicBlock isAtomicString = m_out.newBlock(); keyAsValue = lowString(m_node->child2()); uniquedStringImpl = m_out.loadPtr(keyAsValue, m_heaps.JSString_value); m_out.branch(m_out.notNull(uniquedStringImpl), usually(isNonEmptyString), rarely(slowCase)); lastNext = m_out.appendTo(isNonEmptyString, isAtomicString); LValue isNotAtomic = m_out.testIsZero32(m_out.load32(uniquedStringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIsAtomic())); m_out.branch(isNotAtomic, rarely(slowCase), usually(isAtomicString)); m_out.appendTo(isAtomicString, slowCase); break; } case SymbolUse: { keyAsValue = lowSymbol(m_node->child2()); uniquedStringImpl = m_out.loadPtr(keyAsValue, m_heaps.Symbol_symbolImpl); lastNext = m_out.insertNewBlocksBefore(slowCase); break; } case UntypedUse: { LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock isStringCase = m_out.newBlock(); LBasicBlock notStringCase = m_out.newBlock(); LBasicBlock isNonEmptyString = m_out.newBlock(); LBasicBlock isSymbolCase = m_out.newBlock(); LBasicBlock hasUniquedStringImpl = m_out.newBlock(); keyAsValue = lowJSValue(m_node->child2()); m_out.branch(isCell(keyAsValue), usually(isCellCase), rarely(slowCase)); lastNext = m_out.appendTo(isCellCase, isStringCase); m_out.branch(isString(keyAsValue), unsure(isStringCase), unsure(notStringCase)); m_out.appendTo(isStringCase, isNonEmptyString); LValue implFromString = m_out.loadPtr(keyAsValue, m_heaps.JSString_value); ValueFromBlock stringResult = m_out.anchor(implFromString); m_out.branch(m_out.notNull(implFromString), usually(isNonEmptyString), rarely(slowCase)); m_out.appendTo(isNonEmptyString, notStringCase); LValue isNotAtomic = m_out.testIsZero32(m_out.load32(implFromString, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIsAtomic())); m_out.branch(isNotAtomic, rarely(slowCase), usually(hasUniquedStringImpl)); m_out.appendTo(notStringCase, isSymbolCase); m_out.branch(isSymbol(keyAsValue), unsure(isSymbolCase), unsure(slowCase)); m_out.appendTo(isSymbolCase, hasUniquedStringImpl); ValueFromBlock symbolResult = m_out.anchor(m_out.loadPtr(keyAsValue, m_heaps.Symbol_symbolImpl)); m_out.jump(hasUniquedStringImpl); m_out.appendTo(hasUniquedStringImpl, slowCase); uniquedStringImpl = m_out.phi(pointerType(), stringResult, symbolResult); break; } default: RELEASE_ASSERT_NOT_REACHED(); } ASSERT(keyAsValue); // Note that we don't test if the hash is zero here. AtomicStringImpl's can't have a zero // hash, however, a SymbolImpl may. But, because this is a cache, we don't care. We only // ever load the result from the cache if the cache entry matches what we are querying for. // So we either get super lucky and use zero for the hash and somehow collide with the entity // we're looking for, or we realize we're comparing against another entity, and go to the // slow path anyways. LValue hash = m_out.lShr(m_out.load32(uniquedStringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::s_flagCount)); LValue structureID = m_out.load32(object, m_heaps.JSCell_structureID); LValue index = m_out.add(hash, structureID); index = m_out.zeroExtPtr(m_out.bitAnd(index, m_out.constInt32(HasOwnPropertyCache::mask))); ASSERT(vm().hasOwnPropertyCache()); LValue cache = m_out.constIntPtr(vm().hasOwnPropertyCache()); IndexedAbstractHeap& heap = m_heaps.HasOwnPropertyCache; LValue sameStructureID = m_out.equal(structureID, m_out.load32(m_out.baseIndex(heap, cache, index, JSValue(), HasOwnPropertyCache::Entry::offsetOfStructureID()))); LValue sameImpl = m_out.equal(uniquedStringImpl, m_out.loadPtr(m_out.baseIndex(heap, cache, index, JSValue(), HasOwnPropertyCache::Entry::offsetOfImpl()))); ValueFromBlock fastResult = m_out.anchor(m_out.load8ZeroExt32(m_out.baseIndex(heap, cache, index, JSValue(), HasOwnPropertyCache::Entry::offsetOfResult()))); LValue cacheHit = m_out.bitAnd(sameStructureID, sameImpl); m_out.branch(m_out.notZero32(cacheHit), usually(continuation), rarely(slowCase)); m_out.appendTo(slowCase, continuation); ValueFromBlock slowResult; slowResult = m_out.anchor(vmCall(Int32, m_out.operation(operationHasOwnProperty), m_callFrame, object, keyAsValue)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, fastResult, slowResult)); } void compileParseInt() { RELEASE_ASSERT(m_node->child1().useKind() == UntypedUse || m_node->child1().useKind() == StringUse); LValue result; if (m_node->child2()) { LValue radix = lowInt32(m_node->child2()); if (m_node->child1().useKind() == UntypedUse) result = vmCall(Int64, m_out.operation(operationParseIntGeneric), m_callFrame, lowJSValue(m_node->child1()), radix); else result = vmCall(Int64, m_out.operation(operationParseIntString), m_callFrame, lowString(m_node->child1()), radix); } else { if (m_node->child1().useKind() == UntypedUse) result = vmCall(Int64, m_out.operation(operationParseIntNoRadixGeneric), m_callFrame, lowJSValue(m_node->child1())); else result = vmCall(Int64, m_out.operation(operationParseIntStringNoRadix), m_callFrame, lowString(m_node->child1())); } setJSValue(result); } void compileOverridesHasInstance() { FrozenValue* defaultHasInstanceFunction = m_node->cellOperand(); ASSERT(defaultHasInstanceFunction->cell()->inherits(vm(), JSFunction::info())); LValue constructor = lowCell(m_node->child1()); LValue hasInstance = lowJSValue(m_node->child2()); LBasicBlock defaultHasInstance = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); // Unlike in the DFG, we don't worry about cleaning this code up for the case where we have proven the hasInstanceValue is a constant as B3 should fix it for us. ValueFromBlock notDefaultHasInstanceResult = m_out.anchor(m_out.booleanTrue); m_out.branch(m_out.notEqual(hasInstance, frozenPointer(defaultHasInstanceFunction)), unsure(continuation), unsure(defaultHasInstance)); LBasicBlock lastNext = m_out.appendTo(defaultHasInstance, continuation); ValueFromBlock implementsDefaultHasInstanceResult = m_out.anchor(m_out.testIsZero32( m_out.load8ZeroExt32(constructor, m_heaps.JSCell_typeInfoFlags), m_out.constInt32(ImplementsDefaultHasInstance))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, implementsDefaultHasInstanceResult, notDefaultHasInstanceResult)); } void compileCheckTypeInfoFlags() { speculate( BadTypeInfoFlags, noValue(), 0, m_out.testIsZero32( m_out.load8ZeroExt32(lowCell(m_node->child1()), m_heaps.JSCell_typeInfoFlags), m_out.constInt32(m_node->typeInfoOperand()))); } void compileInstanceOf() { LValue cell; if (m_node->child1().useKind() == UntypedUse) cell = lowJSValue(m_node->child1()); else cell = lowCell(m_node->child1()); LValue prototype = lowCell(m_node->child2()); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock loop = m_out.newBlock(); LBasicBlock notYetInstance = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock loadPrototypeDirect = m_out.newBlock(); LBasicBlock defaultHasInstanceSlow = m_out.newBlock(); LValue condition; if (m_node->child1().useKind() == UntypedUse) condition = isCell(cell, provenType(m_node->child1())); else condition = m_out.booleanTrue; ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); m_out.branch(condition, unsure(isCellCase), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isCellCase, loop); speculate(BadType, noValue(), 0, isNotObject(prototype, provenType(m_node->child2()))); ValueFromBlock originalValue = m_out.anchor(cell); m_out.jump(loop); m_out.appendTo(loop, loadPrototypeDirect); LValue value = m_out.phi(Int64, originalValue); LValue type = m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoType); m_out.branch( m_out.notEqual(type, m_out.constInt32(ProxyObjectType)), usually(loadPrototypeDirect), rarely(defaultHasInstanceSlow)); m_out.appendTo(loadPrototypeDirect, notYetInstance); LValue structure = loadStructure(value); LValue currentPrototype = m_out.load64(structure, m_heaps.Structure_prototype); ValueFromBlock isInstanceResult = m_out.anchor(m_out.booleanTrue); m_out.branch( m_out.equal(currentPrototype, prototype), unsure(continuation), unsure(notYetInstance)); m_out.appendTo(notYetInstance, defaultHasInstanceSlow); ValueFromBlock notInstanceResult = m_out.anchor(m_out.booleanFalse); m_out.addIncomingToPhi(value, m_out.anchor(currentPrototype)); m_out.branch(isCell(currentPrototype), unsure(loop), unsure(continuation)); m_out.appendTo(defaultHasInstanceSlow, continuation); // We can use the value that we're looping with because we // can just continue off from wherever we bailed from the // loop. ValueFromBlock defaultHasInstanceResult = m_out.anchor( vmCall(Int32, m_out.operation(operationDefaultHasInstance), m_callFrame, value, prototype)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean( m_out.phi(Int32, notCellResult, isInstanceResult, notInstanceResult, defaultHasInstanceResult)); } void compileInstanceOfCustom() { LValue value = lowJSValue(m_node->child1()); LValue constructor = lowCell(m_node->child2()); LValue hasInstance = lowJSValue(m_node->child3()); setBoolean(m_out.logicalNot(m_out.equal(m_out.constInt32(0), vmCall(Int32, m_out.operation(operationInstanceOfCustom), m_callFrame, value, constructor, hasInstance)))); } void compileCountExecution() { TypedPointer counter = m_out.absolute(m_node->executionCounter()->address()); m_out.store64(m_out.add(m_out.load64(counter), m_out.constInt64(1)), counter); } void compileStoreBarrier() { emitStoreBarrier(lowCell(m_node->child1()), m_node->op() == FencedStoreBarrier); } void compileHasIndexedProperty() { switch (m_node->arrayMode().type()) { case Array::Int32: case Array::Contiguous: { LValue base = lowCell(m_node->child1()); LValue index = lowInt32(m_node->child2()); LValue storage = lowStorage(m_node->child3()); LValue internalMethodType = m_out.constInt32(static_cast(m_node->internalMethodType())); IndexedAbstractHeap& heap = m_node->arrayMode().type() == Array::Int32 ? m_heaps.indexedInt32Properties : m_heaps.indexedContiguousProperties; LBasicBlock checkHole = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); if (!m_node->arrayMode().isInBounds()) { m_out.branch( m_out.aboveOrEqual( index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)), rarely(slowCase), usually(checkHole)); } else m_out.jump(checkHole); LBasicBlock lastNext = m_out.appendTo(checkHole, slowCase); LValue checkHoleResultValue = m_out.notZero64(m_out.load64(baseIndex(heap, storage, index, m_node->child2()))); ValueFromBlock checkHoleResult = m_out.anchor(checkHoleResultValue); m_out.branch(checkHoleResultValue, usually(continuation), rarely(slowCase)); m_out.appendTo(slowCase, continuation); ValueFromBlock slowResult = m_out.anchor(m_out.equal( m_out.constInt64(JSValue::encode(jsBoolean(true))), vmCall(Int64, m_out.operation(operationHasIndexedProperty), m_callFrame, base, index, internalMethodType))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, checkHoleResult, slowResult)); return; } case Array::Double: { LValue base = lowCell(m_node->child1()); LValue index = lowInt32(m_node->child2()); LValue storage = lowStorage(m_node->child3()); LValue internalMethodType = m_out.constInt32(static_cast(m_node->internalMethodType())); IndexedAbstractHeap& heap = m_heaps.indexedDoubleProperties; LBasicBlock checkHole = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); if (!m_node->arrayMode().isInBounds()) { m_out.branch( m_out.aboveOrEqual( index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)), rarely(slowCase), usually(checkHole)); } else m_out.jump(checkHole); LBasicBlock lastNext = m_out.appendTo(checkHole, slowCase); LValue doubleValue = m_out.loadDouble(baseIndex(heap, storage, index, m_node->child2())); LValue checkHoleResultValue = m_out.doubleEqual(doubleValue, doubleValue); ValueFromBlock checkHoleResult = m_out.anchor(checkHoleResultValue); m_out.branch(checkHoleResultValue, usually(continuation), rarely(slowCase)); m_out.appendTo(slowCase, continuation); ValueFromBlock slowResult = m_out.anchor(m_out.equal( m_out.constInt64(JSValue::encode(jsBoolean(true))), vmCall(Int64, m_out.operation(operationHasIndexedProperty), m_callFrame, base, index, internalMethodType))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, checkHoleResult, slowResult)); return; } default: RELEASE_ASSERT_NOT_REACHED(); return; } } void compileHasGenericProperty() { LValue base = lowJSValue(m_node->child1()); LValue property = lowCell(m_node->child2()); setJSValue(vmCall(Int64, m_out.operation(operationHasGenericProperty), m_callFrame, base, property)); } void compileHasStructureProperty() { LValue base = lowJSValue(m_node->child1()); LValue property = lowString(m_node->child2()); LValue enumerator = lowCell(m_node->child3()); LBasicBlock correctStructure = m_out.newBlock(); LBasicBlock wrongStructure = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(m_out.notEqual( m_out.load32(base, m_heaps.JSCell_structureID), m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)), rarely(wrongStructure), usually(correctStructure)); LBasicBlock lastNext = m_out.appendTo(correctStructure, wrongStructure); ValueFromBlock correctStructureResult = m_out.anchor(m_out.booleanTrue); m_out.jump(continuation); m_out.appendTo(wrongStructure, continuation); ValueFromBlock wrongStructureResult = m_out.anchor( m_out.equal( m_out.constInt64(JSValue::encode(jsBoolean(true))), vmCall(Int64, m_out.operation(operationHasGenericProperty), m_callFrame, base, property))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, correctStructureResult, wrongStructureResult)); } void compileGetDirectPname() { LValue base = lowCell(m_graph.varArgChild(m_node, 0)); LValue property = lowCell(m_graph.varArgChild(m_node, 1)); LValue index = lowInt32(m_graph.varArgChild(m_node, 2)); LValue enumerator = lowCell(m_graph.varArgChild(m_node, 3)); LBasicBlock checkOffset = m_out.newBlock(); LBasicBlock inlineLoad = m_out.newBlock(); LBasicBlock outOfLineLoad = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(m_out.notEqual( m_out.load32(base, m_heaps.JSCell_structureID), m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedStructureID)), rarely(slowCase), usually(checkOffset)); LBasicBlock lastNext = m_out.appendTo(checkOffset, inlineLoad); m_out.branch(m_out.aboveOrEqual(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity)), unsure(outOfLineLoad), unsure(inlineLoad)); m_out.appendTo(inlineLoad, outOfLineLoad); ValueFromBlock inlineResult = m_out.anchor( m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), base, m_out.zeroExt(index, Int64), ScaleEight, JSObject::offsetOfInlineStorage()))); m_out.jump(continuation); m_out.appendTo(outOfLineLoad, slowCase); LValue storage = m_out.loadPtr(base, m_heaps.JSObject_butterfly); LValue realIndex = m_out.signExt32To64( m_out.neg(m_out.sub(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_cachedInlineCapacity)))); int32_t offsetOfFirstProperty = static_cast(offsetInButterfly(firstOutOfLineOffset)) * sizeof(EncodedJSValue); ValueFromBlock outOfLineResult = m_out.anchor( m_out.load64(m_out.baseIndex(m_heaps.properties.atAnyNumber(), storage, realIndex, ScaleEight, offsetOfFirstProperty))); m_out.jump(continuation); m_out.appendTo(slowCase, continuation); ValueFromBlock slowCaseResult = m_out.anchor( vmCall(Int64, m_out.operation(operationGetByVal), m_callFrame, base, property)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, inlineResult, outOfLineResult, slowCaseResult)); } void compileGetEnumerableLength() { LValue enumerator = lowCell(m_node->child1()); setInt32(m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_indexLength)); } void compileGetPropertyEnumerator() { LValue base = lowCell(m_node->child1()); setJSValue(vmCall(Int64, m_out.operation(operationGetPropertyEnumerator), m_callFrame, base)); } void compileGetEnumeratorStructurePname() { LValue enumerator = lowCell(m_node->child1()); LValue index = lowInt32(m_node->child2()); LBasicBlock inBounds = m_out.newBlock(); LBasicBlock outOfBounds = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endStructurePropertyIndex)), usually(inBounds), rarely(outOfBounds)); LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds); LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector); ValueFromBlock inBoundsResult = m_out.anchor( m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index)))); m_out.jump(continuation); m_out.appendTo(outOfBounds, continuation); ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(ValueNull)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, inBoundsResult, outOfBoundsResult)); } void compileGetEnumeratorGenericPname() { LValue enumerator = lowCell(m_node->child1()); LValue index = lowInt32(m_node->child2()); LBasicBlock inBounds = m_out.newBlock(); LBasicBlock outOfBounds = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(m_out.below(index, m_out.load32(enumerator, m_heaps.JSPropertyNameEnumerator_endGenericPropertyIndex)), usually(inBounds), rarely(outOfBounds)); LBasicBlock lastNext = m_out.appendTo(inBounds, outOfBounds); LValue storage = m_out.loadPtr(enumerator, m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVector); ValueFromBlock inBoundsResult = m_out.anchor( m_out.loadPtr(m_out.baseIndex(m_heaps.JSPropertyNameEnumerator_cachedPropertyNamesVectorContents, storage, m_out.zeroExtPtr(index)))); m_out.jump(continuation); m_out.appendTo(outOfBounds, continuation); ValueFromBlock outOfBoundsResult = m_out.anchor(m_out.constInt64(ValueNull)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(Int64, inBoundsResult, outOfBoundsResult)); } void compileToIndexString() { LValue index = lowInt32(m_node->child1()); setJSValue(vmCall(Int64, m_out.operation(operationToIndexString), m_callFrame, index)); } void compileCheckStructureImmediate() { LValue structure = lowCell(m_node->child1()); checkStructure( structure, noValue(), BadCache, m_node->structureSet(), [this] (RegisteredStructure structure) { return weakStructure(structure); }); } void compileMaterializeNewObject() { ObjectMaterializationData& data = m_node->objectMaterializationData(); // Lower the values first, to avoid creating values inside a control flow diamond. Vector values; for (unsigned i = 0; i < data.m_properties.size(); ++i) { Edge edge = m_graph.varArgChild(m_node, 1 + i); switch (data.m_properties[i].kind()) { case PublicLengthPLoc: case VectorLengthPLoc: values.append(lowInt32(edge)); break; default: values.append(lowJSValue(edge)); break; } } RegisteredStructureSet set = m_node->structureSet(); Vector blocks(set.size()); for (unsigned i = set.size(); i--;) blocks[i] = m_out.newBlock(); LBasicBlock dummyDefault = m_out.newBlock(); LBasicBlock outerContinuation = m_out.newBlock(); Vector cases(set.size()); for (unsigned i = set.size(); i--;) cases[i] = SwitchCase(weakStructure(set.at(i)), blocks[i], Weight(1)); m_out.switchInstruction( lowCell(m_graph.varArgChild(m_node, 0)), cases, dummyDefault, Weight(0)); LBasicBlock outerLastNext = m_out.m_nextBlock; Vector results; for (unsigned i = set.size(); i--;) { m_out.appendTo(blocks[i], i + 1 < set.size() ? blocks[i + 1] : dummyDefault); RegisteredStructure structure = set.at(i); LValue object; LValue butterfly; if (structure->outOfLineCapacity() || hasIndexedProperties(structure->indexingType())) { size_t allocationSize = JSFinalObject::allocationSize(structure->inlineCapacity()); MarkedAllocator* cellAllocator = subspaceFor(vm())->allocatorFor(allocationSize); DFG_ASSERT(m_graph, m_node, cellAllocator); bool hasIndexingHeader = hasIndexedProperties(structure->indexingType()); unsigned indexingHeaderSize = 0; LValue indexingPayloadSizeInBytes = m_out.intPtrZero; LValue vectorLength = m_out.int32Zero; LValue publicLength = m_out.int32Zero; if (hasIndexingHeader) { indexingHeaderSize = sizeof(IndexingHeader); for (unsigned i = data.m_properties.size(); i--;) { PromotedLocationDescriptor descriptor = data.m_properties[i]; switch (descriptor.kind()) { case PublicLengthPLoc: publicLength = values[i]; break; case VectorLengthPLoc: vectorLength = values[i]; break; default: break; } } indexingPayloadSizeInBytes = m_out.mul(m_out.zeroExtPtr(vectorLength), m_out.intPtrEight); } LValue butterflySize = m_out.add( m_out.constIntPtr( structure->outOfLineCapacity() * sizeof(JSValue) + indexingHeaderSize), indexingPayloadSizeInBytes); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); ValueFromBlock noButterfly = m_out.anchor(m_out.intPtrZero); LValue startOfStorage = allocateHeapCell( allocatorForSize(vm().auxiliarySpace, butterflySize, slowPath), slowPath); LValue fastButterflyValue = m_out.add( startOfStorage, m_out.constIntPtr( structure->outOfLineCapacity() * sizeof(JSValue) + sizeof(IndexingHeader))); ValueFromBlock haveButterfly = m_out.anchor(fastButterflyValue); splatWords( fastButterflyValue, m_out.constInt32(-structure->outOfLineCapacity() - 1), m_out.constInt32(-1), m_out.int64Zero, m_heaps.properties.atAnyNumber()); m_out.store32(vectorLength, fastButterflyValue, m_heaps.Butterfly_vectorLength); LValue fastObjectValue = allocateObject( m_out.constIntPtr(cellAllocator), structure, fastButterflyValue, slowPath); ValueFromBlock fastObject = m_out.anchor(fastObjectValue); ValueFromBlock fastButterfly = m_out.anchor(fastButterflyValue); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); LValue butterflyValue = m_out.phi(pointerType(), noButterfly, haveButterfly); LValue slowObjectValue; if (hasIndexingHeader) { slowObjectValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationNewObjectWithButterflyWithIndexingHeaderAndVectorLength, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR(), locations[2].directGPR()); }, vectorLength, butterflyValue); } else { slowObjectValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationNewObjectWithButterfly, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR()); }, butterflyValue); } ValueFromBlock slowObject = m_out.anchor(slowObjectValue); ValueFromBlock slowButterfly = m_out.anchor( m_out.loadPtr(slowObjectValue, m_heaps.JSObject_butterfly)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); object = m_out.phi(pointerType(), fastObject, slowObject); butterfly = m_out.phi(pointerType(), fastButterfly, slowButterfly); m_out.store32(publicLength, butterfly, m_heaps.Butterfly_publicLength); initializeArrayElements(m_out.constInt32(structure->indexingType()), m_out.int32Zero, vectorLength, butterfly); HashMap::Hash, WTF::UnsignedWithZeroKeyHashTraits> indexMap; Vector indices; for (unsigned i = data.m_properties.size(); i--;) { PromotedLocationDescriptor descriptor = data.m_properties[i]; if (descriptor.kind() != IndexedPropertyPLoc) continue; int32_t index = static_cast(descriptor.info()); auto result = indexMap.add(index, values[i]); DFG_ASSERT(m_graph, m_node, result); // Duplicates are illegal. indices.append(index); } if (!indices.isEmpty()) { std::sort(indices.begin(), indices.end()); Vector blocksWithStores(indices.size()); Vector blocksWithChecks(indices.size()); for (unsigned i = indices.size(); i--;) { blocksWithStores[i] = m_out.newBlock(); blocksWithChecks[i] = m_out.newBlock(); // blocksWithChecks[0] is the continuation. } LBasicBlock indexLastNext = m_out.m_nextBlock; for (unsigned i = indices.size(); i--;) { int32_t index = indices[i]; LValue value = indexMap.get(index); m_out.branch( m_out.below(m_out.constInt32(index), publicLength), unsure(blocksWithStores[i]), unsure(blocksWithChecks[i])); m_out.appendTo(blocksWithStores[i], blocksWithChecks[i]); // This has to type-check and convert its inputs, but it cannot do so in a // way that updates AI. That's a bit annoying, but if you think about how // sinking works, it's actually not a bad thing. We are virtually guaranteed // that these type checks will not fail, since the type checks that guarded // the original stores to the array are still somewhere above this point. Output::StoreType storeType; IndexedAbstractHeap* heap; switch (structure->indexingType()) { case ALL_INT32_INDEXING_TYPES: // FIXME: This could use the proven type if we had the Edge for the // value. https://bugs.webkit.org/show_bug.cgi?id=155311 speculate(BadType, noValue(), nullptr, isNotInt32(value)); storeType = Output::Store64; heap = &m_heaps.indexedInt32Properties; break; case ALL_DOUBLE_INDEXING_TYPES: { // FIXME: If the source is ValueRep, we should avoid emitting any // checks. We could also avoid emitting checks if we had the Edge of // this value. https://bugs.webkit.org/show_bug.cgi?id=155311 LBasicBlock intCase = m_out.newBlock(); LBasicBlock doubleCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isInt32(value), unsure(intCase), unsure(doubleCase)); LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase); ValueFromBlock intResult = m_out.anchor(m_out.intToDouble(unboxInt32(value))); m_out.jump(continuation); m_out.appendTo(doubleCase, continuation); speculate(BadType, noValue(), nullptr, isNumber(value)); ValueFromBlock doubleResult = m_out.anchor(unboxDouble(value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); value = m_out.phi(Double, intResult, doubleResult); storeType = Output::StoreDouble; heap = &m_heaps.indexedDoubleProperties; break; } case ALL_CONTIGUOUS_INDEXING_TYPES: storeType = Output::Store64; heap = &m_heaps.indexedContiguousProperties; break; default: DFG_CRASH(m_graph, m_node, "Invalid indexing type"); break; } m_out.store(value, m_out.address(butterfly, heap->at(index)), storeType); m_out.jump(blocksWithChecks[i]); m_out.appendTo( blocksWithChecks[i], i ? blocksWithStores[i - 1] : indexLastNext); } } } else { // In the easy case where we can do a one-shot allocation, we simply allocate the // object to directly have the desired structure. object = allocateObject(structure); butterfly = nullptr; // Don't have one, don't need one. } BitVector setInlineOffsets; for (PropertyMapEntry entry : structure->getPropertiesConcurrently()) { for (unsigned i = data.m_properties.size(); i--;) { PromotedLocationDescriptor descriptor = data.m_properties[i]; if (descriptor.kind() != NamedPropertyPLoc) continue; if (m_graph.identifiers()[descriptor.info()] != entry.key) continue; LValue base; if (isInlineOffset(entry.offset)) { setInlineOffsets.set(entry.offset); base = object; } else base = butterfly; storeProperty(values[i], base, descriptor.info(), entry.offset); break; } } for (unsigned i = structure->inlineCapacity(); i--;) { if (!setInlineOffsets.get(i)) m_out.store64(m_out.int64Zero, m_out.address(m_heaps.properties.atAnyNumber(), object, offsetRelativeToBase(i))); } results.append(m_out.anchor(object)); m_out.jump(outerContinuation); } m_out.appendTo(dummyDefault, outerContinuation); m_out.unreachable(); m_out.appendTo(outerContinuation, outerLastNext); setJSValue(m_out.phi(pointerType(), results)); mutatorFence(); } void compileMaterializeCreateActivation() { ObjectMaterializationData& data = m_node->objectMaterializationData(); Vector values; for (unsigned i = 0; i < data.m_properties.size(); ++i) values.append(lowJSValue(m_graph.varArgChild(m_node, 2 + i))); LValue scope = lowCell(m_graph.varArgChild(m_node, 1)); SymbolTable* table = m_node->castOperand(); ASSERT(table == m_graph.varArgChild(m_node, 0)->castConstant(vm())); RegisteredStructure structure = m_graph.registerStructure(m_graph.globalObjectFor(m_node->origin.semantic)->activationStructure()); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); LValue fastObject = allocateObject( JSLexicalEnvironment::allocationSize(table), structure, m_out.intPtrZero, slowPath); m_out.storePtr(scope, fastObject, m_heaps.JSScope_next); m_out.storePtr(weakPointer(table), fastObject, m_heaps.JSSymbolTableObject_symbolTable); ValueFromBlock fastResult = m_out.anchor(fastObject); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); // We ensure allocation sinking explictly sets bottom values for all field members. // Therefore, it doesn't matter what JSValue we pass in as the initialization value // because all fields will be overwritten. // FIXME: It may be worth creating an operation that calls a constructor on JSLexicalEnvironment that // doesn't initialize every slot because we are guaranteed to do that here. LValue callResult = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationCreateActivationDirect, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(structure.get()), locations[1].directGPR(), CCallHelpers::TrustedImmPtr(table), CCallHelpers::TrustedImm64(JSValue::encode(jsUndefined()))); }, scope); ValueFromBlock slowResult = m_out.anchor(callResult); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); LValue activation = m_out.phi(pointerType(), fastResult, slowResult); RELEASE_ASSERT(data.m_properties.size() == table->scopeSize()); for (unsigned i = 0; i < data.m_properties.size(); ++i) { PromotedLocationDescriptor descriptor = data.m_properties[i]; ASSERT(descriptor.kind() == ClosureVarPLoc); m_out.store64( values[i], activation, m_heaps.JSEnvironmentRecord_variables[descriptor.info()]); } if (validationEnabled()) { // Validate to make sure every slot in the scope has one value. ConcurrentJSLocker locker(table->m_lock); for (auto iter = table->begin(locker), end = table->end(locker); iter != end; ++iter) { bool found = false; for (unsigned i = 0; i < data.m_properties.size(); ++i) { PromotedLocationDescriptor descriptor = data.m_properties[i]; ASSERT(descriptor.kind() == ClosureVarPLoc); if (iter->value.scopeOffset().offset() == descriptor.info()) { found = true; break; } } ASSERT_UNUSED(found, found); } } mutatorFence(); setJSValue(activation); } void compileCheckWatchdogTimer() { LBasicBlock timerDidFire = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue state = m_out.load8ZeroExt32(m_out.absolute(vm().watchdog()->timerDidFireAddress())); m_out.branch(m_out.isZero32(state), usually(continuation), rarely(timerDidFire)); LBasicBlock lastNext = m_out.appendTo(timerDidFire, continuation); lazySlowPath( [=] (const Vector&) -> RefPtr { return createLazyCallGenerator(operationHandleWatchdogTimer, InvalidGPRReg); }); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void compileRegExpExec() { LValue globalObject = lowCell(m_node->child1()); if (m_node->child2().useKind() == RegExpObjectUse) { LValue base = lowRegExpObject(m_node->child2()); if (m_node->child3().useKind() == StringUse) { LValue argument = lowString(m_node->child3()); LValue result = vmCall( Int64, m_out.operation(operationRegExpExecString), m_callFrame, globalObject, base, argument); setJSValue(result); return; } LValue argument = lowJSValue(m_node->child3()); LValue result = vmCall( Int64, m_out.operation(operationRegExpExec), m_callFrame, globalObject, base, argument); setJSValue(result); return; } LValue base = lowJSValue(m_node->child2()); LValue argument = lowJSValue(m_node->child3()); LValue result = vmCall( Int64, m_out.operation(operationRegExpExecGeneric), m_callFrame, globalObject, base, argument); setJSValue(result); } void compileRegExpTest() { LValue globalObject = lowCell(m_node->child1()); if (m_node->child2().useKind() == RegExpObjectUse) { LValue base = lowRegExpObject(m_node->child2()); if (m_node->child3().useKind() == StringUse) { LValue argument = lowString(m_node->child3()); LValue result = vmCall( Int32, m_out.operation(operationRegExpTestString), m_callFrame, globalObject, base, argument); setBoolean(result); return; } LValue argument = lowJSValue(m_node->child3()); LValue result = vmCall( Int32, m_out.operation(operationRegExpTest), m_callFrame, globalObject, base, argument); setBoolean(result); return; } LValue base = lowJSValue(m_node->child2()); LValue argument = lowJSValue(m_node->child3()); LValue result = vmCall( Int32, m_out.operation(operationRegExpTestGeneric), m_callFrame, globalObject, base, argument); setBoolean(result); } void compileNewRegexp() { FrozenValue* regexp = m_node->cellOperand(); ASSERT(regexp->cell()->inherits(vm(), RegExp::info())); LValue result = vmCall( pointerType(), m_out.operation(operationNewRegexp), m_callFrame, frozenPointer(regexp)); setJSValue(result); } void compileSetFunctionName() { vmCall(Void, m_out.operation(operationSetFunctionName), m_callFrame, lowCell(m_node->child1()), lowJSValue(m_node->child2())); } void compileStringReplace() { if (m_node->child1().useKind() == StringUse && m_node->child2().useKind() == RegExpObjectUse && m_node->child3().useKind() == StringUse) { if (JSString* replace = m_node->child3()->dynamicCastConstant(vm())) { if (!replace->length()) { LValue string = lowString(m_node->child1()); LValue regExp = lowRegExpObject(m_node->child2()); LValue result = vmCall( Int64, m_out.operation(operationStringProtoFuncReplaceRegExpEmptyStr), m_callFrame, string, regExp); setJSValue(result); return; } } LValue string = lowString(m_node->child1()); LValue regExp = lowRegExpObject(m_node->child2()); LValue replace = lowString(m_node->child3()); LValue result = vmCall( Int64, m_out.operation(operationStringProtoFuncReplaceRegExpString), m_callFrame, string, regExp, replace); setJSValue(result); return; } LValue search; if (m_node->child2().useKind() == StringUse) search = lowString(m_node->child2()); else search = lowJSValue(m_node->child2()); LValue result = vmCall( Int64, m_out.operation(operationStringProtoFuncReplaceGeneric), m_callFrame, lowJSValue(m_node->child1()), search, lowJSValue(m_node->child3())); setJSValue(result); } void compileGetRegExpObjectLastIndex() { setJSValue(m_out.load64(lowRegExpObject(m_node->child1()), m_heaps.RegExpObject_lastIndex)); } void compileSetRegExpObjectLastIndex() { LValue regExp = lowRegExpObject(m_node->child1()); LValue value = lowJSValue(m_node->child2()); speculate( ExoticObjectMode, noValue(), nullptr, m_out.isZero32(m_out.load8ZeroExt32(regExp, m_heaps.RegExpObject_lastIndexIsWritable))); m_out.store64(value, regExp, m_heaps.RegExpObject_lastIndex); } void compileLogShadowChickenPrologue() { LValue packet = ensureShadowChickenPacket(); LValue scope = lowCell(m_node->child1()); m_out.storePtr(m_callFrame, packet, m_heaps.ShadowChicken_Packet_frame); m_out.storePtr(m_out.loadPtr(addressFor(0)), packet, m_heaps.ShadowChicken_Packet_callerFrame); m_out.storePtr(m_out.loadPtr(payloadFor(CallFrameSlot::callee)), packet, m_heaps.ShadowChicken_Packet_callee); m_out.storePtr(scope, packet, m_heaps.ShadowChicken_Packet_scope); } void compileLogShadowChickenTail() { LValue packet = ensureShadowChickenPacket(); LValue thisValue = lowJSValue(m_node->child1()); LValue scope = lowCell(m_node->child2()); CallSiteIndex callSiteIndex = m_ftlState.jitCode->common.addCodeOrigin(m_node->origin.semantic); m_out.storePtr(m_callFrame, packet, m_heaps.ShadowChicken_Packet_frame); m_out.storePtr(m_out.constIntPtr(bitwise_cast(ShadowChicken::Packet::tailMarker())), packet, m_heaps.ShadowChicken_Packet_callee); m_out.store64(thisValue, packet, m_heaps.ShadowChicken_Packet_thisValue); m_out.storePtr(scope, packet, m_heaps.ShadowChicken_Packet_scope); // We don't want the CodeBlock to have a weak pointer to itself because // that would cause it to always get collected. m_out.storePtr(m_out.constIntPtr(bitwise_cast(codeBlock())), packet, m_heaps.ShadowChicken_Packet_codeBlock); m_out.store32(m_out.constInt32(callSiteIndex.bits()), packet, m_heaps.ShadowChicken_Packet_callSiteIndex); } void compileRecordRegExpCachedResult() { Edge constructorEdge = m_graph.varArgChild(m_node, 0); Edge regExpEdge = m_graph.varArgChild(m_node, 1); Edge stringEdge = m_graph.varArgChild(m_node, 2); Edge startEdge = m_graph.varArgChild(m_node, 3); Edge endEdge = m_graph.varArgChild(m_node, 4); LValue constructor = lowCell(constructorEdge); LValue regExp = lowCell(regExpEdge); LValue string = lowCell(stringEdge); LValue start = lowInt32(startEdge); LValue end = lowInt32(endEdge); m_out.storePtr(regExp, constructor, m_heaps.RegExpConstructor_cachedResult_lastRegExp); m_out.storePtr(string, constructor, m_heaps.RegExpConstructor_cachedResult_lastInput); m_out.store32(start, constructor, m_heaps.RegExpConstructor_cachedResult_result_start); m_out.store32(end, constructor, m_heaps.RegExpConstructor_cachedResult_result_end); m_out.store32As8( m_out.constInt32(0), m_out.address(constructor, m_heaps.RegExpConstructor_cachedResult_reified)); } struct ArgumentsLength { ArgumentsLength() : isKnown(false) , known(UINT_MAX) , value(nullptr) { } bool isKnown; unsigned known; LValue value; }; ArgumentsLength getArgumentsLength(InlineCallFrame* inlineCallFrame) { ArgumentsLength length; if (inlineCallFrame && !inlineCallFrame->isVarargs()) { length.known = inlineCallFrame->arguments.size() - 1; length.isKnown = true; length.value = m_out.constInt32(length.known); } else { length.known = UINT_MAX; length.isKnown = false; VirtualRegister argumentCountRegister; if (!inlineCallFrame) argumentCountRegister = VirtualRegister(CallFrameSlot::argumentCount); else argumentCountRegister = inlineCallFrame->argumentCountRegister; length.value = m_out.sub(m_out.load32(payloadFor(argumentCountRegister)), m_out.int32One); } return length; } ArgumentsLength getArgumentsLength() { return getArgumentsLength(m_node->origin.semantic.inlineCallFrame); } LValue getCurrentCallee() { if (InlineCallFrame* frame = m_node->origin.semantic.inlineCallFrame) { if (frame->isClosureCall) return m_out.loadPtr(addressFor(frame->calleeRecovery.virtualRegister())); return weakPointer(frame->calleeRecovery.constant().asCell()); } return m_out.loadPtr(addressFor(CallFrameSlot::callee)); } LValue getArgumentsStart(InlineCallFrame* inlineCallFrame, unsigned offset = 0) { VirtualRegister start = AssemblyHelpers::argumentsStart(inlineCallFrame) + offset; return addressFor(start).value(); } LValue getArgumentsStart() { return getArgumentsStart(m_node->origin.semantic.inlineCallFrame); } template void checkStructure( LValue structureDiscriminant, const FormattedValue& formattedValue, ExitKind exitKind, RegisteredStructureSet set, const Functor& weakStructureDiscriminant) { if (set.isEmpty()) { terminate(exitKind); return; } if (set.size() == 1) { speculate( exitKind, formattedValue, 0, m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set[0]))); return; } LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); for (unsigned i = 0; i < set.size() - 1; ++i) { LBasicBlock nextStructure = m_out.newBlock(); m_out.branch( m_out.equal(structureDiscriminant, weakStructureDiscriminant(set[i])), unsure(continuation), unsure(nextStructure)); m_out.appendTo(nextStructure); } speculate( exitKind, formattedValue, 0, m_out.notEqual(structureDiscriminant, weakStructureDiscriminant(set.last()))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } LValue numberOrNotCellToInt32(Edge edge, LValue value) { LBasicBlock intCase = m_out.newBlock(); LBasicBlock notIntCase = m_out.newBlock(); LBasicBlock doubleCase = 0; LBasicBlock notNumberCase = 0; if (edge.useKind() == NotCellUse) { doubleCase = m_out.newBlock(); notNumberCase = m_out.newBlock(); } LBasicBlock continuation = m_out.newBlock(); Vector results; m_out.branch(isNotInt32(value), unsure(notIntCase), unsure(intCase)); LBasicBlock lastNext = m_out.appendTo(intCase, notIntCase); results.append(m_out.anchor(unboxInt32(value))); m_out.jump(continuation); if (edge.useKind() == NumberUse) { m_out.appendTo(notIntCase, continuation); FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isCellOrMisc(value)); results.append(m_out.anchor(doubleToInt32(unboxDouble(value)))); m_out.jump(continuation); } else { m_out.appendTo(notIntCase, doubleCase); m_out.branch( isCellOrMisc(value, provenType(edge)), unsure(notNumberCase), unsure(doubleCase)); m_out.appendTo(doubleCase, notNumberCase); results.append(m_out.anchor(doubleToInt32(unboxDouble(value)))); m_out.jump(continuation); m_out.appendTo(notNumberCase, continuation); FTL_TYPE_CHECK(jsValueValue(value), edge, ~SpecCell, isCell(value)); LValue specialResult = m_out.select( m_out.equal(value, m_out.constInt64(JSValue::encode(jsBoolean(true)))), m_out.int32One, m_out.int32Zero); results.append(m_out.anchor(specialResult)); m_out.jump(continuation); } m_out.appendTo(continuation, lastNext); return m_out.phi(Int32, results); } void checkInferredType(Edge edge, LValue value, const InferredType::Descriptor& type) { // This cannot use FTL_TYPE_CHECK or typeCheck() because it is called partially, as in a node like: // // MultiPutByOffset(...) // // may be lowered to: // // switch (object->structure) { // case 42: // checkInferredType(..., type1); // ... // break; // case 43: // checkInferredType(..., type2); // ... // break; // } // // where type1 and type2 are different. Using typeCheck() would mean that the edge would be // filtered by type1 & type2, instead of type1 | type2. switch (type.kind()) { case InferredType::Bottom: speculate(BadType, jsValueValue(value), edge.node(), m_out.booleanTrue); return; case InferredType::Boolean: speculate(BadType, jsValueValue(value), edge.node(), isNotBoolean(value, provenType(edge))); return; case InferredType::Other: speculate(BadType, jsValueValue(value), edge.node(), isNotOther(value, provenType(edge))); return; case InferredType::Int32: speculate(BadType, jsValueValue(value), edge.node(), isNotInt32(value, provenType(edge))); return; case InferredType::Number: speculate(BadType, jsValueValue(value), edge.node(), isNotNumber(value, provenType(edge))); return; case InferredType::String: speculate(BadType, jsValueValue(value), edge.node(), isNotCell(value, provenType(edge))); speculate(BadType, jsValueValue(value), edge.node(), isNotString(value, provenType(edge))); return; case InferredType::Symbol: speculate(BadType, jsValueValue(value), edge.node(), isNotCell(value, provenType(edge))); speculate(BadType, jsValueValue(value), edge.node(), isNotSymbol(value, provenType(edge))); return; case InferredType::ObjectWithStructure: { RegisteredStructure structure = m_graph.registerStructure(type.structure()); speculate(BadType, jsValueValue(value), edge.node(), isNotCell(value, provenType(edge))); if (!abstractValue(edge).m_structure.isSubsetOf(RegisteredStructureSet(structure))) { speculate( BadType, jsValueValue(value), edge.node(), m_out.notEqual( m_out.load32(value, m_heaps.JSCell_structureID), weakStructureID(structure))); } return; } case InferredType::ObjectWithStructureOrOther: { LBasicBlock cellCase = m_out.newBlock(); LBasicBlock notCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase); RegisteredStructure structure = m_graph.registerStructure(type.structure()); if (!abstractValue(edge).m_structure.isSubsetOf(RegisteredStructureSet(structure))) { speculate( BadType, jsValueValue(value), edge.node(), m_out.notEqual( m_out.load32(value, m_heaps.JSCell_structureID), weakStructureID(structure))); } m_out.jump(continuation); m_out.appendTo(notCellCase, continuation); speculate( BadType, jsValueValue(value), edge.node(), isNotOther(value, provenType(edge) & ~SpecCell)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return; } case InferredType::Object: speculate(BadType, jsValueValue(value), edge.node(), isNotCell(value, provenType(edge))); speculate(BadType, jsValueValue(value), edge.node(), isNotObject(value, provenType(edge))); return; case InferredType::ObjectOrOther: { LBasicBlock cellCase = m_out.newBlock(); LBasicBlock notCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase); speculate( BadType, jsValueValue(value), edge.node(), isNotObject(value, provenType(edge) & SpecCell)); m_out.jump(continuation); m_out.appendTo(notCellCase, continuation); speculate( BadType, jsValueValue(value), edge.node(), isNotOther(value, provenType(edge) & ~SpecCell)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return; } case InferredType::Top: return; } DFG_CRASH(m_graph, m_node, "Bad inferred type"); } LValue loadProperty(LValue storage, unsigned identifierNumber, PropertyOffset offset) { return m_out.load64(addressOfProperty(storage, identifierNumber, offset)); } void storeProperty( LValue value, LValue storage, unsigned identifierNumber, PropertyOffset offset) { m_out.store64(value, addressOfProperty(storage, identifierNumber, offset)); } TypedPointer addressOfProperty( LValue storage, unsigned identifierNumber, PropertyOffset offset) { return m_out.address( m_heaps.properties[identifierNumber], storage, offsetRelativeToBase(offset)); } LValue storageForTransition( LValue object, PropertyOffset offset, Structure* previousStructure, Structure* nextStructure) { if (isInlineOffset(offset)) return object; if (previousStructure->outOfLineCapacity() == nextStructure->outOfLineCapacity()) return m_out.loadPtr(object, m_heaps.JSObject_butterfly); LValue result; if (!previousStructure->outOfLineCapacity()) result = allocatePropertyStorage(object, previousStructure); else { result = reallocatePropertyStorage( object, m_out.loadPtr(object, m_heaps.JSObject_butterfly), previousStructure, nextStructure); } nukeStructureAndSetButterfly(result, object); return result; } void initializeArrayElements(LValue indexingType, LValue begin, LValue end, LValue butterfly) { if (begin == end) return; if (indexingType->hasInt32()) { IndexingType rawIndexingType = static_cast(indexingType->asInt32()); if (hasUndecided(rawIndexingType)) return; IndexedAbstractHeap* heap = m_heaps.forIndexingType(rawIndexingType); DFG_ASSERT(m_graph, m_node, heap); LValue hole; if (hasDouble(rawIndexingType)) hole = m_out.constInt64(bitwise_cast(PNaN)); else hole = m_out.constInt64(JSValue::encode(JSValue())); splatWords(butterfly, begin, end, hole, heap->atAnyIndex()); } else { LValue hole = m_out.select( m_out.equal(m_out.bitAnd(indexingType, m_out.constInt32(IndexingShapeMask)), m_out.constInt32(DoubleShape)), m_out.constInt64(bitwise_cast(PNaN)), m_out.constInt64(JSValue::encode(JSValue()))); splatWords(butterfly, begin, end, hole, m_heaps.root); } } void splatWords(LValue base, LValue begin, LValue end, LValue value, const AbstractHeap& heap) { const uint64_t unrollingLimit = 10; if (begin->hasInt() && end->hasInt()) { uint64_t beginConst = static_cast(begin->asInt()); uint64_t endConst = static_cast(end->asInt()); if (endConst - beginConst <= unrollingLimit) { for (uint64_t i = beginConst; i < endConst; ++i) { LValue pointer = m_out.add(base, m_out.constIntPtr(i * sizeof(uint64_t))); m_out.store64(value, TypedPointer(heap, pointer)); } return; } } LBasicBlock initLoop = m_out.newBlock(); LBasicBlock initDone = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(initLoop); ValueFromBlock originalIndex = m_out.anchor(end); ValueFromBlock originalPointer = m_out.anchor( m_out.add(base, m_out.shl(m_out.signExt32ToPtr(begin), m_out.constInt32(3)))); m_out.branch(m_out.notEqual(end, begin), unsure(initLoop), unsure(initDone)); m_out.appendTo(initLoop, initDone); LValue index = m_out.phi(Int32, originalIndex); LValue pointer = m_out.phi(pointerType(), originalPointer); m_out.store64(value, TypedPointer(heap, pointer)); LValue nextIndex = m_out.sub(index, m_out.int32One); m_out.addIncomingToPhi(index, m_out.anchor(nextIndex)); m_out.addIncomingToPhi(pointer, m_out.anchor(m_out.add(pointer, m_out.intPtrEight))); m_out.branch( m_out.notEqual(nextIndex, begin), unsure(initLoop), unsure(initDone)); m_out.appendTo(initDone, lastNext); } LValue allocatePropertyStorage(LValue object, Structure* previousStructure) { if (previousStructure->couldHaveIndexingHeader()) { return vmCall( pointerType(), m_out.operation(operationAllocateComplexPropertyStorageWithInitialCapacity), m_callFrame, object); } LValue result = allocatePropertyStorageWithSizeImpl(initialOutOfLineCapacity); splatWords( result, m_out.constInt32(-initialOutOfLineCapacity - 1), m_out.constInt32(-1), m_out.int64Zero, m_heaps.properties.atAnyNumber()); return result; } LValue reallocatePropertyStorage( LValue object, LValue oldStorage, Structure* previous, Structure* next) { size_t oldSize = previous->outOfLineCapacity(); size_t newSize = oldSize * outOfLineGrowthFactor; ASSERT_UNUSED(next, newSize == next->outOfLineCapacity()); if (previous->couldHaveIndexingHeader()) { LValue newAllocSize = m_out.constIntPtr(newSize); return vmCall(pointerType(), m_out.operation(operationAllocateComplexPropertyStorage), m_callFrame, object, newAllocSize); } LValue result = allocatePropertyStorageWithSizeImpl(newSize); ptrdiff_t headerSize = -sizeof(IndexingHeader) - sizeof(void*); ptrdiff_t endStorage = headerSize - static_cast(oldSize * sizeof(JSValue)); for (ptrdiff_t offset = headerSize; offset > endStorage; offset -= sizeof(void*)) { LValue loaded = m_out.loadPtr(m_out.address(m_heaps.properties.atAnyNumber(), oldStorage, offset)); m_out.storePtr(loaded, m_out.address(m_heaps.properties.atAnyNumber(), result, offset)); } splatWords( result, m_out.constInt32(-newSize - 1), m_out.constInt32(-oldSize - 1), m_out.int64Zero, m_heaps.properties.atAnyNumber()); return result; } LValue allocatePropertyStorageWithSizeImpl(size_t sizeInValues) { LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); size_t sizeInBytes = sizeInValues * sizeof(JSValue); MarkedAllocator* allocator = vm().auxiliarySpace.allocatorFor(sizeInBytes); LValue startOfStorage = allocateHeapCell(m_out.constIntPtr(allocator), slowPath); ValueFromBlock fastButterfly = m_out.anchor( m_out.add(m_out.constIntPtr(sizeInBytes + sizeof(IndexingHeader)), startOfStorage)); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); LValue slowButterflyValue; if (sizeInValues == initialOutOfLineCapacity) { slowButterflyValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationAllocateSimplePropertyStorageWithInitialCapacity, locations[0].directGPR()); }); } else { slowButterflyValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationAllocateSimplePropertyStorage, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(sizeInValues)); }); } ValueFromBlock slowButterfly = m_out.anchor(slowButterflyValue); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(pointerType(), fastButterfly, slowButterfly); } LValue getById(LValue base, AccessType type) { Node* node = m_node; UniquedStringImpl* uid = m_graph.identifiers()[node->identifierNumber()]; B3::PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendSomeRegister(base); patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister)); // FIXME: If this is a GetByIdFlush, we might get some performance boost if we claim that it // clobbers volatile registers late. It's not necessary for correctness, though, since the // IC code is super smart about saving registers. // https://bugs.webkit.org/show_bug.cgi?id=152848 patchpoint->clobber(RegisterSet::macroScratchRegisters()); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); CallSiteIndex callSiteIndex = state->jitCode->common.addUniqueCallSiteIndex(node->origin.semantic); // This is the direct exit target for operation calls. Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); // This is the exit for call IC's created by the getById for getters. We don't have // to do anything weird other than call this, since it will associate the exit with // the callsite index. exceptionHandle->scheduleExitCreationForUnwind(params, callSiteIndex); auto generator = Box::create( jit.codeBlock(), node->origin.semantic, callSiteIndex, params.unavailableRegisters(), uid, JSValueRegs(params[1].gpr()), JSValueRegs(params[0].gpr()), type); generator->generateFastPath(jit); CCallHelpers::Label done = jit.label(); params.addLatePath( [=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); J_JITOperation_ESsiJI optimizationFunction; if (type == AccessType::Get) optimizationFunction = operationGetByIdOptimize; else optimizationFunction = operationTryGetByIdOptimize; generator->slowPathJump().link(&jit); CCallHelpers::Label slowPathBegin = jit.label(); CCallHelpers::Call slowPathCall = callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), optimizationFunction, params[0].gpr(), CCallHelpers::TrustedImmPtr(generator->stubInfo()), params[1].gpr(), CCallHelpers::TrustedImmPtr(uid)).call(); jit.jump().linkTo(done, &jit); generator->reportSlowPathCall(slowPathBegin, slowPathCall); jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { generator->finalize(linkBuffer); }); }); }); return patchpoint; } LValue isFastTypedArray(LValue object) { return m_out.equal( m_out.load32(object, m_heaps.JSArrayBufferView_mode), m_out.constInt32(FastTypedArray)); } TypedPointer baseIndex(IndexedAbstractHeap& heap, LValue storage, LValue index, Edge edge, ptrdiff_t offset = 0) { return m_out.baseIndex( heap, storage, m_out.zeroExtPtr(index), provenValue(edge), offset); } template void compare( const IntFunctor& intFunctor, const DoubleFunctor& doubleFunctor, C_JITOperation_TT stringIdentFunction, C_JITOperation_B_EJssJss stringFunction, S_JITOperation_EJJ fallbackFunction) { if (m_node->isBinaryUseKind(Int32Use)) { LValue left = lowInt32(m_node->child1()); LValue right = lowInt32(m_node->child2()); setBoolean(intFunctor(left, right)); return; } if (m_node->isBinaryUseKind(Int52RepUse)) { Int52Kind kind; LValue left = lowWhicheverInt52(m_node->child1(), kind); LValue right = lowInt52(m_node->child2(), kind); setBoolean(intFunctor(left, right)); return; } if (m_node->isBinaryUseKind(DoubleRepUse)) { LValue left = lowDouble(m_node->child1()); LValue right = lowDouble(m_node->child2()); setBoolean(doubleFunctor(left, right)); return; } if (m_node->isBinaryUseKind(StringIdentUse)) { LValue left = lowStringIdent(m_node->child1()); LValue right = lowStringIdent(m_node->child2()); setBoolean(m_out.callWithoutSideEffects(Int32, stringIdentFunction, left, right)); return; } if (m_node->isBinaryUseKind(StringUse)) { LValue left = lowCell(m_node->child1()); LValue right = lowCell(m_node->child2()); speculateString(m_node->child1(), left); speculateString(m_node->child2(), right); LValue result = vmCall( Int32, m_out.operation(stringFunction), m_callFrame, left, right); setBoolean(result); return; } DFG_ASSERT(m_graph, m_node, m_node->isBinaryUseKind(UntypedUse)); nonSpeculativeCompare(intFunctor, fallbackFunction); } void compileToLowerCase() { LBasicBlock notRope = m_out.newBlock(); LBasicBlock is8Bit = m_out.newBlock(); LBasicBlock loopTop = m_out.newBlock(); LBasicBlock loopBody = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue string = lowString(m_node->child1()); ValueFromBlock startIndex = m_out.anchor(m_out.constInt32(0)); ValueFromBlock startIndexForCall = m_out.anchor(m_out.constInt32(0)); LValue impl = m_out.loadPtr(string, m_heaps.JSString_value); m_out.branch(m_out.isZero64(impl), unsure(slowPath), unsure(notRope)); LBasicBlock lastNext = m_out.appendTo(notRope, is8Bit); m_out.branch( m_out.testIsZero32( m_out.load32(impl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), unsure(slowPath), unsure(is8Bit)); m_out.appendTo(is8Bit, loopTop); LValue length = m_out.load32(impl, m_heaps.StringImpl_length); LValue buffer = m_out.loadPtr(impl, m_heaps.StringImpl_data); ValueFromBlock fastResult = m_out.anchor(string); m_out.jump(loopTop); m_out.appendTo(loopTop, loopBody); LValue index = m_out.phi(Int32, startIndex); ValueFromBlock indexFromBlock = m_out.anchor(index); m_out.branch(m_out.below(index, length), unsure(loopBody), unsure(continuation)); m_out.appendTo(loopBody, slowPath); LValue byte = m_out.load8ZeroExt32(m_out.baseIndex(m_heaps.characters8, buffer, m_out.zeroExtPtr(index))); LValue isInvalidAsciiRange = m_out.bitAnd(byte, m_out.constInt32(~0x7F)); LValue isUpperCase = m_out.belowOrEqual(m_out.sub(byte, m_out.constInt32('A')), m_out.constInt32('Z' - 'A')); LValue isBadCharacter = m_out.bitOr(isInvalidAsciiRange, isUpperCase); m_out.addIncomingToPhi(index, m_out.anchor(m_out.add(index, m_out.int32One))); m_out.branch(isBadCharacter, unsure(slowPath), unsure(loopTop)); m_out.appendTo(slowPath, continuation); LValue slowPathIndex = m_out.phi(Int32, startIndexForCall, indexFromBlock); ValueFromBlock slowResult = m_out.anchor(vmCall(pointerType(), m_out.operation(operationToLowerCase), m_callFrame, string, slowPathIndex)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setJSValue(m_out.phi(pointerType(), fastResult, slowResult)); } void compileNumberToStringWithRadix() { bool validRadixIsGuaranteed = false; if (m_node->child2()->isInt32Constant()) { int32_t radix = m_node->child2()->asInt32(); if (radix >= 2 && radix <= 36) validRadixIsGuaranteed = true; } switch (m_node->child1().useKind()) { case Int32Use: setJSValue(vmCall(pointerType(), m_out.operation(validRadixIsGuaranteed ? operationInt32ToStringWithValidRadix : operationInt32ToString), m_callFrame, lowInt32(m_node->child1()), lowInt32(m_node->child2()))); break; case Int52RepUse: setJSValue(vmCall(pointerType(), m_out.operation(validRadixIsGuaranteed ? operationInt52ToStringWithValidRadix : operationInt52ToString), m_callFrame, lowStrictInt52(m_node->child1()), lowInt32(m_node->child2()))); break; case DoubleRepUse: setJSValue(vmCall(pointerType(), m_out.operation(validRadixIsGuaranteed ? operationDoubleToStringWithValidRadix : operationDoubleToString), m_callFrame, lowDouble(m_node->child1()), lowInt32(m_node->child2()))); break; default: RELEASE_ASSERT_NOT_REACHED(); } } void compileResolveScope() { UniquedStringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()]; setJSValue(vmCall(pointerType(), m_out.operation(operationResolveScope), m_callFrame, lowCell(m_node->child1()), m_out.constIntPtr(uid))); } void compileGetDynamicVar() { UniquedStringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()]; setJSValue(vmCall(Int64, m_out.operation(operationGetDynamicVar), m_callFrame, lowCell(m_node->child1()), m_out.constIntPtr(uid), m_out.constInt32(m_node->getPutInfo()))); } void compilePutDynamicVar() { UniquedStringImpl* uid = m_graph.identifiers()[m_node->identifierNumber()]; setJSValue(vmCall(Void, m_out.operation(operationPutDynamicVar), m_callFrame, lowCell(m_node->child1()), lowJSValue(m_node->child2()), m_out.constIntPtr(uid), m_out.constInt32(m_node->getPutInfo()))); } void compileUnreachable() { // It's so tempting to assert that AI has proved that this is unreachable. But that's // simply not a requirement of the Unreachable opcode at all. If you emit an opcode that // *you* know will not return, then it's fine to end the basic block with Unreachable // after that opcode. You don't have to also prove to AI that your opcode does not return. // Hence, there is nothing to do here but emit code that will crash, so that we catch // cases where you said Unreachable but you lied. crash(); } void compileCheckDOM() { LValue cell = lowCell(m_node->child1()); DOMJIT::Patchpoint* domJIT = m_node->checkDOMPatchpoint(); PatchpointValue* patchpoint = m_out.patchpoint(Void); patchpoint->appendSomeRegister(cell); patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister)); NodeOrigin origin = m_origin; unsigned osrExitArgumentOffset = patchpoint->numChildren(); OSRExitDescriptor* exitDescriptor = appendOSRExitDescriptor(jsValueValue(cell), m_node->child1().node()); patchpoint->appendColdAnys(buildExitArguments(exitDescriptor, origin.forExit, jsValueValue(cell))); patchpoint->numGPScratchRegisters = domJIT->numGPScratchRegisters; patchpoint->numFPScratchRegisters = domJIT->numFPScratchRegisters; patchpoint->clobber(RegisterSet::macroScratchRegisters()); State* state = &m_ftlState; Node* node = m_node; JSValue child1Constant = m_state.forNode(m_node->child1()).value(); patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); Vector gpScratch; Vector fpScratch; Vector regs; regs.append(DOMJIT::Value(params[0].gpr(), child1Constant)); for (unsigned i = 0; i < domJIT->numGPScratchRegisters; ++i) gpScratch.append(params.gpScratch(i)); for (unsigned i = 0; i < domJIT->numFPScratchRegisters; ++i) fpScratch.append(params.fpScratch(i)); RefPtr handle = exitDescriptor->emitOSRExitLater(*state, BadType, origin, params, osrExitArgumentOffset); DOMJITPatchpointParams domJITParams(*state, params, node, nullptr, WTFMove(regs), WTFMove(gpScratch), WTFMove(fpScratch)); CCallHelpers::JumpList failureCases = domJIT->generator()->run(jit, domJITParams); jit.addLinkTask([=] (LinkBuffer& linkBuffer) { linkBuffer.link(failureCases, linkBuffer.locationOf(handle->label)); }); }); patchpoint->effects = Effects::forCheck(); } void compileCallDOM() { const DOMJIT::Signature* signature = m_node->signature(); // FIXME: We should have a way to call functions with the vector of registers. // https://bugs.webkit.org/show_bug.cgi?id=163099 Vector operands; unsigned index = 0; DFG_NODE_DO_TO_CHILDREN(m_graph, m_node, [&](Node*, Edge edge) { if (!index) operands.append(lowCell(edge)); else { switch (signature->arguments[index - 1]) { case SpecString: operands.append(lowString(edge)); break; case SpecInt32Only: operands.append(lowInt32(edge)); break; case SpecBoolean: operands.append(lowBoolean(edge)); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } } ++index; }); unsigned argumentCountIncludingThis = signature->argumentCount + 1; LValue result; switch (argumentCountIncludingThis) { case 1: result = vmCall(Int64, m_out.operation(reinterpret_cast(signature->unsafeFunction)), m_callFrame, operands[0]); break; case 2: result = vmCall(Int64, m_out.operation(reinterpret_cast(signature->unsafeFunction)), m_callFrame, operands[0], operands[1]); break; case 3: result = vmCall(Int64, m_out.operation(reinterpret_cast(signature->unsafeFunction)), m_callFrame, operands[0], operands[1], operands[2]); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } setJSValue(result); } void compileCallDOMGetter() { DOMJIT::CallDOMGetterPatchpoint* domJIT = m_node->callDOMGetterData()->patchpoint; Edge& baseEdge = m_node->child1(); LValue base = lowCell(baseEdge); JSValue baseConstant = m_state.forNode(baseEdge).value(); LValue globalObject; JSValue globalObjectConstant; if (domJIT->requireGlobalObject) { Edge& globalObjectEdge = m_node->child2(); globalObject = lowCell(globalObjectEdge); globalObjectConstant = m_state.forNode(globalObjectEdge).value(); } PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendSomeRegister(base); if (domJIT->requireGlobalObject) patchpoint->appendSomeRegister(globalObject); patchpoint->append(m_tagMask, ValueRep::reg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::reg(GPRInfo::tagTypeNumberRegister)); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->numGPScratchRegisters = domJIT->numGPScratchRegisters; patchpoint->numFPScratchRegisters = domJIT->numFPScratchRegisters; patchpoint->resultConstraint = ValueRep::SomeEarlyRegister; State* state = &m_ftlState; Node* node = m_node; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); Vector gpScratch; Vector fpScratch; Vector regs; regs.append(JSValueRegs(params[0].gpr())); regs.append(DOMJIT::Value(params[1].gpr(), baseConstant)); if (domJIT->requireGlobalObject) regs.append(DOMJIT::Value(params[2].gpr(), globalObjectConstant)); for (unsigned i = 0; i < domJIT->numGPScratchRegisters; ++i) gpScratch.append(params.gpScratch(i)); for (unsigned i = 0; i < domJIT->numFPScratchRegisters; ++i) fpScratch.append(params.fpScratch(i)); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); DOMJITPatchpointParams domJITParams(*state, params, node, exceptions, WTFMove(regs), WTFMove(gpScratch), WTFMove(fpScratch)); domJIT->generator()->run(jit, domJITParams); }); patchpoint->effects = Effects::forCall(); setJSValue(patchpoint); } void compareEqObjectOrOtherToObject(Edge leftChild, Edge rightChild) { LValue rightCell = lowCell(rightChild); LValue leftValue = lowJSValue(leftChild, ManualOperandSpeculation); speculateTruthyObject(rightChild, rightCell, SpecObject); LBasicBlock leftCellCase = m_out.newBlock(); LBasicBlock leftNotCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( isCell(leftValue, provenType(leftChild)), unsure(leftCellCase), unsure(leftNotCellCase)); LBasicBlock lastNext = m_out.appendTo(leftCellCase, leftNotCellCase); speculateTruthyObject(leftChild, leftValue, SpecObject | (~SpecCell)); ValueFromBlock cellResult = m_out.anchor(m_out.equal(rightCell, leftValue)); m_out.jump(continuation); m_out.appendTo(leftNotCellCase, continuation); FTL_TYPE_CHECK( jsValueValue(leftValue), leftChild, SpecOther | SpecCell, isNotOther(leftValue)); ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, cellResult, notCellResult)); } void speculateTruthyObject(Edge edge, LValue cell, SpeculatedType filter) { if (masqueradesAsUndefinedWatchpointIsStillValid()) { FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell)); return; } FTL_TYPE_CHECK(jsValueValue(cell), edge, filter, isNotObject(cell)); speculate( BadType, jsValueValue(cell), edge.node(), m_out.testNonZero32( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoFlags), m_out.constInt32(MasqueradesAsUndefined))); } template void nonSpeculativeCompare(const IntFunctor& intFunctor, S_JITOperation_EJJ helperFunction) { LValue left = lowJSValue(m_node->child1()); LValue right = lowJSValue(m_node->child2()); LBasicBlock leftIsInt = m_out.newBlock(); LBasicBlock fastPath = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isNotInt32(left, provenType(m_node->child1())), rarely(slowPath), usually(leftIsInt)); LBasicBlock lastNext = m_out.appendTo(leftIsInt, fastPath); m_out.branch(isNotInt32(right, provenType(m_node->child2())), rarely(slowPath), usually(fastPath)); m_out.appendTo(fastPath, slowPath); ValueFromBlock fastResult = m_out.anchor(intFunctor(unboxInt32(left), unboxInt32(right))); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); ValueFromBlock slowResult = m_out.anchor(m_out.notNull(vmCall( pointerType(), m_out.operation(helperFunction), m_callFrame, left, right))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); setBoolean(m_out.phi(Int32, fastResult, slowResult)); } LValue stringsEqual(LValue leftJSString, LValue rightJSString) { LBasicBlock notTriviallyUnequalCase = m_out.newBlock(); LBasicBlock notEmptyCase = m_out.newBlock(); LBasicBlock leftReadyCase = m_out.newBlock(); LBasicBlock rightReadyCase = m_out.newBlock(); LBasicBlock left8BitCase = m_out.newBlock(); LBasicBlock right8BitCase = m_out.newBlock(); LBasicBlock loop = m_out.newBlock(); LBasicBlock bytesEqual = m_out.newBlock(); LBasicBlock trueCase = m_out.newBlock(); LBasicBlock falseCase = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue length = m_out.load32(leftJSString, m_heaps.JSString_length); m_out.branch( m_out.notEqual(length, m_out.load32(rightJSString, m_heaps.JSString_length)), unsure(falseCase), unsure(notTriviallyUnequalCase)); LBasicBlock lastNext = m_out.appendTo(notTriviallyUnequalCase, notEmptyCase); m_out.branch(m_out.isZero32(length), unsure(trueCase), unsure(notEmptyCase)); m_out.appendTo(notEmptyCase, leftReadyCase); LValue left = m_out.loadPtr(leftJSString, m_heaps.JSString_value); LValue right = m_out.loadPtr(rightJSString, m_heaps.JSString_value); m_out.branch(m_out.notNull(left), usually(leftReadyCase), rarely(slowCase)); m_out.appendTo(leftReadyCase, rightReadyCase); m_out.branch(m_out.notNull(right), usually(rightReadyCase), rarely(slowCase)); m_out.appendTo(rightReadyCase, left8BitCase); m_out.branch( m_out.testIsZero32( m_out.load32(left, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), unsure(slowCase), unsure(left8BitCase)); m_out.appendTo(left8BitCase, right8BitCase); m_out.branch( m_out.testIsZero32( m_out.load32(right, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), unsure(slowCase), unsure(right8BitCase)); m_out.appendTo(right8BitCase, loop); LValue leftData = m_out.loadPtr(left, m_heaps.StringImpl_data); LValue rightData = m_out.loadPtr(right, m_heaps.StringImpl_data); ValueFromBlock indexAtStart = m_out.anchor(length); m_out.jump(loop); m_out.appendTo(loop, bytesEqual); LValue indexAtLoopTop = m_out.phi(Int32, indexAtStart); LValue indexInLoop = m_out.sub(indexAtLoopTop, m_out.int32One); LValue leftByte = m_out.load8ZeroExt32( m_out.baseIndex(m_heaps.characters8, leftData, m_out.zeroExtPtr(indexInLoop))); LValue rightByte = m_out.load8ZeroExt32( m_out.baseIndex(m_heaps.characters8, rightData, m_out.zeroExtPtr(indexInLoop))); m_out.branch(m_out.notEqual(leftByte, rightByte), unsure(falseCase), unsure(bytesEqual)); m_out.appendTo(bytesEqual, trueCase); ValueFromBlock indexForNextIteration = m_out.anchor(indexInLoop); m_out.addIncomingToPhi(indexAtLoopTop, indexForNextIteration); m_out.branch(m_out.notZero32(indexInLoop), unsure(loop), unsure(trueCase)); m_out.appendTo(trueCase, falseCase); ValueFromBlock trueResult = m_out.anchor(m_out.booleanTrue); m_out.jump(continuation); m_out.appendTo(falseCase, slowCase); ValueFromBlock falseResult = m_out.anchor(m_out.booleanFalse); m_out.jump(continuation); m_out.appendTo(slowCase, continuation); LValue slowResultValue = vmCall( Int64, m_out.operation(operationCompareStringEq), m_callFrame, leftJSString, rightJSString); ValueFromBlock slowResult = m_out.anchor(unboxBoolean(slowResultValue)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int32, trueResult, falseResult, slowResult); } enum ScratchFPRUsage { DontNeedScratchFPR, NeedScratchFPR }; template void emitBinarySnippet(J_JITOperation_EJJ slowPathFunction) { Node* node = m_node; LValue left = lowJSValue(node->child1()); LValue right = lowJSValue(node->child2()); SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType()); SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType()); PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendSomeRegister(left); patchpoint->appendSomeRegister(right); patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister)); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->numGPScratchRegisters = 1; patchpoint->numFPScratchRegisters = 2; if (scratchFPRUsage == NeedScratchFPR) patchpoint->numFPScratchRegisters++; patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->resultConstraint = ValueRep::SomeEarlyRegister; State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); auto generator = Box::create( leftOperand, rightOperand, JSValueRegs(params[0].gpr()), JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()), params.fpScratch(0), params.fpScratch(1), params.gpScratch(0), scratchFPRUsage == NeedScratchFPR ? params.fpScratch(2) : InvalidFPRReg); generator->generateFastPath(jit); if (generator->didEmitFastPath()) { generator->endJumpList().link(&jit); CCallHelpers::Label done = jit.label(); params.addLatePath( [=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); generator->slowPathJumpList().link(&jit); callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), slowPathFunction, params[0].gpr(), params[1].gpr(), params[2].gpr()); jit.jump().linkTo(done, &jit); }); } else { callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), slowPathFunction, params[0].gpr(), params[1].gpr(), params[2].gpr()); } }); setJSValue(patchpoint); } template void emitBinaryBitOpSnippet(J_JITOperation_EJJ slowPathFunction) { Node* node = m_node; LValue left = lowJSValue(node->child1()); LValue right = lowJSValue(node->child2()); SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType()); SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType()); PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendSomeRegister(left); patchpoint->appendSomeRegister(right); patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister)); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->numGPScratchRegisters = 1; patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->resultConstraint = ValueRep::SomeEarlyRegister; State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); auto generator = Box::create( leftOperand, rightOperand, JSValueRegs(params[0].gpr()), JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()), params.gpScratch(0)); generator->generateFastPath(jit); generator->endJumpList().link(&jit); CCallHelpers::Label done = jit.label(); params.addLatePath( [=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); generator->slowPathJumpList().link(&jit); callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), slowPathFunction, params[0].gpr(), params[1].gpr(), params[2].gpr()); jit.jump().linkTo(done, &jit); }); }); setJSValue(patchpoint); } void emitRightShiftSnippet(JITRightShiftGenerator::ShiftType shiftType) { Node* node = m_node; // FIXME: Make this do exceptions. // https://bugs.webkit.org/show_bug.cgi?id=151686 LValue left = lowJSValue(node->child1()); LValue right = lowJSValue(node->child2()); SnippetOperand leftOperand(m_state.forNode(node->child1()).resultType()); SnippetOperand rightOperand(m_state.forNode(node->child2()).resultType()); PatchpointValue* patchpoint = m_out.patchpoint(Int64); patchpoint->appendSomeRegister(left); patchpoint->appendSomeRegister(right); patchpoint->append(m_tagMask, ValueRep::lateReg(GPRInfo::tagMaskRegister)); patchpoint->append(m_tagTypeNumber, ValueRep::lateReg(GPRInfo::tagTypeNumberRegister)); RefPtr exceptionHandle = preparePatchpointForExceptions(patchpoint); patchpoint->numGPScratchRegisters = 1; patchpoint->numFPScratchRegisters = 1; patchpoint->clobber(RegisterSet::macroScratchRegisters()); patchpoint->resultConstraint = ValueRep::SomeEarlyRegister; State* state = &m_ftlState; patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { AllowMacroScratchRegisterUsage allowScratch(jit); Box exceptions = exceptionHandle->scheduleExitCreation(params)->jumps(jit); auto generator = Box::create( leftOperand, rightOperand, JSValueRegs(params[0].gpr()), JSValueRegs(params[1].gpr()), JSValueRegs(params[2].gpr()), params.fpScratch(0), params.gpScratch(0), InvalidFPRReg, shiftType); generator->generateFastPath(jit); generator->endJumpList().link(&jit); CCallHelpers::Label done = jit.label(); params.addLatePath( [=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); generator->slowPathJumpList().link(&jit); J_JITOperation_EJJ slowPathFunction = shiftType == JITRightShiftGenerator::SignedShift ? operationValueBitRShift : operationValueBitURShift; callOperation( *state, params.unavailableRegisters(), jit, node->origin.semantic, exceptions.get(), slowPathFunction, params[0].gpr(), params[1].gpr(), params[2].gpr()); jit.jump().linkTo(done, &jit); }); }); setJSValue(patchpoint); } LValue allocateHeapCell(LValue allocator, LBasicBlock slowPath) { MarkedAllocator* actualAllocator = nullptr; if (allocator->hasIntPtr()) actualAllocator = bitwise_cast(allocator->asIntPtr()); if (!actualAllocator) { // This means that either we know that the allocator is null or we don't know what the // allocator is. In either case, we need the null check. LBasicBlock haveAllocator = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(haveAllocator); m_out.branch(allocator, usually(haveAllocator), rarely(slowPath)); m_out.appendTo(haveAllocator, lastNext); } LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); PatchpointValue* patchpoint = m_out.patchpoint(pointerType()); patchpoint->effects.terminal = true; patchpoint->appendSomeRegister(allocator); patchpoint->numGPScratchRegisters++; patchpoint->resultConstraint = ValueRep::SomeEarlyRegister; m_out.appendSuccessor(usually(continuation)); m_out.appendSuccessor(rarely(slowPath)); patchpoint->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { CCallHelpers::JumpList jumpToSlowPath; // We use a patchpoint to emit the allocation path because whenever we mess with // allocation paths, we already reason about them at the machine code level. We know // exactly what instruction sequence we want. We're confident that no compiler // optimization could make this code better. So, it's best to have the code in // AssemblyHelpers::emitAllocate(). That way, the same optimized path is shared by // all of the compiler tiers. jit.emitAllocateWithNonNullAllocator( params[0].gpr(), actualAllocator, params[1].gpr(), params.gpScratch(0), jumpToSlowPath); CCallHelpers::Jump jumpToSuccess; if (!params.fallsThroughToSuccessor(0)) jumpToSuccess = jit.jump(); Vector> labels = params.successorLabels(); params.addLatePath( [=] (CCallHelpers& jit) { jumpToSlowPath.linkTo(*labels[1], &jit); if (jumpToSuccess.isSet()) jumpToSuccess.linkTo(*labels[0], &jit); }); }); m_out.appendTo(continuation, lastNext); return patchpoint; } void storeStructure(LValue object, Structure* structure) { m_out.store32(m_out.constInt32(structure->id()), object, m_heaps.JSCell_structureID); m_out.store32( m_out.constInt32(structure->objectInitializationBlob()), object, m_heaps.JSCell_usefulBytes); } void storeStructure(LValue object, LValue structure) { if (structure->hasIntPtr()) { storeStructure(object, bitwise_cast(structure->asIntPtr())); return; } LValue id = m_out.load32(structure, m_heaps.Structure_structureID); m_out.store32(id, object, m_heaps.JSCell_structureID); LValue blob = m_out.load32(structure, m_heaps.Structure_indexingTypeIncludingHistory); m_out.store32(blob, object, m_heaps.JSCell_usefulBytes); } template LValue allocateCell(LValue allocator, StructureType structure, LBasicBlock slowPath) { LValue result = allocateHeapCell(allocator, slowPath); storeStructure(result, structure); return result; } LValue allocateObject(LValue allocator, RegisteredStructure structure, LValue butterfly, LBasicBlock slowPath) { return allocateObject(allocator, weakStructure(structure), butterfly, slowPath); } LValue allocateObject(LValue allocator, LValue structure, LValue butterfly, LBasicBlock slowPath) { LValue result = allocateCell(allocator, structure, slowPath); if (structure->hasIntPtr()) { splatWords( result, m_out.constInt32(JSFinalObject::offsetOfInlineStorage() / 8), m_out.constInt32(JSFinalObject::offsetOfInlineStorage() / 8 + bitwise_cast(structure->asIntPtr())->inlineCapacity()), m_out.int64Zero, m_heaps.properties.atAnyNumber()); } else { LValue end = m_out.add( m_out.constInt32(JSFinalObject::offsetOfInlineStorage() / 8), m_out.load8ZeroExt32(structure, m_heaps.Structure_inlineCapacity)); splatWords( result, m_out.constInt32(JSFinalObject::offsetOfInlineStorage() / 8), end, m_out.int64Zero, m_heaps.properties.atAnyNumber()); } m_out.storePtr(butterfly, result, m_heaps.JSObject_butterfly); return result; } template LValue allocateObject( size_t size, StructureType structure, LValue butterfly, LBasicBlock slowPath) { MarkedAllocator* allocator = subspaceFor(vm())->allocatorFor(size); return allocateObject(m_out.constIntPtr(allocator), structure, butterfly, slowPath); } template LValue allocateObject(StructureType structure, LValue butterfly, LBasicBlock slowPath) { return allocateObject( ClassType::allocationSize(0), structure, butterfly, slowPath); } LValue allocatorForSize(LValue subspace, LValue size, LBasicBlock slowPath) { static_assert(!(MarkedSpace::sizeStep & (MarkedSpace::sizeStep - 1)), "MarkedSpace::sizeStep must be a power of two."); // Try to do some constant-folding here. if (subspace->hasIntPtr() && size->hasIntPtr()) { Subspace* actualSubspace = bitwise_cast(subspace->asIntPtr()); size_t actualSize = size->asIntPtr(); MarkedAllocator* actualAllocator = actualSubspace->allocatorFor(actualSize); if (!actualAllocator) { LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); m_out.jump(slowPath); m_out.appendTo(continuation, lastNext); return m_out.intPtrZero; } return m_out.constIntPtr(actualAllocator); } unsigned stepShift = getLSBSet(MarkedSpace::sizeStep); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); LValue sizeClassIndex = m_out.lShr( m_out.add(size, m_out.constIntPtr(MarkedSpace::sizeStep - 1)), m_out.constInt32(stepShift)); m_out.branch( m_out.above(sizeClassIndex, m_out.constIntPtr(MarkedSpace::largeCutoff >> stepShift)), rarely(slowPath), usually(continuation)); m_out.appendTo(continuation, lastNext); return m_out.loadPtr( m_out.baseIndex( m_heaps.Subspace_allocatorForSizeStep, subspace, m_out.sub(sizeClassIndex, m_out.intPtrOne))); } LValue allocatorForSize(Subspace& subspace, LValue size, LBasicBlock slowPath) { return allocatorForSize(m_out.constIntPtr(&subspace), size, slowPath); } template LValue allocateVariableSizedObject( LValue size, RegisteredStructure structure, LValue butterfly, LBasicBlock slowPath) { LValue allocator = allocatorForSize( *subspaceFor(vm()), size, slowPath); return allocateObject(allocator, structure, butterfly, slowPath); } template LValue allocateVariableSizedCell( LValue size, Structure* structure, LBasicBlock slowPath) { LValue allocator = allocatorForSize( *subspaceFor(vm()), size, slowPath); return allocateCell(allocator, structure, slowPath); } LValue allocateObject(RegisteredStructure structure) { size_t allocationSize = JSFinalObject::allocationSize(structure.get()->inlineCapacity()); MarkedAllocator* allocator = subspaceFor(vm())->allocatorFor(allocationSize); // FIXME: If the allocator is null, we could simply emit a normal C call to the allocator // instead of putting it on the slow path. // https://bugs.webkit.org/show_bug.cgi?id=161062 LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); ValueFromBlock fastResult = m_out.anchor(allocateObject( m_out.constIntPtr(allocator), structure, m_out.intPtrZero, slowPath)); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); LValue slowResultValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationNewObject, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(structure.get())); }); ValueFromBlock slowResult = m_out.anchor(slowResultValue); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(pointerType(), fastResult, slowResult); } struct ArrayValues { ArrayValues() : array(0) , butterfly(0) { } ArrayValues(LValue array, LValue butterfly) : array(array) , butterfly(butterfly) { } LValue array; LValue butterfly; }; ArrayValues allocateJSArray(LValue publicLength, LValue structure, LValue indexingType, bool shouldInitializeElements = true, bool shouldLargeArraySizeCreateArrayStorage = true) { JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); if (indexingType->hasInt32()) { IndexingType type = static_cast(indexingType->asInt32()); ASSERT_UNUSED(type, hasUndecided(type) || hasInt32(type) || hasDouble(type) || hasContiguous(type)); } LBasicBlock fastCase = m_out.newBlock(); LBasicBlock largeCase = m_out.newBlock(); LBasicBlock failCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock slowCase = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(fastCase); ValueFromBlock noButterfly = m_out.anchor(m_out.intPtrZero); LValue predicate; if (shouldLargeArraySizeCreateArrayStorage) predicate = m_out.aboveOrEqual(publicLength, m_out.constInt32(MIN_ARRAY_STORAGE_CONSTRUCTION_LENGTH)); else predicate = m_out.booleanFalse; m_out.branch(predicate, rarely(largeCase), usually(fastCase)); m_out.appendTo(fastCase, largeCase); LValue vectorLength = nullptr; if (publicLength->hasInt32() && structure->hasIntPtr()) { unsigned publicLengthConst = static_cast(publicLength->asInt32()); if (publicLengthConst <= MAX_STORAGE_VECTOR_LENGTH) { vectorLength = m_out.constInt32( Butterfly::optimalContiguousVectorLength( bitwise_cast(structure->asIntPtr())->outOfLineCapacity(), publicLengthConst)); } } if (!vectorLength) { // We don't compute the optimal vector length for new Array(blah) where blah is not // statically known, since the compute effort of doing it here is probably not worth it. vectorLength = publicLength; } LValue payloadSize = m_out.shl(m_out.zeroExt(vectorLength, pointerType()), m_out.constIntPtr(3)); LValue butterflySize = m_out.add( payloadSize, m_out.constIntPtr(sizeof(IndexingHeader))); LValue allocator = allocatorForSize(vm().auxiliarySpace, butterflySize, failCase); LValue startOfStorage = allocateHeapCell(allocator, failCase); LValue butterfly = m_out.add(startOfStorage, m_out.constIntPtr(sizeof(IndexingHeader))); m_out.store32(publicLength, butterfly, m_heaps.Butterfly_publicLength); m_out.store32(vectorLength, butterfly, m_heaps.Butterfly_vectorLength); initializeArrayElements( indexingType, shouldInitializeElements ? m_out.int32Zero : publicLength, vectorLength, butterfly); ValueFromBlock haveButterfly = m_out.anchor(butterfly); LValue object = allocateObject(structure, butterfly, failCase); ValueFromBlock fastResult = m_out.anchor(object); ValueFromBlock fastButterfly = m_out.anchor(butterfly); m_out.jump(continuation); m_out.appendTo(largeCase, failCase); ValueFromBlock largeStructure = m_out.anchor( weakStructure(m_graph.registerStructure(globalObject->arrayStructureForIndexingTypeDuringAllocation(ArrayWithArrayStorage)))); m_out.jump(slowCase); m_out.appendTo(failCase, slowCase); ValueFromBlock failStructure = m_out.anchor(structure); m_out.jump(slowCase); m_out.appendTo(slowCase, continuation); LValue structureValue = m_out.phi(pointerType(), largeStructure, failStructure); LValue butterflyValue = m_out.phi(pointerType(), noButterfly, haveButterfly); LValue slowResultValue = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationNewArrayWithSize, locations[0].directGPR(), locations[1].directGPR(), locations[2].directGPR(), locations[3].directGPR()); }, structureValue, publicLength, butterflyValue); ValueFromBlock slowResult = m_out.anchor(slowResultValue); ValueFromBlock slowButterfly = m_out.anchor( m_out.loadPtr(slowResultValue, m_heaps.JSObject_butterfly)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return ArrayValues( m_out.phi(pointerType(), fastResult, slowResult), m_out.phi(pointerType(), fastButterfly, slowButterfly)); } ArrayValues allocateUninitializedContiguousJSArray(LValue publicLength, RegisteredStructure structure) { bool shouldInitializeElements = false; bool shouldLargeArraySizeCreateArrayStorage = false; return allocateJSArray( publicLength, weakStructure(structure), m_out.constInt32(structure->indexingType()), shouldInitializeElements, shouldLargeArraySizeCreateArrayStorage); } LValue ensureShadowChickenPacket() { LBasicBlock slowCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); TypedPointer addressOfLogCursor = m_out.absolute(vm().shadowChicken().addressOfLogCursor()); LValue logCursor = m_out.loadPtr(addressOfLogCursor); ValueFromBlock fastResult = m_out.anchor(logCursor); m_out.branch( m_out.below(logCursor, m_out.constIntPtr(vm().shadowChicken().logEnd())), usually(continuation), rarely(slowCase)); LBasicBlock lastNext = m_out.appendTo(slowCase, continuation); vmCall(Void, m_out.operation(operationProcessShadowChickenLog), m_callFrame); ValueFromBlock slowResult = m_out.anchor(m_out.loadPtr(addressOfLogCursor)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); LValue result = m_out.phi(pointerType(), fastResult, slowResult); m_out.storePtr( m_out.add(result, m_out.constIntPtr(sizeof(ShadowChicken::Packet))), addressOfLogCursor); return result; } LValue boolify(Edge edge) { switch (edge.useKind()) { case BooleanUse: case KnownBooleanUse: return lowBoolean(edge); case Int32Use: return m_out.notZero32(lowInt32(edge)); case DoubleRepUse: return m_out.doubleNotEqualAndOrdered(lowDouble(edge), m_out.doubleZero); case ObjectOrOtherUse: return m_out.logicalNot( equalNullOrUndefined( edge, CellCaseSpeculatesObject, SpeculateNullOrUndefined, ManualOperandSpeculation)); case StringUse: { LValue stringValue = lowString(edge); LValue length = m_out.load32NonNegative(stringValue, m_heaps.JSString_length); return m_out.notEqual(length, m_out.int32Zero); } case StringOrOtherUse: { LValue value = lowJSValue(edge, ManualOperandSpeculation); LBasicBlock cellCase = m_out.newBlock(); LBasicBlock notCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase); FTL_TYPE_CHECK(jsValueValue(value), edge, (~SpecCell) | SpecString, isNotString(value)); LValue length = m_out.load32NonNegative(value, m_heaps.JSString_length); ValueFromBlock cellResult = m_out.anchor(m_out.notEqual(length, m_out.int32Zero)); m_out.jump(continuation); m_out.appendTo(notCellCase, continuation); FTL_TYPE_CHECK(jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value)); ValueFromBlock notCellResult = m_out.anchor(m_out.booleanFalse); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int32, cellResult, notCellResult); } case UntypedUse: { LValue value = lowJSValue(edge); // Implements the following control flow structure: // if (value is cell) { // if (value is string) // result = !!value->length // else { // do evil things for masquerades-as-undefined // result = true // } // } else if (value is int32) { // result = !!unboxInt32(value) // } else if (value is number) { // result = !!unboxDouble(value) // } else { // result = value == jsTrue // } LBasicBlock cellCase = m_out.newBlock(); LBasicBlock stringCase = m_out.newBlock(); LBasicBlock notStringCase = m_out.newBlock(); LBasicBlock notCellCase = m_out.newBlock(); LBasicBlock int32Case = m_out.newBlock(); LBasicBlock notInt32Case = m_out.newBlock(); LBasicBlock doubleCase = m_out.newBlock(); LBasicBlock notDoubleCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); Vector results; m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, stringCase); m_out.branch( isString(value, provenType(edge) & SpecCell), unsure(stringCase), unsure(notStringCase)); m_out.appendTo(stringCase, notStringCase); LValue nonEmptyString = m_out.notZero32( m_out.load32NonNegative(value, m_heaps.JSString_length)); results.append(m_out.anchor(nonEmptyString)); m_out.jump(continuation); m_out.appendTo(notStringCase, notCellCase); LValue isTruthyObject; if (masqueradesAsUndefinedWatchpointIsStillValid()) isTruthyObject = m_out.booleanTrue; else { LBasicBlock masqueradesCase = m_out.newBlock(); results.append(m_out.anchor(m_out.booleanTrue)); m_out.branch( m_out.testIsZero32( m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoFlags), m_out.constInt32(MasqueradesAsUndefined)), usually(continuation), rarely(masqueradesCase)); m_out.appendTo(masqueradesCase); isTruthyObject = m_out.notEqual( weakPointer(m_graph.globalObjectFor(m_node->origin.semantic)), m_out.loadPtr(loadStructure(value), m_heaps.Structure_globalObject)); } results.append(m_out.anchor(isTruthyObject)); m_out.jump(continuation); m_out.appendTo(notCellCase, int32Case); m_out.branch( isInt32(value, provenType(edge) & ~SpecCell), unsure(int32Case), unsure(notInt32Case)); m_out.appendTo(int32Case, notInt32Case); results.append(m_out.anchor(m_out.notZero32(unboxInt32(value)))); m_out.jump(continuation); m_out.appendTo(notInt32Case, doubleCase); m_out.branch( isNumber(value, provenType(edge) & ~SpecCell), unsure(doubleCase), unsure(notDoubleCase)); m_out.appendTo(doubleCase, notDoubleCase); LValue doubleIsTruthy = m_out.doubleNotEqualAndOrdered( unboxDouble(value), m_out.constDouble(0)); results.append(m_out.anchor(doubleIsTruthy)); m_out.jump(continuation); m_out.appendTo(notDoubleCase, continuation); LValue miscIsTruthy = m_out.equal( value, m_out.constInt64(JSValue::encode(jsBoolean(true)))); results.append(m_out.anchor(miscIsTruthy)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int32, results); } default: DFG_CRASH(m_graph, m_node, "Bad use kind"); return 0; } } enum StringOrObjectMode { AllCellsAreFalse, CellCaseSpeculatesObject }; enum EqualNullOrUndefinedMode { EqualNull, EqualUndefined, EqualNullOrUndefined, SpeculateNullOrUndefined }; LValue equalNullOrUndefined( Edge edge, StringOrObjectMode cellMode, EqualNullOrUndefinedMode primitiveMode, OperandSpeculationMode operandMode = AutomaticOperandSpeculation) { bool validWatchpoint = masqueradesAsUndefinedWatchpointIsStillValid(); LValue value = lowJSValue(edge, operandMode); LBasicBlock cellCase = m_out.newBlock(); LBasicBlock primitiveCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase); Vector results; switch (cellMode) { case AllCellsAreFalse: break; case CellCaseSpeculatesObject: FTL_TYPE_CHECK( jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value)); break; } if (validWatchpoint) { results.append(m_out.anchor(m_out.booleanFalse)); m_out.jump(continuation); } else { LBasicBlock masqueradesCase = m_out.newBlock(); results.append(m_out.anchor(m_out.booleanFalse)); m_out.branch( m_out.testNonZero32( m_out.load8ZeroExt32(value, m_heaps.JSCell_typeInfoFlags), m_out.constInt32(MasqueradesAsUndefined)), rarely(masqueradesCase), usually(continuation)); m_out.appendTo(masqueradesCase, primitiveCase); LValue structure = loadStructure(value); results.append(m_out.anchor( m_out.equal( weakPointer(m_graph.globalObjectFor(m_node->origin.semantic)), m_out.loadPtr(structure, m_heaps.Structure_globalObject)))); m_out.jump(continuation); } m_out.appendTo(primitiveCase, continuation); LValue primitiveResult; switch (primitiveMode) { case EqualNull: primitiveResult = m_out.equal(value, m_out.constInt64(ValueNull)); break; case EqualUndefined: primitiveResult = m_out.equal(value, m_out.constInt64(ValueUndefined)); break; case EqualNullOrUndefined: primitiveResult = isOther(value, provenType(edge)); break; case SpeculateNullOrUndefined: FTL_TYPE_CHECK( jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value)); primitiveResult = m_out.booleanTrue; break; } results.append(m_out.anchor(primitiveResult)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int32, results); } template void contiguousPutByValOutOfBounds( FunctionType slowPathFunction, LValue base, LValue storage, LValue index, LValue value, LBasicBlock continuation) { LValue isNotInBounds = m_out.aboveOrEqual( index, m_out.load32NonNegative(storage, m_heaps.Butterfly_publicLength)); if (!m_node->arrayMode().isInBounds()) { LBasicBlock notInBoundsCase = m_out.newBlock(); LBasicBlock performStore = m_out.newBlock(); m_out.branch(isNotInBounds, unsure(notInBoundsCase), unsure(performStore)); LBasicBlock lastNext = m_out.appendTo(notInBoundsCase, performStore); LValue isOutOfBounds = m_out.aboveOrEqual( index, m_out.load32NonNegative(storage, m_heaps.Butterfly_vectorLength)); if (!m_node->arrayMode().isOutOfBounds()) speculate(OutOfBounds, noValue(), 0, isOutOfBounds); else { LBasicBlock outOfBoundsCase = m_out.newBlock(); LBasicBlock holeCase = m_out.newBlock(); m_out.branch(isOutOfBounds, rarely(outOfBoundsCase), usually(holeCase)); LBasicBlock innerLastNext = m_out.appendTo(outOfBoundsCase, holeCase); vmCall( Void, m_out.operation(slowPathFunction), m_callFrame, base, index, value); m_out.jump(continuation); m_out.appendTo(holeCase, innerLastNext); } m_out.store32( m_out.add(index, m_out.int32One), storage, m_heaps.Butterfly_publicLength); m_out.jump(performStore); m_out.appendTo(performStore, lastNext); } } void buildSwitch(SwitchData* data, LType type, LValue switchValue) { ASSERT(type == pointerType() || type == Int32); Vector cases; for (unsigned i = 0; i < data->cases.size(); ++i) { SwitchCase newCase; if (type == pointerType()) { newCase = SwitchCase(m_out.constIntPtr(data->cases[i].value.switchLookupValue(data->kind)), lowBlock(data->cases[i].target.block), Weight(data->cases[i].target.count)); } else if (type == Int32) { newCase = SwitchCase(m_out.constInt32(data->cases[i].value.switchLookupValue(data->kind)), lowBlock(data->cases[i].target.block), Weight(data->cases[i].target.count)); } else CRASH(); cases.append(newCase); } m_out.switchInstruction( switchValue, cases, lowBlock(data->fallThrough.block), Weight(data->fallThrough.count)); } void switchString(SwitchData* data, LValue string) { bool canDoBinarySwitch = true; unsigned totalLength = 0; for (DFG::SwitchCase myCase : data->cases) { StringImpl* string = myCase.value.stringImpl(); if (!string->is8Bit()) { canDoBinarySwitch = false; break; } if (string->length() > Options::maximumBinaryStringSwitchCaseLength()) { canDoBinarySwitch = false; break; } totalLength += string->length(); } if (!canDoBinarySwitch || totalLength > Options::maximumBinaryStringSwitchTotalLength()) { switchStringSlow(data, string); return; } LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value); LValue length = m_out.load32(string, m_heaps.JSString_length); LBasicBlock hasImplBlock = m_out.newBlock(); LBasicBlock is8BitBlock = m_out.newBlock(); LBasicBlock slowBlock = m_out.newBlock(); m_out.branch(m_out.isNull(stringImpl), unsure(slowBlock), unsure(hasImplBlock)); LBasicBlock lastNext = m_out.appendTo(hasImplBlock, is8BitBlock); m_out.branch( m_out.testIsZero32( m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIs8Bit())), unsure(slowBlock), unsure(is8BitBlock)); m_out.appendTo(is8BitBlock, slowBlock); LValue buffer = m_out.loadPtr(stringImpl, m_heaps.StringImpl_data); // FIXME: We should propagate branch weight data to the cases of this switch. // https://bugs.webkit.org/show_bug.cgi?id=144368 Vector cases; for (DFG::SwitchCase myCase : data->cases) cases.append(StringSwitchCase(myCase.value.stringImpl(), lowBlock(myCase.target.block))); std::sort(cases.begin(), cases.end()); switchStringRecurse(data, buffer, length, cases, 0, 0, cases.size(), 0, false); m_out.appendTo(slowBlock, lastNext); switchStringSlow(data, string); } // The code for string switching is based closely on the same code in the DFG backend. While it // would be nice to reduce the amount of similar-looking code, it seems like this is one of // those algorithms where factoring out the common bits would result in more code than just // duplicating. struct StringSwitchCase { StringSwitchCase() { } StringSwitchCase(StringImpl* string, LBasicBlock target) : string(string) , target(target) { } bool operator<(const StringSwitchCase& other) const { return stringLessThan(*string, *other.string); } StringImpl* string; LBasicBlock target; }; struct CharacterCase { CharacterCase() : character(0) , begin(0) , end(0) { } CharacterCase(LChar character, unsigned begin, unsigned end) : character(character) , begin(begin) , end(end) { } bool operator<(const CharacterCase& other) const { return character < other.character; } LChar character; unsigned begin; unsigned end; }; void switchStringRecurse( SwitchData* data, LValue buffer, LValue length, const Vector& cases, unsigned numChecked, unsigned begin, unsigned end, unsigned alreadyCheckedLength, unsigned checkedExactLength) { LBasicBlock fallThrough = lowBlock(data->fallThrough.block); if (begin == end) { m_out.jump(fallThrough); return; } unsigned minLength = cases[begin].string->length(); unsigned commonChars = minLength; bool allLengthsEqual = true; for (unsigned i = begin + 1; i < end; ++i) { unsigned myCommonChars = numChecked; unsigned limit = std::min(cases[begin].string->length(), cases[i].string->length()); for (unsigned j = numChecked; j < limit; ++j) { if (cases[begin].string->at(j) != cases[i].string->at(j)) break; myCommonChars++; } commonChars = std::min(commonChars, myCommonChars); if (minLength != cases[i].string->length()) allLengthsEqual = false; minLength = std::min(minLength, cases[i].string->length()); } if (checkedExactLength) { DFG_ASSERT(m_graph, m_node, alreadyCheckedLength == minLength); DFG_ASSERT(m_graph, m_node, allLengthsEqual); } DFG_ASSERT(m_graph, m_node, minLength >= commonChars); if (!allLengthsEqual && alreadyCheckedLength < minLength) m_out.check(m_out.below(length, m_out.constInt32(minLength)), unsure(fallThrough)); if (allLengthsEqual && (alreadyCheckedLength < minLength || !checkedExactLength)) m_out.check(m_out.notEqual(length, m_out.constInt32(minLength)), unsure(fallThrough)); for (unsigned i = numChecked; i < commonChars; ++i) { m_out.check( m_out.notEqual( m_out.load8ZeroExt32(buffer, m_heaps.characters8[i]), m_out.constInt32(static_cast(cases[begin].string->at(i)))), unsure(fallThrough)); } if (minLength == commonChars) { // This is the case where one of the cases is a prefix of all of the other cases. // We've already checked that the input string is a prefix of all of the cases, // so we just check length to jump to that case. DFG_ASSERT(m_graph, m_node, cases[begin].string->length() == commonChars); for (unsigned i = begin + 1; i < end; ++i) DFG_ASSERT(m_graph, m_node, cases[i].string->length() > commonChars); if (allLengthsEqual) { DFG_ASSERT(m_graph, m_node, end == begin + 1); m_out.jump(cases[begin].target); return; } m_out.check( m_out.equal(length, m_out.constInt32(commonChars)), unsure(cases[begin].target)); // We've checked if the length is >= minLength, and then we checked if the length is // == commonChars. We get to this point if it is >= minLength but not == commonChars. // Hence we know that it now must be > minLength, i.e. that it's >= minLength + 1. switchStringRecurse( data, buffer, length, cases, commonChars, begin + 1, end, minLength + 1, false); return; } // At this point we know that the string is longer than commonChars, and we've only verified // commonChars. Use a binary switch on the next unchecked character, i.e. // string[commonChars]. DFG_ASSERT(m_graph, m_node, end >= begin + 2); LValue uncheckedChar = m_out.load8ZeroExt32(buffer, m_heaps.characters8[commonChars]); Vector characterCases; CharacterCase currentCase(cases[begin].string->at(commonChars), begin, begin + 1); for (unsigned i = begin + 1; i < end; ++i) { LChar currentChar = cases[i].string->at(commonChars); if (currentChar != currentCase.character) { currentCase.end = i; characterCases.append(currentCase); currentCase = CharacterCase(currentChar, i, i + 1); } else currentCase.end = i + 1; } characterCases.append(currentCase); Vector characterBlocks; for (unsigned i = characterCases.size(); i--;) characterBlocks.append(m_out.newBlock()); Vector switchCases; for (unsigned i = 0; i < characterCases.size(); ++i) { if (i) DFG_ASSERT(m_graph, m_node, characterCases[i - 1].character < characterCases[i].character); switchCases.append(SwitchCase( m_out.constInt32(characterCases[i].character), characterBlocks[i], Weight())); } m_out.switchInstruction(uncheckedChar, switchCases, fallThrough, Weight()); LBasicBlock lastNext = m_out.m_nextBlock; characterBlocks.append(lastNext); // Makes it convenient to set nextBlock. for (unsigned i = 0; i < characterCases.size(); ++i) { m_out.appendTo(characterBlocks[i], characterBlocks[i + 1]); switchStringRecurse( data, buffer, length, cases, commonChars + 1, characterCases[i].begin, characterCases[i].end, minLength, allLengthsEqual); } DFG_ASSERT(m_graph, m_node, m_out.m_nextBlock == lastNext); } void switchStringSlow(SwitchData* data, LValue string) { // FIXME: We ought to be able to use computed gotos here. We would save the labels of the // blocks we want to jump to, and then request their addresses after compilation completes. // https://bugs.webkit.org/show_bug.cgi?id=144369 LValue branchOffset = vmCall( Int32, m_out.operation(operationSwitchStringAndGetBranchOffset), m_callFrame, m_out.constIntPtr(data->switchTableIndex), string); StringJumpTable& table = codeBlock()->stringSwitchJumpTable(data->switchTableIndex); Vector cases; std::unordered_set alreadyHandled; // These may be negative, or zero, or probably other stuff, too. We don't want to mess with HashSet's corner cases and we don't really care about throughput here. for (unsigned i = 0; i < data->cases.size(); ++i) { // FIXME: The fact that we're using the bytecode's switch table means that the // following DFG IR transformation would be invalid. // // Original code: // switch (v) { // case "foo": // case "bar": // things(); // break; // default: // break; // } // // New code: // switch (v) { // case "foo": // instrumentFoo(); // goto _things; // case "bar": // instrumentBar(); // _things: // things(); // break; // default: // break; // } // // Luckily, we don't currently do any such transformation. But it's kind of silly that // this is an issue. // https://bugs.webkit.org/show_bug.cgi?id=144635 DFG::SwitchCase myCase = data->cases[i]; StringJumpTable::StringOffsetTable::iterator iter = table.offsetTable.find(myCase.value.stringImpl()); DFG_ASSERT(m_graph, m_node, iter != table.offsetTable.end()); if (!alreadyHandled.insert(iter->value.branchOffset).second) continue; cases.append(SwitchCase( m_out.constInt32(iter->value.branchOffset), lowBlock(myCase.target.block), Weight(myCase.target.count))); } m_out.switchInstruction( branchOffset, cases, lowBlock(data->fallThrough.block), Weight(data->fallThrough.count)); } // Calls the functor at the point of code generation where we know what the result type is. // You can emit whatever code you like at that point. Expects you to terminate the basic block. // When buildTypeOf() returns, it will have terminated all basic blocks that it created. So, if // you aren't using this as the terminator of a high-level block, you should create your own // contination and set it as the nextBlock (m_out.insertNewBlocksBefore(continuation)) before // calling this. For example: // // LBasicBlock continuation = m_out.newBlock(); // LBasicBlock lastNext = m_out.insertNewBlocksBefore(continuation); // buildTypeOf( // child, value, // [&] (TypeofType type) { // do things; // m_out.jump(continuation); // }); // m_out.appendTo(continuation, lastNext); template void buildTypeOf(Edge child, LValue value, const Functor& functor) { JSGlobalObject* globalObject = m_graph.globalObjectFor(m_node->origin.semantic); // Implements the following branching structure: // // if (is cell) { // if (is object) { // if (is function) { // return function; // } else if (doesn't have call trap and doesn't masquerade as undefined) { // return object // } else { // return slowPath(); // } // } else if (is string) { // return string // } else { // return symbol // } // } else if (is number) { // return number // } else if (is null) { // return object // } else if (is boolean) { // return boolean // } else { // return undefined // } LBasicBlock cellCase = m_out.newBlock(); LBasicBlock objectCase = m_out.newBlock(); LBasicBlock functionCase = m_out.newBlock(); LBasicBlock notFunctionCase = m_out.newBlock(); LBasicBlock reallyObjectCase = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock unreachable = m_out.newBlock(); LBasicBlock notObjectCase = m_out.newBlock(); LBasicBlock stringCase = m_out.newBlock(); LBasicBlock symbolCase = m_out.newBlock(); LBasicBlock notCellCase = m_out.newBlock(); LBasicBlock numberCase = m_out.newBlock(); LBasicBlock notNumberCase = m_out.newBlock(); LBasicBlock notNullCase = m_out.newBlock(); LBasicBlock booleanCase = m_out.newBlock(); LBasicBlock undefinedCase = m_out.newBlock(); m_out.branch(isCell(value, provenType(child)), unsure(cellCase), unsure(notCellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, objectCase); m_out.branch(isObject(value, provenType(child)), unsure(objectCase), unsure(notObjectCase)); m_out.appendTo(objectCase, functionCase); m_out.branch( isFunction(value, provenType(child) & SpecObject), unsure(functionCase), unsure(notFunctionCase)); m_out.appendTo(functionCase, notFunctionCase); functor(TypeofType::Function); m_out.appendTo(notFunctionCase, reallyObjectCase); m_out.branch( isExoticForTypeof(value, provenType(child) & (SpecObject - SpecFunction)), rarely(slowPath), usually(reallyObjectCase)); m_out.appendTo(reallyObjectCase, slowPath); functor(TypeofType::Object); m_out.appendTo(slowPath, unreachable); LValue result = lazySlowPath( [=] (const Vector& locations) -> RefPtr { return createLazyCallGenerator( operationTypeOfObjectAsTypeofType, locations[0].directGPR(), CCallHelpers::TrustedImmPtr(globalObject), locations[1].directGPR()); }, value); Vector cases; cases.append(SwitchCase(m_out.constInt32(static_cast(TypeofType::Undefined)), undefinedCase)); cases.append(SwitchCase(m_out.constInt32(static_cast(TypeofType::Object)), reallyObjectCase)); cases.append(SwitchCase(m_out.constInt32(static_cast(TypeofType::Function)), functionCase)); m_out.switchInstruction(m_out.castToInt32(result), cases, unreachable, Weight()); m_out.appendTo(unreachable, notObjectCase); m_out.unreachable(); m_out.appendTo(notObjectCase, stringCase); m_out.branch( isString(value, provenType(child) & (SpecCell - SpecObject)), unsure(stringCase), unsure(symbolCase)); m_out.appendTo(stringCase, symbolCase); functor(TypeofType::String); m_out.appendTo(symbolCase, notCellCase); functor(TypeofType::Symbol); m_out.appendTo(notCellCase, numberCase); m_out.branch( isNumber(value, provenType(child) & ~SpecCell), unsure(numberCase), unsure(notNumberCase)); m_out.appendTo(numberCase, notNumberCase); functor(TypeofType::Number); m_out.appendTo(notNumberCase, notNullCase); LValue isNull; if (provenType(child) & SpecOther) isNull = m_out.equal(value, m_out.constInt64(ValueNull)); else isNull = m_out.booleanFalse; m_out.branch(isNull, unsure(reallyObjectCase), unsure(notNullCase)); m_out.appendTo(notNullCase, booleanCase); m_out.branch( isBoolean(value, provenType(child) & ~(SpecCell | SpecFullNumber)), unsure(booleanCase), unsure(undefinedCase)); m_out.appendTo(booleanCase, undefinedCase); functor(TypeofType::Boolean); m_out.appendTo(undefinedCase, lastNext); functor(TypeofType::Undefined); } LValue doubleToInt32(LValue doubleValue, double low, double high, bool isSigned = true) { LBasicBlock greatEnough = m_out.newBlock(); LBasicBlock withinRange = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); Vector results; m_out.branch( m_out.doubleGreaterThanOrEqual(doubleValue, m_out.constDouble(low)), unsure(greatEnough), unsure(slowPath)); LBasicBlock lastNext = m_out.appendTo(greatEnough, withinRange); m_out.branch( m_out.doubleLessThanOrEqual(doubleValue, m_out.constDouble(high)), unsure(withinRange), unsure(slowPath)); m_out.appendTo(withinRange, slowPath); LValue fastResult; if (isSigned) fastResult = m_out.doubleToInt(doubleValue); else fastResult = m_out.doubleToUInt(doubleValue); results.append(m_out.anchor(fastResult)); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); results.append(m_out.anchor(m_out.call(Int32, m_out.operation(operationToInt32), doubleValue))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int32, results); } LValue doubleToInt32(LValue doubleValue) { if (hasSensibleDoubleToInt()) return sensibleDoubleToInt32(doubleValue); double limit = pow(2, 31) - 1; return doubleToInt32(doubleValue, -limit, limit); } LValue sensibleDoubleToInt32(LValue doubleValue) { LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue fastResultValue = m_out.doubleToInt(doubleValue); ValueFromBlock fastResult = m_out.anchor(fastResultValue); m_out.branch( m_out.equal(fastResultValue, m_out.constInt32(0x80000000)), rarely(slowPath), usually(continuation)); LBasicBlock lastNext = m_out.appendTo(slowPath, continuation); ValueFromBlock slowResult = m_out.anchor( m_out.call(Int32, m_out.operation(operationToInt32SensibleSlow), doubleValue)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int32, fastResult, slowResult); } // This is a mechanism for creating a code generator that fills in a gap in the code using our // own MacroAssembler. This is useful for slow paths that involve a lot of code and we don't want // to pay the price of B3 optimizing it. A lazy slow path will only be generated if it actually // executes. On the other hand, a lazy slow path always incurs the cost of two additional jumps. // Also, the lazy slow path's register allocation state is slaved to whatever B3 did, so you // have to use a ScratchRegisterAllocator to try to use some unused registers and you may have // to spill to top of stack if there aren't enough registers available. // // Lazy slow paths involve three different stages of execution. Each stage has unique // capabilities and knowledge. The stages are: // // 1) DFG->B3 lowering, i.e. code that runs in this phase. Lowering is the last time you will // have access to LValues. If there is an LValue that needs to be fed as input to a lazy slow // path, then you must pass it as an argument here (as one of the varargs arguments after the // functor). But, lowering doesn't know which registers will be used for those LValues. Hence // you pass a lambda to lazySlowPath() and that lambda will run during stage (2): // // 2) FTLCompile.cpp's fixFunctionBasedOnStackMaps. This code is the only stage at which we know // the mapping from arguments passed to this method in (1) and the registers that B3 // selected for those arguments. You don't actually want to generate any code here, since then // the slow path wouldn't actually be lazily generated. Instead, you want to save the // registers being used for the arguments and defer code generation to stage (3) by creating // and returning a LazySlowPath::Generator: // // 3) LazySlowPath's generate() method. This code runs in response to the lazy slow path // executing for the first time. It will call the generator you created in stage (2). // // Note that each time you invoke stage (1), stage (2) may be invoked zero, one, or many times. // Stage (2) will usually be invoked once for stage (1). But, B3 may kill the code, in which // case stage (2) won't run. B3 may duplicate the code (for example via tail duplication), // leading to many calls to your stage (2) lambda. Stage (3) may be called zero or once for each // stage (2). It will be called zero times if the slow path never runs. This is what you hope for // whenever you use the lazySlowPath() mechanism. // // A typical use of lazySlowPath() will look like the example below, which just creates a slow // path that adds some value to the input and returns it. // // // Stage (1) is here. This is your last chance to figure out which LValues to use as inputs. // // Notice how we pass "input" as an argument to lazySlowPath(). // LValue input = ...; // int addend = ...; // LValue output = lazySlowPath( // [=] (const Vector& locations) -> RefPtr { // // Stage (2) is here. This is your last chance to figure out which registers are used // // for which values. Location zero is always the return value. You can ignore it if // // you don't want to return anything. Location 1 is the register for the first // // argument to the lazySlowPath(), i.e. "input". Note that the Location object could // // also hold an FPR, if you are passing a double. // GPRReg outputGPR = locations[0].directGPR(); // GPRReg inputGPR = locations[1].directGPR(); // return LazySlowPath::createGenerator( // [=] (CCallHelpers& jit, LazySlowPath::GenerationParams& params) { // // Stage (3) is here. This is when you generate code. You have access to the // // registers you collected in stage (2) because this lambda closes over those // // variables (outputGPR and inputGPR). You also have access to whatever extra // // data you collected in stage (1), such as the addend in this case. // jit.add32(TrustedImm32(addend), inputGPR, outputGPR); // // You have to end by jumping to done. There is nothing to fall through to. // // You can also jump to the exception handler (see LazySlowPath.h for more // // info). Note that currently you cannot OSR exit. // params.doneJumps.append(jit.jump()); // }); // }, // input); // // You can basically pass as many inputs as you like, either using this varargs form, or by // passing a Vector of LValues. // // Note that if your slow path is only doing a call, you can use the createLazyCallGenerator() // helper. For example: // // LValue input = ...; // LValue output = lazySlowPath( // [=] (const Vector& locations) -> RefPtr { // return createLazyCallGenerator( // operationDoThings, locations[0].directGPR(), locations[1].directGPR()); // }, input); // // Finally, note that all of the lambdas - both the stage (2) lambda and the stage (3) lambda - // run after the function that created them returns. Hence, you should not use by-reference // capture (i.e. [&]) in any of these lambdas. template PatchpointValue* lazySlowPath(const Functor& functor, ArgumentTypes... arguments) { return lazySlowPath(functor, Vector{ arguments... }); } template PatchpointValue* lazySlowPath(const Functor& functor, const Vector& userArguments) { CodeOrigin origin = m_node->origin.semantic; PatchpointValue* result = m_out.patchpoint(B3::Int64); for (LValue arg : userArguments) result->append(ConstrainedValue(arg, B3::ValueRep::SomeRegister)); RefPtr exceptionHandle = preparePatchpointForExceptions(result); result->clobber(RegisterSet::macroScratchRegisters()); State* state = &m_ftlState; result->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams& params) { Vector locations; for (const B3::ValueRep& rep : params) locations.append(Location::forValueRep(rep)); RefPtr generator = functor(locations); CCallHelpers::PatchableJump patchableJump = jit.patchableJump(); CCallHelpers::Label done = jit.label(); RegisterSet usedRegisters = params.unavailableRegisters(); RefPtr exceptionTarget = exceptionHandle->scheduleExitCreation(params); // FIXME: As part of handling exceptions, we need to create a concrete OSRExit here. // Doing so should automagically register late paths that emit exit thunks. params.addLatePath( [=] (CCallHelpers& jit) { AllowMacroScratchRegisterUsage allowScratch(jit); patchableJump.m_jump.link(&jit); unsigned index = state->jitCode->lazySlowPaths.size(); state->jitCode->lazySlowPaths.append(nullptr); jit.pushToSaveImmediateWithoutTouchingRegisters( CCallHelpers::TrustedImm32(index)); CCallHelpers::Jump generatorJump = jit.jump(); // Note that so long as we're here, we don't really know if our late path // runs before or after any other late paths that we might depend on, like // the exception thunk. RefPtr jitCode = state->jitCode; VM* vm = &state->graph.m_vm; jit.addLinkTask( [=] (LinkBuffer& linkBuffer) { linkBuffer.link( generatorJump, CodeLocationLabel( vm->getCTIStub( lazySlowPathGenerationThunkGenerator).code())); CodeLocationJump linkedPatchableJump = CodeLocationJump( linkBuffer.locationOf(patchableJump)); CodeLocationLabel linkedDone = linkBuffer.locationOf(done); CallSiteIndex callSiteIndex = jitCode->common.addUniqueCallSiteIndex(origin); std::unique_ptr lazySlowPath = std::make_unique( linkedPatchableJump, linkedDone, exceptionTarget->label(linkBuffer), usedRegisters, callSiteIndex, generator); jitCode->lazySlowPaths[index] = WTFMove(lazySlowPath); }); }); }); return result; } void speculate( ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition) { appendOSRExit(kind, lowValue, highValue, failCondition, m_origin); } void terminate(ExitKind kind) { speculate(kind, noValue(), nullptr, m_out.booleanTrue); didAlreadyTerminate(); } void didAlreadyTerminate() { m_state.setIsValid(false); } void typeCheck( FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough, LValue failCondition, ExitKind exitKind = BadType) { appendTypeCheck(lowValue, highValue, typesPassedThrough, failCondition, exitKind); } void appendTypeCheck( FormattedValue lowValue, Edge highValue, SpeculatedType typesPassedThrough, LValue failCondition, ExitKind exitKind) { if (!m_interpreter.needsTypeCheck(highValue, typesPassedThrough)) return; ASSERT(mayHaveTypeCheck(highValue.useKind())); appendOSRExit(exitKind, lowValue, highValue.node(), failCondition, m_origin); m_interpreter.filter(highValue, typesPassedThrough); } LValue lowInt32(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || (edge.useKind() == Int32Use || edge.useKind() == KnownInt32Use)); if (edge->hasConstant()) { JSValue value = edge->asJSValue(); if (!value.isInt32()) { terminate(Uncountable); return m_out.int32Zero; } return m_out.constInt32(value.asInt32()); } LoweredNodeValue value = m_int32Values.get(edge.node()); if (isValid(value)) return value.value(); value = m_strictInt52Values.get(edge.node()); if (isValid(value)) return strictInt52ToInt32(edge, value.value()); value = m_int52Values.get(edge.node()); if (isValid(value)) return strictInt52ToInt32(edge, int52ToStrictInt52(value.value())); value = m_jsValueValues.get(edge.node()); if (isValid(value)) { LValue boxedResult = value.value(); FTL_TYPE_CHECK( jsValueValue(boxedResult), edge, SpecInt32Only, isNotInt32(boxedResult)); LValue result = unboxInt32(boxedResult); setInt32(edge.node(), result); return result; } DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecInt32Only)); terminate(Uncountable); return m_out.int32Zero; } enum Int52Kind { StrictInt52, Int52 }; LValue lowInt52(Edge edge, Int52Kind kind) { DFG_ASSERT(m_graph, m_node, edge.useKind() == Int52RepUse); LoweredNodeValue value; switch (kind) { case Int52: value = m_int52Values.get(edge.node()); if (isValid(value)) return value.value(); value = m_strictInt52Values.get(edge.node()); if (isValid(value)) return strictInt52ToInt52(value.value()); break; case StrictInt52: value = m_strictInt52Values.get(edge.node()); if (isValid(value)) return value.value(); value = m_int52Values.get(edge.node()); if (isValid(value)) return int52ToStrictInt52(value.value()); break; } DFG_ASSERT(m_graph, m_node, !provenType(edge)); terminate(Uncountable); return m_out.int64Zero; } LValue lowInt52(Edge edge) { return lowInt52(edge, Int52); } LValue lowStrictInt52(Edge edge) { return lowInt52(edge, StrictInt52); } bool betterUseStrictInt52(Node* node) { return !isValid(m_int52Values.get(node)); } bool betterUseStrictInt52(Edge edge) { return betterUseStrictInt52(edge.node()); } template Int52Kind bestInt52Kind(T node) { return betterUseStrictInt52(node) ? StrictInt52 : Int52; } Int52Kind opposite(Int52Kind kind) { switch (kind) { case Int52: return StrictInt52; case StrictInt52: return Int52; } DFG_CRASH(m_graph, m_node, "Bad use kind"); return Int52; } LValue lowWhicheverInt52(Edge edge, Int52Kind& kind) { kind = bestInt52Kind(edge); return lowInt52(edge, kind); } LValue lowCell(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || DFG::isCell(edge.useKind())); if (edge->op() == JSConstant) { FrozenValue* value = edge->constant(); if (!value->value().isCell()) { terminate(Uncountable); return m_out.intPtrZero; } return frozenPointer(value); } LoweredNodeValue value = m_jsValueValues.get(edge.node()); if (isValid(value)) { LValue uncheckedValue = value.value(); FTL_TYPE_CHECK( jsValueValue(uncheckedValue), edge, SpecCell, isNotCell(uncheckedValue)); return uncheckedValue; } DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecCell)); terminate(Uncountable); return m_out.intPtrZero; } LValue lowObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse); LValue result = lowCell(edge, mode); speculateObject(edge, result); return result; } LValue lowRegExpObject(Edge edge) { LValue result = lowCell(edge); speculateRegExpObject(edge, result); return result; } LValue lowMapObject(Edge edge) { LValue result = lowCell(edge); speculateMapObject(edge, result); return result; } LValue lowSetObject(Edge edge) { LValue result = lowCell(edge); speculateSetObject(edge, result); return result; } LValue lowString(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringUse || edge.useKind() == KnownStringUse || edge.useKind() == StringIdentUse); LValue result = lowCell(edge, mode); speculateString(edge, result); return result; } LValue lowStringIdent(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == StringIdentUse); LValue string = lowString(edge, mode); LValue stringImpl = m_out.loadPtr(string, m_heaps.JSString_value); speculateStringIdent(edge, string, stringImpl); return stringImpl; } LValue lowSymbol(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == SymbolUse); LValue result = lowCell(edge, mode); speculateSymbol(edge, result); return result; } LValue lowNonNullObject(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == ObjectUse); LValue result = lowCell(edge, mode); speculateNonNullObject(edge, result); return result; } LValue lowBoolean(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { ASSERT_UNUSED(mode, mode == ManualOperandSpeculation || edge.useKind() == BooleanUse || edge.useKind() == KnownBooleanUse); if (edge->hasConstant()) { JSValue value = edge->asJSValue(); if (!value.isBoolean()) { terminate(Uncountable); return m_out.booleanFalse; } return m_out.constBool(value.asBoolean()); } LoweredNodeValue value = m_booleanValues.get(edge.node()); if (isValid(value)) return value.value(); value = m_jsValueValues.get(edge.node()); if (isValid(value)) { LValue unboxedResult = value.value(); FTL_TYPE_CHECK( jsValueValue(unboxedResult), edge, SpecBoolean, isNotBoolean(unboxedResult)); LValue result = unboxBoolean(unboxedResult); setBoolean(edge.node(), result); return result; } DFG_ASSERT(m_graph, m_node, !(provenType(edge) & SpecBoolean)); terminate(Uncountable); return m_out.booleanFalse; } LValue lowDouble(Edge edge) { DFG_ASSERT(m_graph, m_node, isDouble(edge.useKind())); LoweredNodeValue value = m_doubleValues.get(edge.node()); if (isValid(value)) return value.value(); DFG_ASSERT(m_graph, m_node, !provenType(edge)); terminate(Uncountable); return m_out.doubleZero; } LValue lowJSValue(Edge edge, OperandSpeculationMode mode = AutomaticOperandSpeculation) { DFG_ASSERT(m_graph, m_node, mode == ManualOperandSpeculation || edge.useKind() == UntypedUse); DFG_ASSERT(m_graph, m_node, !isDouble(edge.useKind())); DFG_ASSERT(m_graph, m_node, edge.useKind() != Int52RepUse); if (edge->hasConstant()) return m_out.constInt64(JSValue::encode(edge->asJSValue())); LoweredNodeValue value = m_jsValueValues.get(edge.node()); if (isValid(value)) return value.value(); value = m_int32Values.get(edge.node()); if (isValid(value)) { LValue result = boxInt32(value.value()); setJSValue(edge.node(), result); return result; } value = m_booleanValues.get(edge.node()); if (isValid(value)) { LValue result = boxBoolean(value.value()); setJSValue(edge.node(), result); return result; } DFG_CRASH(m_graph, m_node, "Value not defined"); return 0; } LValue lowNotCell(Edge edge) { LValue result = lowJSValue(edge, ManualOperandSpeculation); FTL_TYPE_CHECK(jsValueValue(result), edge, ~SpecCell, isCell(result)); return result; } LValue lowStorage(Edge edge) { LoweredNodeValue value = m_storageValues.get(edge.node()); if (isValid(value)) return value.value(); LValue result = lowCell(edge); setStorage(edge.node(), result); return result; } LValue lowMapBucket(Edge edge) { LoweredNodeValue value = m_mapBucketValues.get(edge.node()); if (isValid(value)) return value.value(); LValue result = lowCell(edge); setStorage(edge.node(), result); return result; } LValue strictInt52ToInt32(Edge edge, LValue value) { LValue result = m_out.castToInt32(value); FTL_TYPE_CHECK( noValue(), edge, SpecInt32Only, m_out.notEqual(m_out.signExt32To64(result), value)); setInt32(edge.node(), result); return result; } LValue strictInt52ToDouble(LValue value) { return m_out.intToDouble(value); } LValue strictInt52ToJSValue(LValue value) { LBasicBlock isInt32 = m_out.newBlock(); LBasicBlock isDouble = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); Vector results; LValue int32Value = m_out.castToInt32(value); m_out.branch( m_out.equal(m_out.signExt32To64(int32Value), value), unsure(isInt32), unsure(isDouble)); LBasicBlock lastNext = m_out.appendTo(isInt32, isDouble); results.append(m_out.anchor(boxInt32(int32Value))); m_out.jump(continuation); m_out.appendTo(isDouble, continuation); results.append(m_out.anchor(boxDouble(m_out.intToDouble(value)))); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int64, results); } LValue strictInt52ToInt52(LValue value) { return m_out.shl(value, m_out.constInt64(JSValue::int52ShiftAmount)); } LValue int52ToStrictInt52(LValue value) { return m_out.aShr(value, m_out.constInt64(JSValue::int52ShiftAmount)); } LValue isInt32(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, SpecInt32Only)) return proven; return m_out.aboveOrEqual(jsValue, m_tagTypeNumber); } LValue isNotInt32(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, ~SpecInt32Only)) return proven; return m_out.below(jsValue, m_tagTypeNumber); } LValue unboxInt32(LValue jsValue) { return m_out.castToInt32(jsValue); } LValue boxInt32(LValue value) { return m_out.add(m_out.zeroExt(value, Int64), m_tagTypeNumber); } LValue isCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, SpecCell | SpecMisc)) return proven; return m_out.testIsZero64(jsValue, m_tagTypeNumber); } LValue isNotCellOrMisc(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, ~(SpecCell | SpecMisc))) return proven; return m_out.testNonZero64(jsValue, m_tagTypeNumber); } LValue unboxDouble(LValue jsValue) { return m_out.bitCast(m_out.add(jsValue, m_tagTypeNumber), Double); } LValue boxDouble(LValue doubleValue) { return m_out.sub(m_out.bitCast(doubleValue, Int64), m_tagTypeNumber); } LValue jsValueToStrictInt52(Edge edge, LValue boxedValue) { LBasicBlock intCase = m_out.newBlock(); LBasicBlock doubleCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue isNotInt32; if (!m_interpreter.needsTypeCheck(edge, SpecInt32Only)) isNotInt32 = m_out.booleanFalse; else if (!m_interpreter.needsTypeCheck(edge, ~SpecInt32Only)) isNotInt32 = m_out.booleanTrue; else isNotInt32 = this->isNotInt32(boxedValue); m_out.branch(isNotInt32, unsure(doubleCase), unsure(intCase)); LBasicBlock lastNext = m_out.appendTo(intCase, doubleCase); ValueFromBlock intToInt52 = m_out.anchor( m_out.signExt32To64(unboxInt32(boxedValue))); m_out.jump(continuation); m_out.appendTo(doubleCase, continuation); LValue possibleResult = m_out.call( Int64, m_out.operation(operationConvertBoxedDoubleToInt52), boxedValue); FTL_TYPE_CHECK( jsValueValue(boxedValue), edge, SpecInt32Only | SpecAnyIntAsDouble, m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52))); ValueFromBlock doubleToInt52 = m_out.anchor(possibleResult); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); return m_out.phi(Int64, intToInt52, doubleToInt52); } LValue doubleToStrictInt52(Edge edge, LValue value) { LValue possibleResult = m_out.call( Int64, m_out.operation(operationConvertDoubleToInt52), value); FTL_TYPE_CHECK_WITH_EXIT_KIND(Int52Overflow, doubleValue(value), edge, SpecAnyIntAsDouble, m_out.equal(possibleResult, m_out.constInt64(JSValue::notInt52))); return possibleResult; } LValue convertDoubleToInt32(LValue value, bool shouldCheckNegativeZero) { LValue integerValue = m_out.doubleToInt(value); LValue integerValueConvertedToDouble = m_out.intToDouble(integerValue); LValue valueNotConvertibleToInteger = m_out.doubleNotEqualOrUnordered(value, integerValueConvertedToDouble); speculate(Overflow, FormattedValue(DataFormatDouble, value), m_node, valueNotConvertibleToInteger); if (shouldCheckNegativeZero) { LBasicBlock valueIsZero = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(m_out.isZero32(integerValue), unsure(valueIsZero), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(valueIsZero, continuation); LValue doubleBitcastToInt64 = m_out.bitCast(value, Int64); LValue signBitSet = m_out.lessThan(doubleBitcastToInt64, m_out.constInt64(0)); speculate(NegativeZero, FormattedValue(DataFormatDouble, value), m_node, signBitSet); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } return integerValue; } LValue isNumber(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, SpecFullNumber)) return proven; return isNotCellOrMisc(jsValue); } LValue isNotNumber(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, ~SpecFullNumber)) return proven; return isCellOrMisc(jsValue); } LValue isNotCell(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, ~SpecCell)) return proven; return m_out.testNonZero64(jsValue, m_tagMask); } LValue isCell(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, SpecCell)) return proven; return m_out.testIsZero64(jsValue, m_tagMask); } LValue isNotMisc(LValue value, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, ~SpecMisc)) return proven; return m_out.above(value, m_out.constInt64(TagBitTypeOther | TagBitBool | TagBitUndefined)); } LValue isMisc(LValue value, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, SpecMisc)) return proven; return m_out.logicalNot(isNotMisc(value)); } LValue isNotBoolean(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, ~SpecBoolean)) return proven; return m_out.testNonZero64( m_out.bitXor(jsValue, m_out.constInt64(ValueFalse)), m_out.constInt64(~1)); } LValue isBoolean(LValue jsValue, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, SpecBoolean)) return proven; return m_out.logicalNot(isNotBoolean(jsValue)); } LValue unboxBoolean(LValue jsValue) { // We want to use a cast that guarantees that B3 knows that even the integer // value is just 0 or 1. But for now we do it the dumb way. return m_out.notZero64(m_out.bitAnd(jsValue, m_out.constInt64(1))); } LValue boxBoolean(LValue value) { return m_out.select( value, m_out.constInt64(ValueTrue), m_out.constInt64(ValueFalse)); } LValue isNotOther(LValue value, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, ~SpecOther)) return proven; return m_out.notEqual( m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)), m_out.constInt64(ValueNull)); } LValue isOther(LValue value, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type, SpecOther)) return proven; return m_out.equal( m_out.bitAnd(value, m_out.constInt64(~TagBitUndefined)), m_out.constInt64(ValueNull)); } LValue isProvenValue(SpeculatedType provenType, SpeculatedType wantedType) { if (!(provenType & ~wantedType)) return m_out.booleanTrue; if (!(provenType & wantedType)) return m_out.booleanFalse; return nullptr; } void speculate(Edge edge) { switch (edge.useKind()) { case UntypedUse: break; case KnownInt32Use: case KnownStringUse: case KnownPrimitiveUse: case DoubleRepUse: case Int52RepUse: ASSERT(!m_interpreter.needsTypeCheck(edge)); break; case Int32Use: speculateInt32(edge); break; case CellUse: speculateCell(edge); break; case CellOrOtherUse: speculateCellOrOther(edge); break; case KnownCellUse: ASSERT(!m_interpreter.needsTypeCheck(edge)); break; case AnyIntUse: speculateAnyInt(edge); break; case ObjectUse: speculateObject(edge); break; case ArrayUse: speculateArray(edge); break; case FunctionUse: speculateFunction(edge); break; case ObjectOrOtherUse: speculateObjectOrOther(edge); break; case FinalObjectUse: speculateFinalObject(edge); break; case RegExpObjectUse: speculateRegExpObject(edge); break; case ProxyObjectUse: speculateProxyObject(edge); break; case DerivedArrayUse: speculateDerivedArray(edge); break; case MapObjectUse: speculateMapObject(edge); break; case SetObjectUse: speculateSetObject(edge); break; case StringUse: speculateString(edge); break; case StringOrOtherUse: speculateStringOrOther(edge); break; case StringIdentUse: speculateStringIdent(edge); break; case SymbolUse: speculateSymbol(edge); break; case StringObjectUse: speculateStringObject(edge); break; case StringOrStringObjectUse: speculateStringOrStringObject(edge); break; case NumberUse: speculateNumber(edge); break; case RealNumberUse: speculateRealNumber(edge); break; case DoubleRepRealUse: speculateDoubleRepReal(edge); break; case DoubleRepAnyIntUse: speculateDoubleRepAnyInt(edge); break; case BooleanUse: speculateBoolean(edge); break; case NotStringVarUse: speculateNotStringVar(edge); break; case NotCellUse: speculateNotCell(edge); break; case OtherUse: speculateOther(edge); break; case MiscUse: speculateMisc(edge); break; default: DFG_CRASH(m_graph, m_node, "Unsupported speculation use kind"); } } void speculate(Node*, Edge edge) { speculate(edge); } void speculateInt32(Edge edge) { lowInt32(edge); } void speculateCell(Edge edge) { lowCell(edge); } void speculateNotCell(Edge edge) { if (!m_interpreter.needsTypeCheck(edge)) return; lowNotCell(edge); } void speculateCellOrOther(Edge edge) { LValue value = lowJSValue(edge, ManualOperandSpeculation); LBasicBlock isNotCell = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isCell(value, provenType(edge)), unsure(continuation), unsure(isNotCell)); LBasicBlock lastNext = m_out.appendTo(isNotCell, continuation); FTL_TYPE_CHECK(jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void speculateAnyInt(Edge edge) { if (!m_interpreter.needsTypeCheck(edge)) return; jsValueToStrictInt52(edge, lowJSValue(edge, ManualOperandSpeculation)); } LValue isCellWithType(LValue cell, JSType queriedType, SpeculatedType speculatedTypeForQuery, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, speculatedTypeForQuery)) return proven; return m_out.equal( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType), m_out.constInt32(queriedType)); } LValue isTypedArrayView(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, SpecTypedArrayView)) return proven; LValue jsType = m_out.sub( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType), m_out.constInt32(Int8ArrayType)); return m_out.belowOrEqual( jsType, m_out.constInt32(Float64ArrayType - Int8ArrayType)); } LValue isObject(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, SpecObject)) return proven; return m_out.aboveOrEqual( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType), m_out.constInt32(ObjectType)); } LValue isNotObject(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, ~SpecObject)) return proven; return m_out.below( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType), m_out.constInt32(ObjectType)); } LValue isNotString(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, ~SpecString)) return proven; return m_out.notEqual( m_out.load32(cell, m_heaps.JSCell_structureID), m_out.constInt32(vm().stringStructure->id())); } LValue isString(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, SpecString)) return proven; return m_out.equal( m_out.load32(cell, m_heaps.JSCell_structureID), m_out.constInt32(vm().stringStructure->id())); } LValue isNotSymbol(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, ~SpecSymbol)) return proven; return m_out.notEqual( m_out.load32(cell, m_heaps.JSCell_structureID), m_out.constInt32(vm().symbolStructure->id())); } LValue isSymbol(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, SpecSymbol)) return proven; return m_out.equal( m_out.load32(cell, m_heaps.JSCell_structureID), m_out.constInt32(vm().symbolStructure->id())); } LValue isArrayType(LValue cell, ArrayMode arrayMode) { switch (arrayMode.type()) { case Array::Int32: case Array::Double: case Array::Contiguous: { LValue indexingType = m_out.load8ZeroExt32(cell, m_heaps.JSCell_indexingTypeAndMisc); switch (arrayMode.arrayClass()) { case Array::OriginalArray: DFG_CRASH(m_graph, m_node, "Unexpected original array"); return 0; case Array::Array: return m_out.equal( m_out.bitAnd(indexingType, m_out.constInt32(IsArray | IndexingShapeMask)), m_out.constInt32(IsArray | arrayMode.shapeMask())); case Array::NonArray: case Array::OriginalNonArray: return m_out.equal( m_out.bitAnd(indexingType, m_out.constInt32(IsArray | IndexingShapeMask)), m_out.constInt32(arrayMode.shapeMask())); case Array::PossiblyArray: return m_out.equal( m_out.bitAnd(indexingType, m_out.constInt32(IndexingShapeMask)), m_out.constInt32(arrayMode.shapeMask())); } DFG_CRASH(m_graph, m_node, "Corrupt array class"); } case Array::DirectArguments: return m_out.equal( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType), m_out.constInt32(DirectArgumentsType)); case Array::ScopedArguments: return m_out.equal( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType), m_out.constInt32(ScopedArgumentsType)); default: return m_out.equal( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType), m_out.constInt32(typeForTypedArrayType(arrayMode.typedArrayType()))); } } LValue isFunction(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, SpecFunction)) return proven; return isType(cell, JSFunctionType); } LValue isNotFunction(LValue cell, SpeculatedType type = SpecFullTop) { if (LValue proven = isProvenValue(type & SpecCell, ~SpecFunction)) return proven; return isNotType(cell, JSFunctionType); } LValue isExoticForTypeof(LValue cell, SpeculatedType type = SpecFullTop) { if (!(type & SpecObjectOther)) return m_out.booleanFalse; return m_out.testNonZero32( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoFlags), m_out.constInt32(MasqueradesAsUndefined | TypeOfShouldCallGetCallData)); } LValue isType(LValue cell, JSType type) { return m_out.equal( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoType), m_out.constInt32(type)); } LValue isNotType(LValue cell, JSType type) { return m_out.logicalNot(isType(cell, type)); } void speculateObject(Edge edge, LValue cell) { FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell)); } void speculateObject(Edge edge) { speculateObject(edge, lowCell(edge)); } void speculateArray(Edge edge, LValue cell) { FTL_TYPE_CHECK( jsValueValue(cell), edge, SpecArray, isNotType(cell, ArrayType)); } void speculateArray(Edge edge) { speculateArray(edge, lowCell(edge)); } void speculateFunction(Edge edge, LValue cell) { FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecFunction, isNotFunction(cell)); } void speculateFunction(Edge edge) { speculateFunction(edge, lowCell(edge)); } void speculateObjectOrOther(Edge edge) { if (!m_interpreter.needsTypeCheck(edge)) return; LValue value = lowJSValue(edge, ManualOperandSpeculation); LBasicBlock cellCase = m_out.newBlock(); LBasicBlock primitiveCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isNotCell(value, provenType(edge)), unsure(primitiveCase), unsure(cellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, primitiveCase); FTL_TYPE_CHECK( jsValueValue(value), edge, (~SpecCell) | SpecObject, isNotObject(value)); m_out.jump(continuation); m_out.appendTo(primitiveCase, continuation); FTL_TYPE_CHECK( jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void speculateFinalObject(Edge edge, LValue cell) { FTL_TYPE_CHECK( jsValueValue(cell), edge, SpecFinalObject, isNotType(cell, FinalObjectType)); } void speculateFinalObject(Edge edge) { speculateFinalObject(edge, lowCell(edge)); } void speculateRegExpObject(Edge edge, LValue cell) { FTL_TYPE_CHECK( jsValueValue(cell), edge, SpecRegExpObject, isNotType(cell, RegExpObjectType)); } void speculateRegExpObject(Edge edge) { speculateRegExpObject(edge, lowCell(edge)); } void speculateProxyObject(Edge edge, LValue cell) { FTL_TYPE_CHECK( jsValueValue(cell), edge, SpecProxyObject, isNotType(cell, ProxyObjectType)); } void speculateProxyObject(Edge edge) { speculateProxyObject(edge, lowCell(edge)); } void speculateDerivedArray(Edge edge, LValue cell) { FTL_TYPE_CHECK( jsValueValue(cell), edge, SpecDerivedArray, isNotType(cell, DerivedArrayType)); } void speculateDerivedArray(Edge edge) { speculateDerivedArray(edge, lowCell(edge)); } void speculateMapObject(Edge edge, LValue cell) { FTL_TYPE_CHECK( jsValueValue(cell), edge, SpecMapObject, isNotType(cell, JSMapType)); } void speculateMapObject(Edge edge) { speculateMapObject(edge, lowCell(edge)); } void speculateSetObject(Edge edge, LValue cell) { FTL_TYPE_CHECK( jsValueValue(cell), edge, SpecSetObject, isNotType(cell, JSSetType)); } void speculateSetObject(Edge edge) { speculateSetObject(edge, lowCell(edge)); } void speculateString(Edge edge, LValue cell) { FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecString | ~SpecCell, isNotString(cell)); } void speculateString(Edge edge) { speculateString(edge, lowCell(edge)); } void speculateStringOrOther(Edge edge, LValue value) { LBasicBlock cellCase = m_out.newBlock(); LBasicBlock notCellCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isCell(value, provenType(edge)), unsure(cellCase), unsure(notCellCase)); LBasicBlock lastNext = m_out.appendTo(cellCase, notCellCase); FTL_TYPE_CHECK(jsValueValue(value), edge, (~SpecCell) | SpecString, isNotString(value)); m_out.jump(continuation); m_out.appendTo(notCellCase, continuation); FTL_TYPE_CHECK(jsValueValue(value), edge, SpecCell | SpecOther, isNotOther(value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void speculateStringOrOther(Edge edge) { speculateStringOrOther(edge, lowJSValue(edge, ManualOperandSpeculation)); } void speculateStringIdent(Edge edge, LValue string, LValue stringImpl) { if (!m_interpreter.needsTypeCheck(edge, SpecStringIdent | ~SpecString)) return; speculate(BadType, jsValueValue(string), edge.node(), m_out.isNull(stringImpl)); speculate( BadType, jsValueValue(string), edge.node(), m_out.testIsZero32( m_out.load32(stringImpl, m_heaps.StringImpl_hashAndFlags), m_out.constInt32(StringImpl::flagIsAtomic()))); m_interpreter.filter(edge, SpecStringIdent | ~SpecString); } void speculateStringIdent(Edge edge) { lowStringIdent(edge); } void speculateStringObject(Edge edge) { if (!m_interpreter.needsTypeCheck(edge, SpecStringObject)) return; speculateStringObjectForCell(edge, lowCell(edge)); m_interpreter.filter(edge, SpecStringObject); } void speculateStringOrStringObject(Edge edge) { if (!m_interpreter.needsTypeCheck(edge, SpecString | SpecStringObject)) return; LBasicBlock notString = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue structureID = m_out.load32(lowCell(edge), m_heaps.JSCell_structureID); m_out.branch( m_out.equal(structureID, m_out.constInt32(vm().stringStructure->id())), unsure(continuation), unsure(notString)); LBasicBlock lastNext = m_out.appendTo(notString, continuation); speculateStringObjectForStructureID(edge, structureID); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); m_interpreter.filter(edge, SpecString | SpecStringObject); } void speculateStringObjectForCell(Edge edge, LValue cell) { speculateStringObjectForStructureID(edge, m_out.load32(cell, m_heaps.JSCell_structureID)); } void speculateStringObjectForStructureID(Edge edge, LValue structureID) { RegisteredStructure stringObjectStructure = m_graph.registerStructure(m_graph.globalObjectFor(m_node->origin.semantic)->stringObjectStructure()); if (abstractStructure(edge).isSubsetOf(RegisteredStructureSet(stringObjectStructure))) return; speculate( NotStringObject, noValue(), 0, m_out.notEqual(structureID, weakStructureID(stringObjectStructure))); } void speculateSymbol(Edge edge, LValue cell) { FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecSymbol | ~SpecCell, isNotSymbol(cell)); } void speculateSymbol(Edge edge) { speculateSymbol(edge, lowCell(edge)); } void speculateNonNullObject(Edge edge, LValue cell) { FTL_TYPE_CHECK(jsValueValue(cell), edge, SpecObject, isNotObject(cell)); if (masqueradesAsUndefinedWatchpointIsStillValid()) return; speculate( BadType, jsValueValue(cell), edge.node(), m_out.testNonZero32( m_out.load8ZeroExt32(cell, m_heaps.JSCell_typeInfoFlags), m_out.constInt32(MasqueradesAsUndefined))); } void speculateNumber(Edge edge) { LValue value = lowJSValue(edge, ManualOperandSpeculation); FTL_TYPE_CHECK(jsValueValue(value), edge, SpecBytecodeNumber, isNotNumber(value)); } void speculateRealNumber(Edge edge) { // Do an early return here because lowDouble() can create a lot of control flow. if (!m_interpreter.needsTypeCheck(edge)) return; LValue value = lowJSValue(edge, ManualOperandSpeculation); LValue doubleValue = unboxDouble(value); LBasicBlock intCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch( m_out.doubleEqual(doubleValue, doubleValue), usually(continuation), rarely(intCase)); LBasicBlock lastNext = m_out.appendTo(intCase, continuation); typeCheck( jsValueValue(value), m_node->child1(), SpecBytecodeRealNumber, isNotInt32(value, provenType(m_node->child1()) & ~SpecFullDouble)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void speculateDoubleRepReal(Edge edge) { // Do an early return here because lowDouble() can create a lot of control flow. if (!m_interpreter.needsTypeCheck(edge)) return; LValue value = lowDouble(edge); FTL_TYPE_CHECK( doubleValue(value), edge, SpecDoubleReal, m_out.doubleNotEqualOrUnordered(value, value)); } void speculateDoubleRepAnyInt(Edge edge) { if (!m_interpreter.needsTypeCheck(edge)) return; doubleToStrictInt52(edge, lowDouble(edge)); } void speculateBoolean(Edge edge) { lowBoolean(edge); } void speculateNotStringVar(Edge edge) { if (!m_interpreter.needsTypeCheck(edge, ~SpecStringVar)) return; LValue value = lowJSValue(edge, ManualOperandSpeculation); LBasicBlock isCellCase = m_out.newBlock(); LBasicBlock isStringCase = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); m_out.branch(isCell(value, provenType(edge)), unsure(isCellCase), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isCellCase, isStringCase); m_out.branch(isString(value, provenType(edge)), unsure(isStringCase), unsure(continuation)); m_out.appendTo(isStringCase, continuation); speculateStringIdent(edge, value, m_out.loadPtr(value, m_heaps.JSString_value)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void speculateOther(Edge edge) { if (!m_interpreter.needsTypeCheck(edge)) return; LValue value = lowJSValue(edge, ManualOperandSpeculation); typeCheck(jsValueValue(value), edge, SpecOther, isNotOther(value)); } void speculateMisc(Edge edge) { if (!m_interpreter.needsTypeCheck(edge)) return; LValue value = lowJSValue(edge, ManualOperandSpeculation); typeCheck(jsValueValue(value), edge, SpecMisc, isNotMisc(value)); } void speculateTypedArrayIsNotNeutered(LValue base) { LBasicBlock isWasteful = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LValue mode = m_out.load32(base, m_heaps.JSArrayBufferView_mode); m_out.branch(m_out.equal(mode, m_out.constInt32(WastefulTypedArray)), unsure(isWasteful), unsure(continuation)); LBasicBlock lastNext = m_out.appendTo(isWasteful, continuation); LValue vector = m_out.loadPtr(base, m_heaps.JSArrayBufferView_vector); speculate(Uncountable, jsValueValue(vector), m_node, m_out.isZero64(vector)); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } bool masqueradesAsUndefinedWatchpointIsStillValid() { return m_graph.masqueradesAsUndefinedWatchpointIsStillValid(m_node->origin.semantic); } LValue loadCellState(LValue base) { return m_out.load8ZeroExt32(base, m_heaps.JSCell_cellState); } void emitStoreBarrier(LValue base, bool isFenced) { LBasicBlock recheckPath = nullptr; if (isFenced) recheckPath = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(isFenced ? recheckPath : slowPath); LValue threshold; if (isFenced) threshold = m_out.load32(m_out.absolute(vm().heap.addressOfBarrierThreshold())); else threshold = m_out.constInt32(blackThreshold); m_out.branch( m_out.above(loadCellState(base), threshold), usually(continuation), rarely(isFenced ? recheckPath : slowPath)); if (isFenced) { m_out.appendTo(recheckPath, slowPath); m_out.fence(&m_heaps.root, &m_heaps.JSCell_cellState); m_out.branch( m_out.above(loadCellState(base), m_out.constInt32(blackThreshold)), usually(continuation), rarely(slowPath)); } m_out.appendTo(slowPath, continuation); LValue call = vmCall(Void, m_out.operation(operationWriteBarrierSlowPath), m_callFrame, base); m_heaps.decorateCCallRead(&m_heaps.root, call); m_heaps.decorateCCallWrite(&m_heaps.JSCell_cellState, call); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void mutatorFence() { if (isX86()) { m_out.fence(&m_heaps.root, nullptr); return; } LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(slowPath); m_out.branch( m_out.load8ZeroExt32(m_out.absolute(vm().heap.addressOfMutatorShouldBeFenced())), rarely(slowPath), usually(continuation)); m_out.appendTo(slowPath, continuation); m_out.fence(&m_heaps.root, nullptr); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } void nukeStructureAndSetButterfly(LValue butterfly, LValue object) { if (isX86()) { m_out.store32( m_out.bitOr( m_out.load32(object, m_heaps.JSCell_structureID), m_out.constInt32(nukedStructureIDBit())), object, m_heaps.JSCell_structureID); m_out.fence(&m_heaps.root, nullptr); m_out.storePtr(butterfly, object, m_heaps.JSObject_butterfly); m_out.fence(&m_heaps.root, nullptr); return; } LBasicBlock fastPath = m_out.newBlock(); LBasicBlock slowPath = m_out.newBlock(); LBasicBlock continuation = m_out.newBlock(); LBasicBlock lastNext = m_out.insertNewBlocksBefore(fastPath); m_out.branch( m_out.load8ZeroExt32(m_out.absolute(vm().heap.addressOfMutatorShouldBeFenced())), rarely(slowPath), usually(fastPath)); m_out.appendTo(fastPath, slowPath); m_out.storePtr(butterfly, object, m_heaps.JSObject_butterfly); m_out.jump(continuation); m_out.appendTo(slowPath, continuation); m_out.store32( m_out.bitOr( m_out.load32(object, m_heaps.JSCell_structureID), m_out.constInt32(nukedStructureIDBit())), object, m_heaps.JSCell_structureID); m_out.fence(&m_heaps.root, nullptr); m_out.storePtr(butterfly, object, m_heaps.JSObject_butterfly); m_out.fence(&m_heaps.root, nullptr); m_out.jump(continuation); m_out.appendTo(continuation, lastNext); } template LValue vmCall(LType type, LValue function, Args... args) { callPreflight(); LValue result = m_out.call(type, function, args...); callCheck(); return result; } void callPreflight(CodeOrigin codeOrigin) { CallSiteIndex callSiteIndex = m_ftlState.jitCode->common.addCodeOrigin(codeOrigin); m_out.store32( m_out.constInt32(callSiteIndex.bits()), tagFor(CallFrameSlot::argumentCount)); } void callPreflight() { callPreflight(codeOriginDescriptionOfCallSite()); } CodeOrigin codeOriginDescriptionOfCallSite() const { CodeOrigin codeOrigin = m_node->origin.semantic; if (m_node->op() == TailCallInlinedCaller || m_node->op() == TailCallVarargsInlinedCaller || m_node->op() == TailCallForwardVarargsInlinedCaller || m_node->op() == DirectTailCallInlinedCaller) { // This case arises when you have a situation like this: // foo makes a call to bar, bar is inlined in foo. bar makes a call // to baz and baz is inlined in bar. And then baz makes a tail-call to jaz, // and jaz is inlined in baz. We want the callframe for jaz to appear to // have caller be bar. codeOrigin = *codeOrigin.inlineCallFrame->getCallerSkippingTailCalls(); } return codeOrigin; } void callCheck() { if (Options::useExceptionFuzz()) m_out.call(Void, m_out.operation(operationExceptionFuzz), m_callFrame); LValue exception = m_out.load64(m_out.absolute(vm().addressOfException())); LValue hadException = m_out.notZero64(exception); CodeOrigin opCatchOrigin; HandlerInfo* exceptionHandler; if (m_graph.willCatchExceptionInMachineFrame(m_origin.forExit, opCatchOrigin, exceptionHandler)) { bool exitOK = true; bool isExceptionHandler = true; appendOSRExit( ExceptionCheck, noValue(), nullptr, hadException, m_origin.withForExitAndExitOK(opCatchOrigin, exitOK), isExceptionHandler); return; } LBasicBlock continuation = m_out.newBlock(); m_out.branch( hadException, rarely(m_handleExceptions), usually(continuation)); m_out.appendTo(continuation); } RefPtr preparePatchpointForExceptions(PatchpointValue* value) { CodeOrigin opCatchOrigin; HandlerInfo* exceptionHandler; bool willCatchException = m_graph.willCatchExceptionInMachineFrame(m_origin.forExit, opCatchOrigin, exceptionHandler); if (!willCatchException) return PatchpointExceptionHandle::defaultHandle(m_ftlState); if (verboseCompilationEnabled()) { dataLog(" Patchpoint exception OSR exit #", m_ftlState.jitCode->osrExitDescriptors.size(), " with availability: ", availabilityMap(), "\n"); if (!m_availableRecoveries.isEmpty()) dataLog(" Available recoveries: ", listDump(m_availableRecoveries), "\n"); } bool exitOK = true; NodeOrigin origin = m_origin.withForExitAndExitOK(opCatchOrigin, exitOK); OSRExitDescriptor* exitDescriptor = appendOSRExitDescriptor(noValue(), nullptr); // Compute the offset into the StackmapGenerationParams where we will find the exit arguments // we are about to append. We need to account for both the children we've already added, and // for the possibility of a result value if the patchpoint is not void. unsigned offset = value->numChildren(); if (value->type() != Void) offset++; // Use LateColdAny to ensure that the stackmap arguments interfere with the patchpoint's // result and with any late-clobbered registers. value->appendVectorWithRep( buildExitArguments(exitDescriptor, opCatchOrigin, noValue()), ValueRep::LateColdAny); return PatchpointExceptionHandle::create( m_ftlState, exitDescriptor, origin, offset, *exceptionHandler); } LBasicBlock lowBlock(DFG::BasicBlock* block) { return m_blocks.get(block); } OSRExitDescriptor* appendOSRExitDescriptor(FormattedValue lowValue, Node* highValue) { return &m_ftlState.jitCode->osrExitDescriptors.alloc( lowValue.format(), m_graph.methodOfGettingAValueProfileFor(m_node, highValue), availabilityMap().m_locals.numberOfArguments(), availabilityMap().m_locals.numberOfLocals()); } void appendOSRExit( ExitKind kind, FormattedValue lowValue, Node* highValue, LValue failCondition, NodeOrigin origin, bool isExceptionHandler = false) { if (verboseCompilationEnabled()) { dataLog(" OSR exit #", m_ftlState.jitCode->osrExitDescriptors.size(), " with availability: ", availabilityMap(), "\n"); if (!m_availableRecoveries.isEmpty()) dataLog(" Available recoveries: ", listDump(m_availableRecoveries), "\n"); } DFG_ASSERT(m_graph, m_node, origin.exitOK); if (!isExceptionHandler && Options::useOSRExitFuzz() && canUseOSRExitFuzzing(m_graph.baselineCodeBlockFor(m_node->origin.semantic)) && doOSRExitFuzzing()) { LValue numberOfFuzzChecks = m_out.add( m_out.load32(m_out.absolute(&g_numberOfOSRExitFuzzChecks)), m_out.int32One); m_out.store32(numberOfFuzzChecks, m_out.absolute(&g_numberOfOSRExitFuzzChecks)); if (unsigned atOrAfter = Options::fireOSRExitFuzzAtOrAfter()) { failCondition = m_out.bitOr( failCondition, m_out.aboveOrEqual(numberOfFuzzChecks, m_out.constInt32(atOrAfter))); } if (unsigned at = Options::fireOSRExitFuzzAt()) { failCondition = m_out.bitOr( failCondition, m_out.equal(numberOfFuzzChecks, m_out.constInt32(at))); } } if (failCondition == m_out.booleanFalse) return; blessSpeculation( m_out.speculate(failCondition), kind, lowValue, highValue, origin); } void blessSpeculation(CheckValue* value, ExitKind kind, FormattedValue lowValue, Node* highValue, NodeOrigin origin) { OSRExitDescriptor* exitDescriptor = appendOSRExitDescriptor(lowValue, highValue); value->appendColdAnys(buildExitArguments(exitDescriptor, origin.forExit, lowValue)); State* state = &m_ftlState; value->setGenerator( [=] (CCallHelpers& jit, const B3::StackmapGenerationParams& params) { exitDescriptor->emitOSRExit( *state, kind, origin, jit, params, 0); }); } StackmapArgumentList buildExitArguments( OSRExitDescriptor* exitDescriptor, CodeOrigin exitOrigin, FormattedValue lowValue, unsigned offsetOfExitArgumentsInStackmapLocations = 0) { StackmapArgumentList result; buildExitArguments( exitDescriptor, exitOrigin, result, lowValue, offsetOfExitArgumentsInStackmapLocations); return result; } void buildExitArguments( OSRExitDescriptor* exitDescriptor, CodeOrigin exitOrigin, StackmapArgumentList& arguments, FormattedValue lowValue, unsigned offsetOfExitArgumentsInStackmapLocations = 0) { if (!!lowValue) arguments.append(lowValue.value()); AvailabilityMap availabilityMap = this->availabilityMap(); availabilityMap.pruneByLiveness(m_graph, exitOrigin); HashMap map; availabilityMap.forEachAvailability( [&] (Availability availability) { if (!availability.shouldUseNode()) return; Node* node = availability.node(); if (!node->isPhantomAllocation()) return; auto result = map.add(node, nullptr); if (result.isNewEntry) { result.iterator->value = exitDescriptor->m_materializations.add(node->op(), node->origin.semantic); } }); for (unsigned i = 0; i < exitDescriptor->m_values.size(); ++i) { int operand = exitDescriptor->m_values.operandForIndex(i); Availability availability = availabilityMap.m_locals[i]; if (Options::validateFTLOSRExitLiveness() && m_graph.m_plan.mode != FTLForOSREntryMode) { if (availability.isDead() && m_graph.isLiveInBytecode(VirtualRegister(operand), exitOrigin)) DFG_CRASH(m_graph, m_node, toCString("Live bytecode local not available: operand = ", VirtualRegister(operand), ", availability = ", availability, ", origin = ", exitOrigin).data()); } ExitValue exitValue = exitValueForAvailability(arguments, map, availability); if (exitValue.hasIndexInStackmapLocations()) exitValue.adjustStackmapLocationsIndexByOffset(offsetOfExitArgumentsInStackmapLocations); exitDescriptor->m_values[i] = exitValue; } for (auto heapPair : availabilityMap.m_heap) { Node* node = heapPair.key.base(); ExitTimeObjectMaterialization* materialization = map.get(node); ExitValue exitValue = exitValueForAvailability(arguments, map, heapPair.value); if (exitValue.hasIndexInStackmapLocations()) exitValue.adjustStackmapLocationsIndexByOffset(offsetOfExitArgumentsInStackmapLocations); materialization->add( heapPair.key.descriptor(), exitValue); } if (verboseCompilationEnabled()) { dataLog(" Exit values: ", exitDescriptor->m_values, "\n"); if (!exitDescriptor->m_materializations.isEmpty()) { dataLog(" Materializations: \n"); for (ExitTimeObjectMaterialization* materialization : exitDescriptor->m_materializations) dataLog(" ", pointerDump(materialization), "\n"); } } } ExitValue exitValueForAvailability( StackmapArgumentList& arguments, const HashMap& map, Availability availability) { FlushedAt flush = availability.flushedAt(); switch (flush.format()) { case DeadFlush: case ConflictingFlush: if (availability.hasNode()) return exitValueForNode(arguments, map, availability.node()); // This means that the value is dead. It could be dead in bytecode or it could have // been killed by our DCE, which can sometimes kill things even if they were live in // bytecode. return ExitValue::dead(); case FlushedJSValue: case FlushedCell: case FlushedBoolean: return ExitValue::inJSStack(flush.virtualRegister()); case FlushedInt32: return ExitValue::inJSStackAsInt32(flush.virtualRegister()); case FlushedInt52: return ExitValue::inJSStackAsInt52(flush.virtualRegister()); case FlushedDouble: return ExitValue::inJSStackAsDouble(flush.virtualRegister()); } DFG_CRASH(m_graph, m_node, "Invalid flush format"); return ExitValue::dead(); } ExitValue exitValueForNode( StackmapArgumentList& arguments, const HashMap& map, Node* node) { // NOTE: In FTL->B3, we cannot generate code here, because m_output is positioned after the // stackmap value. Like all values, the stackmap value cannot use a child that is defined after // it. ASSERT(node->shouldGenerate()); ASSERT(node->hasResult()); if (node) { switch (node->op()) { case BottomValue: // This might arise in object materializations. I actually doubt that it would, // but it seems worthwhile to be conservative. return ExitValue::dead(); case JSConstant: case Int52Constant: case DoubleConstant: return ExitValue::constant(node->asJSValue()); default: if (node->isPhantomAllocation()) return ExitValue::materializeNewObject(map.get(node)); break; } } for (unsigned i = 0; i < m_availableRecoveries.size(); ++i) { AvailableRecovery recovery = m_availableRecoveries[i]; if (recovery.node() != node) continue; ExitValue result = ExitValue::recovery( recovery.opcode(), arguments.size(), arguments.size() + 1, recovery.format()); arguments.append(recovery.left()); arguments.append(recovery.right()); return result; } LoweredNodeValue value = m_int32Values.get(node); if (isValid(value)) return exitArgument(arguments, DataFormatInt32, value.value()); value = m_int52Values.get(node); if (isValid(value)) return exitArgument(arguments, DataFormatInt52, value.value()); value = m_strictInt52Values.get(node); if (isValid(value)) return exitArgument(arguments, DataFormatStrictInt52, value.value()); value = m_booleanValues.get(node); if (isValid(value)) return exitArgument(arguments, DataFormatBoolean, value.value()); value = m_jsValueValues.get(node); if (isValid(value)) return exitArgument(arguments, DataFormatJS, value.value()); value = m_doubleValues.get(node); if (isValid(value)) return exitArgument(arguments, DataFormatDouble, value.value()); DFG_CRASH(m_graph, m_node, toCString("Cannot find value for node: ", node).data()); return ExitValue::dead(); } ExitValue exitArgument(StackmapArgumentList& arguments, DataFormat format, LValue value) { ExitValue result = ExitValue::exitArgument(ExitArgument(format, arguments.size())); arguments.append(value); return result; } ExitValue exitValueForTailCall(StackmapArgumentList& arguments, Node* node) { ASSERT(node->shouldGenerate()); ASSERT(node->hasResult()); switch (node->op()) { case JSConstant: case Int52Constant: case DoubleConstant: return ExitValue::constant(node->asJSValue()); default: break; } LoweredNodeValue value = m_jsValueValues.get(node); if (isValid(value)) return exitArgument(arguments, DataFormatJS, value.value()); value = m_int32Values.get(node); if (isValid(value)) return exitArgument(arguments, DataFormatJS, boxInt32(value.value())); value = m_booleanValues.get(node); if (isValid(value)) return exitArgument(arguments, DataFormatJS, boxBoolean(value.value())); // Doubles and Int52 have been converted by ValueRep() DFG_CRASH(m_graph, m_node, toCString("Cannot find value for node: ", node).data()); } void addAvailableRecovery( Node* node, RecoveryOpcode opcode, LValue left, LValue right, DataFormat format) { m_availableRecoveries.append(AvailableRecovery(node, opcode, left, right, format)); } void addAvailableRecovery( Edge edge, RecoveryOpcode opcode, LValue left, LValue right, DataFormat format) { addAvailableRecovery(edge.node(), opcode, left, right, format); } void setInt32(Node* node, LValue value) { m_int32Values.set(node, LoweredNodeValue(value, m_highBlock)); } void setInt52(Node* node, LValue value) { m_int52Values.set(node, LoweredNodeValue(value, m_highBlock)); } void setStrictInt52(Node* node, LValue value) { m_strictInt52Values.set(node, LoweredNodeValue(value, m_highBlock)); } void setInt52(Node* node, LValue value, Int52Kind kind) { switch (kind) { case Int52: setInt52(node, value); return; case StrictInt52: setStrictInt52(node, value); return; } DFG_CRASH(m_graph, m_node, "Corrupt int52 kind"); } void setJSValue(Node* node, LValue value) { m_jsValueValues.set(node, LoweredNodeValue(value, m_highBlock)); } void setBoolean(Node* node, LValue value) { m_booleanValues.set(node, LoweredNodeValue(value, m_highBlock)); } void setStorage(Node* node, LValue value) { m_storageValues.set(node, LoweredNodeValue(value, m_highBlock)); } void setMapBucket(Node* node, LValue value) { m_mapBucketValues.set(node, LoweredNodeValue(value, m_highBlock)); } void setDouble(Node* node, LValue value) { m_doubleValues.set(node, LoweredNodeValue(value, m_highBlock)); } void setInt32(LValue value) { setInt32(m_node, value); } void setInt52(LValue value) { setInt52(m_node, value); } void setStrictInt52(LValue value) { setStrictInt52(m_node, value); } void setInt52(LValue value, Int52Kind kind) { setInt52(m_node, value, kind); } void setJSValue(LValue value) { setJSValue(m_node, value); } void setBoolean(LValue value) { setBoolean(m_node, value); } void setStorage(LValue value) { setStorage(m_node, value); } void setMapBucket(LValue value) { setMapBucket(m_node, value); } void setDouble(LValue value) { setDouble(m_node, value); } bool isValid(const LoweredNodeValue& value) { if (!value) return false; if (!m_graph.m_dominators->dominates(value.block(), m_highBlock)) return false; return true; } void addWeakReference(JSCell* target) { m_graph.m_plan.weakReferences.addLazily(target); } LValue loadStructure(LValue value) { LValue tableIndex = m_out.load32(value, m_heaps.JSCell_structureID); LValue tableBase = m_out.loadPtr( m_out.absolute(vm().heap.structureIDTable().base())); TypedPointer address = m_out.baseIndex( m_heaps.structureTable, tableBase, m_out.zeroExtPtr(tableIndex)); return m_out.loadPtr(address); } LValue weakPointer(JSCell* pointer) { // There are weird relationships in how optimized CodeBlocks // point to other CodeBlocks. We don't want to have them be // part of the weak pointer set. For example, an optimized CodeBlock // having a weak pointer to itself will cause it to get collected. RELEASE_ASSERT(!jsDynamicCast(vm(), pointer)); addWeakReference(pointer); return m_out.weakPointer(m_graph, pointer); } LValue frozenPointer(FrozenValue* value) { return m_out.weakPointer(value); } LValue weakStructureID(RegisteredStructure structure) { return m_out.constInt32(structure->id()); } LValue weakStructure(RegisteredStructure structure) { ASSERT(!!structure.get()); return m_out.weakPointer(m_graph, structure.get()); } TypedPointer addressFor(LValue base, int operand, ptrdiff_t offset = 0) { return m_out.address(base, m_heaps.variables[operand], offset); } TypedPointer payloadFor(LValue base, int operand) { return addressFor(base, operand, PayloadOffset); } TypedPointer tagFor(LValue base, int operand) { return addressFor(base, operand, TagOffset); } TypedPointer addressFor(int operand, ptrdiff_t offset = 0) { return addressFor(VirtualRegister(operand), offset); } TypedPointer addressFor(VirtualRegister operand, ptrdiff_t offset = 0) { if (operand.isLocal()) return addressFor(m_captured, operand.offset(), offset); return addressFor(m_callFrame, operand.offset(), offset); } TypedPointer payloadFor(int operand) { return payloadFor(VirtualRegister(operand)); } TypedPointer payloadFor(VirtualRegister operand) { return addressFor(operand, PayloadOffset); } TypedPointer tagFor(int operand) { return tagFor(VirtualRegister(operand)); } TypedPointer tagFor(VirtualRegister operand) { return addressFor(operand, TagOffset); } AbstractValue abstractValue(Node* node) { return m_state.forNode(node); } AbstractValue abstractValue(Edge edge) { return abstractValue(edge.node()); } SpeculatedType provenType(Node* node) { return abstractValue(node).m_type; } SpeculatedType provenType(Edge edge) { return provenType(edge.node()); } JSValue provenValue(Node* node) { return abstractValue(node).m_value; } JSValue provenValue(Edge edge) { return provenValue(edge.node()); } StructureAbstractValue abstractStructure(Node* node) { return abstractValue(node).m_structure; } StructureAbstractValue abstractStructure(Edge edge) { return abstractStructure(edge.node()); } #if ENABLE(MASM_PROBE) void probe(std::function probeFunc) { UNUSED_PARAM(probeFunc); } #endif void crash() { crash(m_highBlock, m_node); } void crash(DFG::BasicBlock* block, Node* node) { BlockIndex blockIndex = block->index; unsigned nodeIndex = node ? node->index() : UINT_MAX; #if ASSERT_DISABLED m_out.patchpoint(Void)->setGenerator( [=] (CCallHelpers& jit, const StackmapGenerationParams&) { AllowMacroScratchRegisterUsage allowScratch(jit); jit.move(CCallHelpers::TrustedImm32(blockIndex), GPRInfo::regT0); jit.move(CCallHelpers::TrustedImm32(nodeIndex), GPRInfo::regT1); if (node) jit.move(CCallHelpers::TrustedImm32(node->op()), GPRInfo::regT2); jit.abortWithReason(FTLCrash); }); #else m_out.call( Void, m_out.constIntPtr(ftlUnreachable), // We don't want the CodeBlock to have a weak pointer to itself because // that would cause it to always get collected. m_out.constIntPtr(bitwise_cast(codeBlock())), m_out.constInt32(blockIndex), m_out.constInt32(nodeIndex)); #endif m_out.unreachable(); } AvailabilityMap& availabilityMap() { return m_availabilityCalculator.m_availability; } VM& vm() { return m_graph.m_vm; } CodeBlock* codeBlock() { return m_graph.m_codeBlock; } Graph& m_graph; State& m_ftlState; AbstractHeapRepository m_heaps; Output m_out; Procedure& m_proc; LBasicBlock m_prologue; LBasicBlock m_handleExceptions; HashMap m_blocks; LValue m_callFrame; LValue m_captured; LValue m_tagTypeNumber; LValue m_tagMask; HashMap m_int32Values; HashMap m_strictInt52Values; HashMap m_int52Values; HashMap m_jsValueValues; HashMap m_booleanValues; HashMap m_storageValues; HashMap m_mapBucketValues; HashMap m_doubleValues; // This is a bit of a hack. It prevents B3 from having to do CSE on loading of arguments. // It's nice to have these optimizations on our end because we can guarantee them a bit better. // Probably also saves B3 compile time. HashMap m_loadedArgumentValues; HashMap m_phis; LocalOSRAvailabilityCalculator m_availabilityCalculator; Vector m_availableRecoveries; InPlaceAbstractState m_state; AbstractInterpreter m_interpreter; DFG::BasicBlock* m_highBlock; DFG::BasicBlock* m_nextHighBlock; LBasicBlock m_nextLowBlock; NodeOrigin m_origin; unsigned m_nodeIndex; Node* m_node; }; } // anonymous namespace void lowerDFGToB3(State& state) { LowerDFGToB3 lowering(state); lowering.lower(); } } } // namespace JSC::FTL #endif // ENABLE(FTL_JIT)