diff options
Diffstat (limited to 'Source/JavaScriptCore/ftl/FTLCompile.cpp')
| -rw-r--r-- | Source/JavaScriptCore/ftl/FTLCompile.cpp | 458 |
1 files changed, 89 insertions, 369 deletions
diff --git a/Source/JavaScriptCore/ftl/FTLCompile.cpp b/Source/JavaScriptCore/ftl/FTLCompile.cpp index 6c01f4fa5..e85e28778 100644 --- a/Source/JavaScriptCore/ftl/FTLCompile.cpp +++ b/Source/JavaScriptCore/ftl/FTLCompile.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2013 Apple Inc. All rights reserved. + * Copyright (C) 2015-2016 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -28,409 +28,129 @@ #if ENABLE(FTL_JIT) +#include "AirCode.h" +#include "B3Generate.h" +#include "B3ProcedureInlines.h" +#include "B3StackSlot.h" #include "CodeBlockWithJITType.h" #include "CCallHelpers.h" #include "DFGCommon.h" +#include "DFGGraphSafepoint.h" +#include "DFGOperations.h" #include "DataView.h" #include "Disassembler.h" -#include "FTLExitThunkGenerator.h" -#include "FTLInlineCacheSize.h" #include "FTLJITCode.h" #include "FTLThunks.h" -#include "JITStubs.h" -#include "LLVMAPI.h" +#include "JITSubGenerator.h" +#include "JSCInlines.h" #include "LinkBuffer.h" -#include "RepatchBuffer.h" +#include "PCToCodeOriginMap.h" +#include "ScratchRegisterAllocator.h" namespace JSC { namespace FTL { using namespace DFG; -static uint8_t* mmAllocateCodeSection( - void* opaqueState, uintptr_t size, unsigned alignment, unsigned, const char* sectionName) +void compile(State& state, Safepoint::Result& safepointResult) { - - State& state = *static_cast<State*>(opaqueState); - - RELEASE_ASSERT(alignment <= jitAllocationGranule); - - RefPtr<ExecutableMemoryHandle> result = - state.graph.m_vm.executableAllocator.allocate( - state.graph.m_vm, size, state.graph.m_codeBlock, JITCompilationMustSucceed); - - state.jitCode->addHandle(result); - state.codeSectionNames.append(sectionName); - - return static_cast<uint8_t*>(result->start()); -} - -static uint8_t* mmAllocateDataSection( - void* opaqueState, uintptr_t size, unsigned alignment, unsigned sectionID, - const char* sectionName, LLVMBool isReadOnly) -{ - UNUSED_PARAM(sectionID); - UNUSED_PARAM(isReadOnly); - - State& state = *static_cast<State*>(opaqueState); - - RELEASE_ASSERT(alignment <= sizeof(LSectionWord)); - - RefCountedArray<LSectionWord> section( - (size + sizeof(LSectionWord) - 1) / sizeof(LSectionWord)); - - if (!strcmp(sectionName, "__llvm_stackmaps")) - state.stackmapsSection = section; - else { - state.jitCode->addDataSection(section); - state.dataSectionNames.append(sectionName); - } - - return bitwise_cast<uint8_t*>(section.data()); -} + Graph& graph = state.graph; + CodeBlock* codeBlock = graph.m_codeBlock; + VM& vm = graph.m_vm; -static LLVMBool mmApplyPermissions(void*, char**) -{ - return false; -} - -static void mmDestroy(void*) -{ -} + { + GraphSafepoint safepoint(state.graph, safepointResult); -static void dumpDataSection(RefCountedArray<LSectionWord> section, const char* prefix) -{ - for (unsigned j = 0; j < section.size(); ++j) { - char buf[32]; - snprintf(buf, sizeof(buf), "0x%lx", static_cast<unsigned long>(bitwise_cast<uintptr_t>(section.data() + j))); - dataLogF("%s%16s: 0x%016llx\n", prefix, buf, static_cast<long long>(section[j])); + B3::prepareForGeneration(*state.proc); } -} -template<typename DescriptorType> -void generateICFastPath( - State& state, CodeBlock* codeBlock, GeneratedFunction generatedFunction, - StackMaps::RecordMap& recordMap, DescriptorType& ic, size_t sizeOfIC) -{ - VM& vm = state.graph.m_vm; - - StackMaps::RecordMap::iterator iter = recordMap.find(ic.stackmapID()); - if (iter == recordMap.end()) { - // It was optimized out. + if (safepointResult.didGetCancelled()) return; - } + RELEASE_ASSERT(!state.graph.m_vm.heap.collectorBelievesThatTheWorldIsStopped()); - StackMaps::Record& record = iter->value; - - CCallHelpers fastPathJIT(&vm, codeBlock); - ic.m_generator.generateFastPath(fastPathJIT); - - char* startOfIC = - bitwise_cast<char*>(generatedFunction) + record.instructionOffset; - - LinkBuffer linkBuffer(vm, &fastPathJIT, startOfIC, sizeOfIC); - // Note: we could handle the !isValid() case. We just don't appear to have a - // reason to do so, yet. - RELEASE_ASSERT(linkBuffer.isValid()); - - MacroAssembler::AssemblerType_T::fillNops( - startOfIC + linkBuffer.size(), sizeOfIC - linkBuffer.size()); + if (state.allocationFailed) + return; - state.finalizer->sideCodeLinkBuffer->link( - ic.m_slowPathDone, CodeLocationLabel(startOfIC + sizeOfIC)); - - linkBuffer.link( - ic.m_generator.slowPathJump(), - state.finalizer->sideCodeLinkBuffer->locationOf(ic.m_generator.slowPathBegin())); - - ic.m_generator.finalize(linkBuffer, *state.finalizer->sideCodeLinkBuffer); -} - -static void fixFunctionBasedOnStackMaps( - State& state, CodeBlock* codeBlock, JITCode* jitCode, GeneratedFunction generatedFunction, - StackMaps::RecordMap& recordMap) -{ - VM& vm = state.graph.m_vm; - StackMaps stackmaps = jitCode->stackmaps; - - ExitThunkGenerator exitThunkGenerator(state); - exitThunkGenerator.emitThunks(); - if (exitThunkGenerator.didThings()) { - OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer( - vm, &exitThunkGenerator, codeBlock, JITCompilationMustSucceed)); - - ASSERT(state.finalizer->osrExit.size() == state.jitCode->osrExit.size()); - - for (unsigned i = 0; i < state.jitCode->osrExit.size(); ++i) { - OSRExitCompilationInfo& info = state.finalizer->osrExit[i]; - OSRExit& exit = jitCode->osrExit[i]; - - if (Options::verboseCompilation()) - dataLog("Handling OSR stackmap #", exit.m_stackmapID, " for ", exit.m_codeOrigin, "\n"); - - StackMaps::RecordMap::iterator iter = recordMap.find(exit.m_stackmapID); - if (iter == recordMap.end()) { - // It was optimized out. - continue; - } - - info.m_thunkAddress = linkBuffer->locationOf(info.m_thunkLabel); - - exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump); - } - - state.finalizer->exitThunksLinkBuffer = linkBuffer.release(); + std::unique_ptr<RegisterAtOffsetList> registerOffsets = + std::make_unique<RegisterAtOffsetList>(state.proc->calleeSaveRegisters()); + if (shouldDumpDisassembly()) { + dataLog("Unwind info for ", CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT), ":\n"); + dataLog(" ", *registerOffsets, "\n"); } - - if (!state.getByIds.isEmpty() || !state.putByIds.isEmpty()) { - CCallHelpers slowPathJIT(&vm, codeBlock); - - for (unsigned i = state.getByIds.size(); i--;) { - GetByIdDescriptor& getById = state.getByIds[i]; - - if (Options::verboseCompilation()) - dataLog("Handling GetById stackmap #", getById.stackmapID(), "\n"); - - StackMaps::RecordMap::iterator iter = recordMap.find(getById.stackmapID()); - if (iter == recordMap.end()) { - // It was optimized out. - continue; - } - - StackMaps::Record& record = iter->value; - - // FIXME: LLVM should tell us which registers are live. - RegisterSet usedRegisters = RegisterSet::allRegisters(); - - GPRReg result = record.locations[0].directGPR(); - GPRReg callFrameRegister = record.locations[1].directGPR(); - GPRReg base = record.locations[2].directGPR(); - - JITGetByIdGenerator gen( - codeBlock, getById.codeOrigin(), usedRegisters, callFrameRegister, - JSValueRegs(base), JSValueRegs(result), false); - - MacroAssembler::Label begin = slowPathJIT.label(); - - MacroAssembler::Call call = callOperation( - state, usedRegisters, slowPathJIT, operationGetByIdOptimize, result, - callFrameRegister, gen.stubInfo(), base, getById.uid()); - - gen.reportSlowPathCall(begin, call); - - getById.m_slowPathDone = slowPathJIT.jump(); - getById.m_generator = gen; - } - - for (unsigned i = state.putByIds.size(); i--;) { - PutByIdDescriptor& putById = state.putByIds[i]; - - if (Options::verboseCompilation()) - dataLog("Handling PutById stackmap #", putById.stackmapID(), "\n"); - - StackMaps::RecordMap::iterator iter = recordMap.find(putById.stackmapID()); - if (iter == recordMap.end()) { - // It was optimized out. - continue; - } - - StackMaps::Record& record = iter->value; - - // FIXME: LLVM should tell us which registers are live. - RegisterSet usedRegisters = RegisterSet::allRegisters(); - - GPRReg callFrameRegister = record.locations[0].directGPR(); - GPRReg base = record.locations[1].directGPR(); - GPRReg value = record.locations[2].directGPR(); - - JITPutByIdGenerator gen( - codeBlock, putById.codeOrigin(), usedRegisters, callFrameRegister, - JSValueRegs(base), JSValueRegs(value), MacroAssembler::scratchRegister, - false, putById.ecmaMode(), putById.putKind()); - - MacroAssembler::Label begin = slowPathJIT.label(); - - MacroAssembler::Call call = callOperation( - state, usedRegisters, slowPathJIT, gen.slowPathFunction(), callFrameRegister, - gen.stubInfo(), value, base, putById.uid()); - - gen.reportSlowPathCall(begin, call); - - putById.m_slowPathDone = slowPathJIT.jump(); - putById.m_generator = gen; - } - - state.finalizer->sideCodeLinkBuffer = adoptPtr( - new LinkBuffer(vm, &slowPathJIT, codeBlock, JITCompilationMustSucceed)); - - for (unsigned i = state.getByIds.size(); i--;) { - generateICFastPath( - state, codeBlock, generatedFunction, recordMap, state.getByIds[i], - sizeOfGetById()); - } - for (unsigned i = state.putByIds.size(); i--;) { - generateICFastPath( - state, codeBlock, generatedFunction, recordMap, state.putByIds[i], - sizeOfPutById()); - } + state.graph.m_codeBlock->setCalleeSaveRegisters(WTFMove(registerOffsets)); + ASSERT(!(state.proc->frameSize() % sizeof(EncodedJSValue))); + state.jitCode->common.frameRegisterCount = state.proc->frameSize() / sizeof(EncodedJSValue); + + int localsOffset = + state.capturedValue->offsetFromFP() / sizeof(EncodedJSValue) + graph.m_nextMachineLocal; + if (shouldDumpDisassembly()) { + dataLog( + "localsOffset = ", localsOffset, " for stack slot: ", + pointerDump(state.capturedValue), " at ", RawPointer(state.capturedValue), "\n"); } - RepatchBuffer repatchBuffer(codeBlock); - - for (unsigned exitIndex = jitCode->osrExit.size(); exitIndex--;) { - OSRExitCompilationInfo& info = state.finalizer->osrExit[exitIndex]; - OSRExit& exit = jitCode->osrExit[exitIndex]; - StackMaps::RecordMap::iterator iter = recordMap.find(exit.m_stackmapID); - if (iter == recordMap.end()) { - // This could happen if LLVM optimizes out an OSR exit. - continue; - } + for (unsigned i = graph.m_inlineVariableData.size(); i--;) { + InlineCallFrame* inlineCallFrame = graph.m_inlineVariableData[i].inlineCallFrame; - StackMaps::Record& record = iter->value; + if (inlineCallFrame->argumentCountRegister.isValid()) + inlineCallFrame->argumentCountRegister += localsOffset; - CodeLocationLabel source = CodeLocationLabel( - bitwise_cast<char*>(generatedFunction) + record.instructionOffset); - - if (info.m_isInvalidationPoint) { - jitCode->common.jumpReplacements.append(JumpReplacement(source, info.m_thunkAddress)); - continue; + for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) { + inlineCallFrame->arguments[argument] = + inlineCallFrame->arguments[argument].withLocalsOffset(localsOffset); } - repatchBuffer.replaceWithJump(source, info.m_thunkAddress); - } -} + if (inlineCallFrame->isClosureCall) { + inlineCallFrame->calleeRecovery = + inlineCallFrame->calleeRecovery.withLocalsOffset(localsOffset); + } -void compile(State& state) -{ - char* error = 0; - - LLVMMCJITCompilerOptions options; - llvm->InitializeMCJITCompilerOptions(&options, sizeof(options)); - options.OptLevel = Options::llvmBackendOptimizationLevel(); - options.NoFramePointerElim = true; - if (Options::useLLVMSmallCodeModel()) - options.CodeModel = LLVMCodeModelSmall; - options.EnableFastISel = Options::enableLLVMFastISel(); - options.MCJMM = llvm->CreateSimpleMCJITMemoryManager( - &state, mmAllocateCodeSection, mmAllocateDataSection, mmApplyPermissions, mmDestroy); - - LLVMExecutionEngineRef engine; - - if (llvm->CreateMCJITCompilerForModule(&engine, state.module, &options, sizeof(options), &error)) { - dataLog("FATAL: Could not create LLVM execution engine: ", error, "\n"); - CRASH(); } - LLVMPassManagerRef functionPasses = 0; - LLVMPassManagerRef modulePasses; - - if (Options::llvmSimpleOpt()) { - modulePasses = llvm->CreatePassManager(); - llvm->AddTargetData(llvm->GetExecutionEngineTargetData(engine), modulePasses); - llvm->AddPromoteMemoryToRegisterPass(modulePasses); - llvm->AddConstantPropagationPass(modulePasses); - llvm->AddInstructionCombiningPass(modulePasses); - llvm->AddBasicAliasAnalysisPass(modulePasses); - llvm->AddTypeBasedAliasAnalysisPass(modulePasses); - llvm->AddGVNPass(modulePasses); - llvm->AddCFGSimplificationPass(modulePasses); - llvm->RunPassManager(modulePasses, state.module); - } else { - LLVMPassManagerBuilderRef passBuilder = llvm->PassManagerBuilderCreate(); - llvm->PassManagerBuilderSetOptLevel(passBuilder, Options::llvmOptimizationLevel()); - llvm->PassManagerBuilderSetSizeLevel(passBuilder, Options::llvmSizeLevel()); - - functionPasses = llvm->CreateFunctionPassManagerForModule(state.module); - modulePasses = llvm->CreatePassManager(); - - llvm->AddTargetData(llvm->GetExecutionEngineTargetData(engine), modulePasses); - - llvm->PassManagerBuilderPopulateFunctionPassManager(passBuilder, functionPasses); - llvm->PassManagerBuilderPopulateModulePassManager(passBuilder, modulePasses); - - llvm->PassManagerBuilderDispose(passBuilder); - - llvm->InitializeFunctionPassManager(functionPasses); - for (LValue function = llvm->GetFirstFunction(state.module); function; function = llvm->GetNextFunction(function)) - llvm->RunFunctionPassManager(functionPasses, function); - llvm->FinalizeFunctionPassManager(functionPasses); - - llvm->RunPassManager(modulePasses, state.module); + // Note that the scope register could be invalid here if the original code had CallEval but it + // got killed. That's because it takes the CallEval to cause the scope register to be kept alive + // unless the debugger is also enabled. + if (graph.needsScopeRegister() && codeBlock->scopeRegister().isValid()) + codeBlock->setScopeRegister(codeBlock->scopeRegister() + localsOffset); + + for (OSRExitDescriptor& descriptor : state.jitCode->osrExitDescriptors) { + for (unsigned i = descriptor.m_values.size(); i--;) + descriptor.m_values[i] = descriptor.m_values[i].withLocalsOffset(localsOffset); + for (ExitTimeObjectMaterialization* materialization : descriptor.m_materializations) + materialization->accountForLocalsOffset(localsOffset); } - if (DFG::shouldShowDisassembly() || DFG::verboseCompilationEnabled()) - state.dumpState("after optimization"); - - // FIXME: Need to add support for the case where JIT memory allocation failed. - // https://bugs.webkit.org/show_bug.cgi?id=113620 - state.generatedFunction = reinterpret_cast<GeneratedFunction>(llvm->GetPointerToGlobal(engine, state.function)); - if (functionPasses) - llvm->DisposePassManager(functionPasses); - llvm->DisposePassManager(modulePasses); - llvm->DisposeExecutionEngine(engine); - - if (shouldShowDisassembly()) { - for (unsigned i = 0; i < state.jitCode->handles().size(); ++i) { - ExecutableMemoryHandle* handle = state.jitCode->handles()[i].get(); - dataLog( - "Generated LLVM code for ", - CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::DFGJIT), - " #", i, ", ", state.codeSectionNames[i], ":\n"); - disassemble( - MacroAssemblerCodePtr(handle->start()), handle->sizeInBytes(), - " ", WTF::dataFile(), LLVMSubset); - } - - for (unsigned i = 0; i < state.jitCode->dataSections().size(); ++i) { - const RefCountedArray<LSectionWord>& section = state.jitCode->dataSections()[i]; - dataLog( - "Generated LLVM data section for ", - CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::DFGJIT), - " #", i, ", ", state.dataSectionNames[i], ":\n"); - dumpDataSection(section, " "); - } - } - - if (state.stackmapsSection.size()) { - if (shouldShowDisassembly()) { - dataLog( - "Generated LLVM stackmaps section for ", - CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::DFGJIT), ":\n"); - dataLog(" Raw data:\n"); - dumpDataSection(state.stackmapsSection, " "); - } - - RefPtr<DataView> stackmapsData = DataView::create( - ArrayBuffer::create(state.stackmapsSection.data(), state.stackmapsSection.byteSize())); - state.jitCode->stackmaps.parse(stackmapsData.get()); - - if (shouldShowDisassembly()) { - dataLog(" Structured data:\n"); - state.jitCode->stackmaps.dumpMultiline(WTF::dataFile(), " "); - } - - StackMaps::RecordMap recordMap = state.jitCode->stackmaps.getRecordMap(); - fixFunctionBasedOnStackMaps( - state, state.graph.m_codeBlock, state.jitCode.get(), state.generatedFunction, - recordMap); - - if (shouldShowDisassembly()) { - for (unsigned i = 0; i < state.jitCode->handles().size(); ++i) { - if (state.codeSectionNames[i] != "__text") - continue; - - ExecutableMemoryHandle* handle = state.jitCode->handles()[i].get(); - dataLog( - "Generated LLVM code after stackmap-based fix-up for ", - CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::DFGJIT), - " #", i, ", ", state.codeSectionNames[i], ":\n"); - disassemble( - MacroAssemblerCodePtr(handle->start()), handle->sizeInBytes(), - " ", WTF::dataFile(), LLVMSubset); - } - } + // We will add exception handlers while generating. + codeBlock->clearExceptionHandlers(); + + CCallHelpers jit(&vm, codeBlock); + B3::generate(*state.proc, jit); + + // Emit the exception handler. + *state.exceptionHandler = jit.label(); + jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(); + jit.move(MacroAssembler::TrustedImmPtr(jit.vm()), GPRInfo::argumentGPR0); + jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); + CCallHelpers::Call call = jit.call(); + jit.jumpToExceptionHandler(); + jit.addLinkTask( + [=] (LinkBuffer& linkBuffer) { + linkBuffer.link(call, FunctionPtr(lookupExceptionHandler)); + }); + + state.finalizer->b3CodeLinkBuffer = std::make_unique<LinkBuffer>( + vm, jit, codeBlock, JITCompilationCanFail); + if (state.finalizer->b3CodeLinkBuffer->didFailToAllocate()) { + state.allocationFailed = true; + return; } - state.module = 0; // We no longer own the module. + B3::PCToOriginMap originMap = state.proc->releasePCToOriginMap(); + if (vm.shouldBuilderPCToCodeOriginMapping()) + codeBlock->setPCToCodeOriginMap(std::make_unique<PCToCodeOriginMap>(PCToCodeOriginMapBuilder(vm, WTFMove(originMap)), *state.finalizer->b3CodeLinkBuffer)); + + state.generatedFunction = bitwise_cast<GeneratedFunction>( + state.finalizer->b3CodeLinkBuffer->entrypoint().executableAddress()); + state.jitCode->initializeB3Byproducts(state.proc->releaseByproducts()); } } } // namespace JSC::FTL |