diff options
Diffstat (limited to 'Source/JavaScriptCore/jit/ThunkGenerators.cpp')
| -rw-r--r-- | Source/JavaScriptCore/jit/ThunkGenerators.cpp | 930 |
1 files changed, 526 insertions, 404 deletions
diff --git a/Source/JavaScriptCore/jit/ThunkGenerators.cpp b/Source/JavaScriptCore/jit/ThunkGenerators.cpp index f8f5cbaf5..9a65506b3 100644 --- a/Source/JavaScriptCore/jit/ThunkGenerators.cpp +++ b/Source/JavaScriptCore/jit/ThunkGenerators.cpp @@ -1,5 +1,5 @@ /* - * Copyright (C) 2010, 2012, 2013 Apple Inc. All rights reserved. + * Copyright (C) 2010, 2012-2014, 2016 Apple Inc. All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions @@ -27,12 +27,18 @@ #include "ThunkGenerators.h" #include "CodeBlock.h" +#include "DFGSpeculativeJIT.h" +#include "JITExceptions.h" #include "JITOperations.h" #include "JSArray.h" -#include "JSArrayIterator.h" -#include "JSStack.h" -#include "Operations.h" +#include "JSBoundFunction.h" +#include "MathCommon.h" +#include "MaxFrameExtentForSlowPathCall.h" +#include "JSCInlines.h" +#include "JSWebAssemblyInstance.h" +#include "JSWebAssemblyRuntimeError.h" #include "SpecializedThunkJIT.h" +#include "WasmExceptionType.h" #include <wtf/InlineASM.h> #include <wtf/StringPrintStream.h> #include <wtf/text/StringImpl.h> @@ -43,17 +49,14 @@ namespace JSC { inline void emitPointerValidation(CCallHelpers& jit, GPRReg pointerGPR) { -#if !ASSERT_DISABLED + if (ASSERT_DISABLED) + return; CCallHelpers::Jump isNonZero = jit.branchTestPtr(CCallHelpers::NonZero, pointerGPR); - jit.breakpoint(); + jit.abortWithReason(TGInvalidPointer); isNonZero.link(&jit); jit.pushToSave(pointerGPR); jit.load8(pointerGPR, pointerGPR); jit.popToRestore(pointerGPR); -#else - UNUSED_PARAM(jit); - UNUSED_PARAM(pointerGPR); -#endif } // We will jump here if the JIT code tries to make a call, but the @@ -66,86 +69,100 @@ MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(VM* vm) // even though we won't use it. jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR); - // The CallFrame register points to the (failed) callee frame, so we need to pop back one frame. - jit.emitGetCallerFrameFromCallFrameHeaderPtr(GPRInfo::callFrameRegister); + jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(); - jit.setupArgumentsExecState(); + jit.setupArguments(CCallHelpers::TrustedImmPtr(vm), GPRInfo::callFrameRegister); jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(lookupExceptionHandler)), GPRInfo::nonArgGPR0); emitPointerValidation(jit, GPRInfo::nonArgGPR0); jit.call(GPRInfo::nonArgGPR0); jit.jumpToExceptionHandler(); - LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID); + LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID); return FINALIZE_CODE(patchBuffer, ("Throw exception from call slow path thunk")); } static void slowPathFor( - CCallHelpers& jit, VM* vm, P_JITOperation_E slowPathFunction) + CCallHelpers& jit, VM* vm, Sprt_JITOperation_ECli slowPathFunction) { - jit.preserveReturnAddressAfterCall(GPRInfo::nonArgGPR2); - emitPointerValidation(jit, GPRInfo::nonArgGPR2); - jit.emitPutReturnPCToCallFrameHeader(GPRInfo::nonArgGPR2); + jit.emitFunctionPrologue(); jit.storePtr(GPRInfo::callFrameRegister, &vm->topCallFrame); - jit.setupArgumentsExecState(); +#if OS(WINDOWS) && CPU(X86_64) + // Windows X86_64 needs some space pointed to by arg0 for return types larger than 64 bits. + // Other argument values are shift by 1. Use space on the stack for our two return values. + // Moving the stack down maxFrameExtentForSlowPathCall bytes gives us room for our 3 arguments + // and space for the 16 byte return area. + jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister); + jit.move(GPRInfo::regT2, GPRInfo::argumentGPR2); + jit.addPtr(CCallHelpers::TrustedImm32(32), CCallHelpers::stackPointerRegister, GPRInfo::argumentGPR0); + jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0); emitPointerValidation(jit, GPRInfo::nonArgGPR0); jit.call(GPRInfo::nonArgGPR0); - + jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR, 8), GPRInfo::returnValueGPR2); + jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR), GPRInfo::returnValueGPR); + jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister); +#else + if (maxFrameExtentForSlowPathCall) + jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister); + jit.setupArgumentsWithExecState(GPRInfo::regT2); + jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0); + emitPointerValidation(jit, GPRInfo::nonArgGPR0); + jit.call(GPRInfo::nonArgGPR0); + if (maxFrameExtentForSlowPathCall) + jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister); +#endif + // This slow call will return the address of one of the following: // 1) Exception throwing thunk. // 2) Host call return value returner thingy. // 3) The function to call. - jit.emitGetReturnPCFromCallFrameHeaderPtr(GPRInfo::nonPreservedNonReturnGPR); - jit.emitPutReturnPCToCallFrameHeader(CCallHelpers::TrustedImmPtr(0)); - emitPointerValidation(jit, GPRInfo::nonPreservedNonReturnGPR); - jit.restoreReturnAddressBeforeReturn(GPRInfo::nonPreservedNonReturnGPR); + // The second return value GPR will hold a non-zero value for tail calls. + emitPointerValidation(jit, GPRInfo::returnValueGPR); + jit.emitFunctionEpilogue(); + + RELEASE_ASSERT(reinterpret_cast<void*>(KeepTheFrame) == reinterpret_cast<void*>(0)); + CCallHelpers::Jump doNotTrash = jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::returnValueGPR2); + + jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR); + jit.prepareForTailCallSlow(GPRInfo::returnValueGPR); + + doNotTrash.link(&jit); jit.jump(GPRInfo::returnValueGPR); } -static MacroAssemblerCodeRef linkForThunkGenerator( - VM* vm, CodeSpecializationKind kind) +MacroAssemblerCodeRef linkCallThunkGenerator(VM* vm) { // The return address is on the stack or in the link register. We will hence // save the return address to the call frame while we make a C++ function call // to perform linking and lazy compilation if necessary. We expect the callee // to be in regT0/regT1 (payload/tag), the CallFrame to have already // been adjusted, and all other registers to be available for use. - CCallHelpers jit(vm); - slowPathFor(jit, vm, kind == CodeForCall ? operationLinkCall : operationLinkConstruct); + slowPathFor(jit, vm, operationLinkCall); - LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID); - return FINALIZE_CODE( - patchBuffer, - ("Link %s slow path thunk", kind == CodeForCall ? "call" : "construct")); -} - -MacroAssemblerCodeRef linkCallThunkGenerator(VM* vm) -{ - return linkForThunkGenerator(vm, CodeForCall); -} - -MacroAssemblerCodeRef linkConstructThunkGenerator(VM* vm) -{ - return linkForThunkGenerator(vm, CodeForConstruct); + LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID); + return FINALIZE_CODE(patchBuffer, ("Link call slow path thunk")); } // For closure optimizations, we only include calls, since if you're using closures for // object construction then you're going to lose big time anyway. -MacroAssemblerCodeRef linkClosureCallThunkGenerator(VM* vm) +MacroAssemblerCodeRef linkPolymorphicCallThunkGenerator(VM* vm) { CCallHelpers jit(vm); - slowPathFor(jit, vm, operationLinkClosureCall); + slowPathFor(jit, vm, operationLinkPolymorphicCall); - LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID); - return FINALIZE_CODE(patchBuffer, ("Link closure call slow path thunk")); + LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID); + return FINALIZE_CODE(patchBuffer, ("Link polymorphic call slow path thunk")); } -static MacroAssemblerCodeRef virtualForThunkGenerator( - VM* vm, CodeSpecializationKind kind) +// FIXME: We should distinguish between a megamorphic virtual call vs. a slow +// path virtual call so that we can enable fast tail calls for megamorphic +// virtual calls by using the shuffler. +// https://bugs.webkit.org/show_bug.cgi?id=148831 +MacroAssemblerCodeRef virtualThunkFor(VM* vm, CallLinkInfo& callLinkInfo) { // The callee is in regT0 (for JSVALUE32_64, the tag is in regT1). // The return address is on the stack, or in the link register. We will hence @@ -155,6 +172,12 @@ static MacroAssemblerCodeRef virtualForThunkGenerator( CCallHelpers jit(vm); CCallHelpers::JumpList slowCase; + + // This is a slow path execution, and regT2 contains the CallLinkInfo. Count the + // slow path execution for the profiler. + jit.add32( + CCallHelpers::TrustedImm32(1), + CCallHelpers::Address(GPRInfo::regT2, CallLinkInfo::offsetOfSlowPathCount())); // FIXME: we should have a story for eliminating these checks. In many cases, // the DFG knows that the value is definitely a cell, or definitely a function. @@ -169,189 +192,131 @@ static MacroAssemblerCodeRef virtualForThunkGenerator( CCallHelpers::NotEqual, GPRInfo::regT1, CCallHelpers::TrustedImm32(JSValue::CellTag))); #endif - jit.loadPtr(CCallHelpers::Address(GPRInfo::regT0, JSCell::structureOffset()), GPRInfo::nonArgGPR2); - slowCase.append( - jit.branchPtr( - CCallHelpers::NotEqual, - CCallHelpers::Address(GPRInfo::nonArgGPR2, Structure::classInfoOffset()), - CCallHelpers::TrustedImmPtr(JSFunction::info()))); + slowCase.append(jit.branchIfNotType(GPRInfo::regT0, JSFunctionType)); // Now we know we have a JSFunction. jit.loadPtr( CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfExecutable()), - GPRInfo::nonArgGPR2); - slowCase.append( - jit.branch32( - CCallHelpers::LessThan, - CCallHelpers::Address( - GPRInfo::nonArgGPR2, ExecutableBase::offsetOfNumParametersFor(kind)), - CCallHelpers::TrustedImm32(0))); - - // Now we know that we have a CodeBlock, and we're committed to making a fast - // call. - + GPRInfo::regT4); jit.loadPtr( - CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfScopeChain()), - GPRInfo::regT1); -#if USE(JSVALUE64) - jit.store64( - GPRInfo::regT1, - CCallHelpers::Address( - GPRInfo::callFrameRegister, - static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain)); -#else - jit.storePtr( - GPRInfo::regT1, - CCallHelpers::Address( - GPRInfo::callFrameRegister, - static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain + - OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload))); - jit.store32( - CCallHelpers::TrustedImm32(JSValue::CellTag), CCallHelpers::Address( - GPRInfo::callFrameRegister, - static_cast<ptrdiff_t>(sizeof(Register)) * JSStack::ScopeChain + - OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag))); -#endif + GPRInfo::regT4, ExecutableBase::offsetOfJITCodeWithArityCheckFor( + callLinkInfo.specializationKind())), + GPRInfo::regT4); + slowCase.append(jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT4)); - jit.loadPtr( - CCallHelpers::Address(GPRInfo::nonArgGPR2, ExecutableBase::offsetOfJITCodeWithArityCheckFor(kind)), - GPRInfo::regT0); + // Now we know that we have a CodeBlock, and we're committed to making a fast + // call. // Make a tail call. This will return back to JIT code. - emitPointerValidation(jit, GPRInfo::regT0); - jit.jump(GPRInfo::regT0); + emitPointerValidation(jit, GPRInfo::regT4); + if (callLinkInfo.isTailCall()) { + jit.preserveReturnAddressAfterCall(GPRInfo::regT0); + jit.prepareForTailCallSlow(GPRInfo::regT4); + } + jit.jump(GPRInfo::regT4); slowCase.link(&jit); // Here we don't know anything, so revert to the full slow path. - slowPathFor(jit, vm, kind == CodeForCall ? operationVirtualCall : operationVirtualConstruct); + slowPathFor(jit, vm, operationVirtualCall); - LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID); + LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID); return FINALIZE_CODE( patchBuffer, - ("Virtual %s slow path thunk", kind == CodeForCall ? "call" : "construct")); + ("Virtual %s slow path thunk", + callLinkInfo.callMode() == CallMode::Regular ? "call" : callLinkInfo.callMode() == CallMode::Tail ? "tail call" : "construct")); } -MacroAssemblerCodeRef virtualCallThunkGenerator(VM* vm) -{ - return virtualForThunkGenerator(vm, CodeForCall); -} - -MacroAssemblerCodeRef virtualConstructThunkGenerator(VM* vm) -{ - return virtualForThunkGenerator(vm, CodeForConstruct); -} +enum ThunkEntryType { EnterViaCall, EnterViaJumpWithSavedTags, EnterViaJumpWithoutSavedTags }; -static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind) +static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall) { + // FIXME: This should be able to log ShadowChicken prologue packets. + // https://bugs.webkit.org/show_bug.cgi?id=155689 + int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind); JSInterfaceJIT jit(vm); - - jit.emitPutImmediateToCallFrameHeader(0, JSStack::CodeBlock); - jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame); -#if CPU(X86) - // Load caller frame's scope chain into this callframe so that whatever we call can - // get to its global data. - jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0); - jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0); - jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); + switch (entryType) { + case EnterViaCall: + jit.emitFunctionPrologue(); + break; + case EnterViaJumpWithSavedTags: +#if USE(JSVALUE64) + // We're coming from a specialized thunk that has saved the prior tag registers' contents. + // Restore them now. +#if CPU(ARM64) + jit.popPair(JSInterfaceJIT::tagTypeNumberRegister, JSInterfaceJIT::tagMaskRegister); +#else + jit.pop(JSInterfaceJIT::tagMaskRegister); + jit.pop(JSInterfaceJIT::tagTypeNumberRegister); +#endif +#endif + break; + case EnterViaJumpWithoutSavedTags: + jit.move(JSInterfaceJIT::framePointerRegister, JSInterfaceJIT::stackPointerRegister); + break; + } - jit.peek(JSInterfaceJIT::regT1); - jit.emitPutReturnPCToCallFrameHeader(JSInterfaceJIT::regT1); + jit.emitPutToCallFrameHeader(0, CallFrameSlot::codeBlock); + jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame); +#if CPU(X86) // Calling convention: f(ecx, edx, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx); - jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister); // Align stack after call. + jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue. // call the function - jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1); + jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, JSInterfaceJIT::regT1); jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1); - jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction)); - jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister); + jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); #elif CPU(X86_64) - // Load caller frame's scope chain into this callframe so that whatever we call can - // get to its global data. - jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0); - jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0); - jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); - - jit.peek(JSInterfaceJIT::regT1); - jit.emitPutReturnPCToCallFrameHeader(JSInterfaceJIT::regT1); - #if !OS(WINDOWS) // Calling convention: f(edi, esi, edx, ecx, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi); - jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); // Align stack after call. - - jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi); + jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, X86Registers::esi); jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9); - jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction)); - jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); #else // Calling convention: f(ecx, edx, r8, r9, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx); - // Leave space for the callee parameter home addresses and align the stack. - jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); + // Leave space for the callee parameter home addresses. + // At this point the stack is aligned to 16 bytes, but if this changes at some point, we need to emit code to align it. + jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); - jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx); + jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, X86Registers::edx); jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9); - jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction)); - jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); + jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); #endif #elif CPU(ARM64) - COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT1, prev_callframe_not_trampled_by_T1); - COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT3, prev_callframe_not_trampled_by_T3); COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0); COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1); COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2); - // Load caller frame's scope chain into this callframe so that whatever we call can - // get to its global data. - jit.emitGetCallerFrameFromCallFrameHeaderPtr(ARM64Registers::x3); - jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, ARM64Registers::x3); - jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); - - jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved - jit.emitPutReturnPCToCallFrameHeader(ARM64Registers::lr); - // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0); - jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1); + jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, ARM64Registers::x1); jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2); - jit.move(ARM64Registers::x3, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction)); - - jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3); - -#elif CPU(ARM) || CPU(SH4) || CPU(MIPS) - // Load caller frame's scope chain into this callframe so that whatever we call can get to its global data. - jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT2); - jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2); - jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); - - jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved - jit.emitPutReturnPCToCallFrameHeader(JSInterfaceJIT::regT3); - +#elif CPU(ARM) || CPU(MIPS) #if CPU(MIPS) // Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments. jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); @@ -361,8 +326,7 @@ static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind k // Host function signature is f(ExecState*). jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0); - jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1); - jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. + jit.emitGetFromCallFrameHeaderPtr(CallFrameSlot::callee, JSInterfaceJIT::argumentGPR1); jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2); jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction)); @@ -370,12 +334,10 @@ static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind k // Restore stack space jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); #endif - - jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3); #else #error "JIT not supported on this platform." UNUSED_PARAM(executableOffsetToFunction); - breakpoint(); + abortWithReason(TGNotSupported); #endif // Check for an exception @@ -385,40 +347,43 @@ static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind k #else JSInterfaceJIT::Jump exceptionHandler = jit.branch32( JSInterfaceJIT::NotEqual, - JSInterfaceJIT::AbsoluteAddress(reinterpret_cast<char*>(vm->addressOfException()) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), - JSInterfaceJIT::TrustedImm32(JSValue::EmptyValueTag)); + JSInterfaceJIT::AbsoluteAddress(vm->addressOfException()), + JSInterfaceJIT::TrustedImm32(0)); #endif + jit.emitFunctionEpilogue(); // Return. jit.ret(); // Handle an exception exceptionHandler.link(&jit); - // Grab the return address. - jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT1); - - jit.move(JSInterfaceJIT::TrustedImmPtr(&vm->exceptionLocation), JSInterfaceJIT::regT2); - jit.storePtr(JSInterfaceJIT::regT1, JSInterfaceJIT::regT2); - + jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(); jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame); #if CPU(X86) && USE(JSVALUE32_64) jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister); - jit.push(JSInterfaceJIT::callFrameRegister); + jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT0); + jit.push(JSInterfaceJIT::regT0); #else +#if OS(WINDOWS) + // Allocate space on stack for the 4 parameter registers. + jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); +#endif jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0); #endif jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3); jit.call(JSInterfaceJIT::regT3); #if CPU(X86) && USE(JSVALUE32_64) jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); +#elif OS(WINDOWS) + jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); #endif jit.jumpToExceptionHandler(); - LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID); - return FINALIZE_CODE(patchBuffer, ("native %s trampoline", toCString(kind).data())); + LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID); + return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJumpWithSavedTags ? "Tail With Saved Tags " : entryType == EnterViaJumpWithoutSavedTags ? "Tail Without Saved Tags " : "", toCString(kind).data())); } MacroAssemblerCodeRef nativeCallGenerator(VM* vm) @@ -426,43 +391,80 @@ MacroAssemblerCodeRef nativeCallGenerator(VM* vm) return nativeForGenerator(vm, CodeForCall); } +MacroAssemblerCodeRef nativeTailCallGenerator(VM* vm) +{ + return nativeForGenerator(vm, CodeForCall, EnterViaJumpWithSavedTags); +} + +MacroAssemblerCodeRef nativeTailCallWithoutSavedTagsGenerator(VM* vm) +{ + return nativeForGenerator(vm, CodeForCall, EnterViaJumpWithoutSavedTags); +} + MacroAssemblerCodeRef nativeConstructGenerator(VM* vm) { return nativeForGenerator(vm, CodeForConstruct); } -MacroAssemblerCodeRef arityFixup(VM* vm) +MacroAssemblerCodeRef arityFixupGenerator(VM* vm) { JSInterfaceJIT jit(vm); - // We enter with fixup count in regT0 + // We enter with fixup count in argumentGPR0 + // We have the guarantee that a0, a1, a2, t3, t4 and t5 (or t0 for Windows) are all distinct :-) #if USE(JSVALUE64) +#if OS(WINDOWS) + const GPRReg extraTemp = JSInterfaceJIT::regT0; +#else + const GPRReg extraTemp = JSInterfaceJIT::regT5; +#endif # if CPU(X86_64) jit.pop(JSInterfaceJIT::regT4); # endif - jit.neg64(JSInterfaceJIT::regT0); jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3); - jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2); - jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2); - - // Move current frame down regT0 number of slots + jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrameSlot::argumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2); + jit.add32(JSInterfaceJIT::TrustedImm32(CallFrame::headerSizeInRegisters), JSInterfaceJIT::argumentGPR2); + + // Check to see if we have extra slots we can use + jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1); + jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1); + JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1); + jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp); + JSInterfaceJIT::Label fillExtraSlots(jit.label()); + jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight)); + jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2); + jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit); + jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0); + JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0); + noExtraSlot.link(&jit); + + jit.neg64(JSInterfaceJIT::argumentGPR0); + + // Adjust call frame register and stack pointer to account for missing args. + // We need to change the stack pointer first before performing copy/fill loops. + // This stack space below the stack pointer is considered unsed by OS. Therefore, + // OS may corrupt this space when constructing a signal stack. + jit.move(JSInterfaceJIT::argumentGPR0, extraTemp); + jit.lshift64(JSInterfaceJIT::TrustedImm32(3), extraTemp); + jit.addPtr(extraTemp, JSInterfaceJIT::callFrameRegister); + jit.addPtr(extraTemp, JSInterfaceJIT::stackPointerRegister); + + // Move current frame down argumentGPR0 number of slots JSInterfaceJIT::Label copyLoop(jit.label()); - jit.load64(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1); - jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight)); + jit.load64(JSInterfaceJIT::regT3, extraTemp); + jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3); - jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit); + jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit); - // Fill in regT0 missing arg slots with undefined - jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2); - jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT1); + // Fill in argumentGPR0 missing arg slots with undefined + jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2); + jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp); JSInterfaceJIT::Label fillUndefinedLoop(jit.label()); - jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight)); + jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3); - jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit); - - // Adjust call frame register to account for missing args - jit.lshift64(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT0); - jit.addPtr(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); + jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit); + + done.link(&jit); # if CPU(X86_64) jit.push(JSInterfaceJIT::regT4); @@ -472,34 +474,54 @@ MacroAssemblerCodeRef arityFixup(VM* vm) # if CPU(X86) jit.pop(JSInterfaceJIT::regT4); # endif - jit.neg32(JSInterfaceJIT::regT0); jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3); - jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2); - jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2); - - // Move current frame down regT0 number of slots + jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrameSlot::argumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2); + jit.add32(JSInterfaceJIT::TrustedImm32(CallFrame::headerSizeInRegisters), JSInterfaceJIT::argumentGPR2); + + // Check to see if we have extra slots we can use + jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1); + jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1); + JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1); + JSInterfaceJIT::Label fillExtraSlots(jit.label()); + jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5); + jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, PayloadOffset)); + jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5); + jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, TagOffset)); + jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2); + jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit); + jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0); + JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0); + noExtraSlot.link(&jit); + + jit.neg32(JSInterfaceJIT::argumentGPR0); + + // Move current frame down argumentGPR0 number of slots JSInterfaceJIT::Label copyLoop(jit.label()); - jit.load32(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1); - jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight)); - jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, 4), JSInterfaceJIT::regT1); - jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4)); + jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, PayloadOffset), JSInterfaceJIT::regT5); + jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset)); + jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, TagOffset), JSInterfaceJIT::regT5); + jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3); - jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit); + jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit); - // Fill in regT0 missing arg slots with undefined - jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2); + // Fill in argumentGPR0 missing arg slots with undefined + jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2); JSInterfaceJIT::Label fillUndefinedLoop(jit.label()); - jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT1); - jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight)); - jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1); - jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4)); + jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5); + jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset)); + jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5); + jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3); - jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit); + jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit); + + // Adjust call frame register and stack pointer to account for missing args + jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::regT5); + jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT5); + jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::callFrameRegister); + jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::stackPointerRegister); - // Adjust call frame register to account for missing args - jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT0); - jit.addPtr(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); + done.link(&jit); # if CPU(X86) jit.push(JSInterfaceJIT::regT4); @@ -507,10 +529,20 @@ MacroAssemblerCodeRef arityFixup(VM* vm) jit.ret(); #endif - LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID); + LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID); return FINALIZE_CODE(patchBuffer, ("fixup arity")); } +MacroAssemblerCodeRef unreachableGenerator(VM* vm) +{ + JSInterfaceJIT jit(vm); + + jit.breakpoint(); + + LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID); + return FINALIZE_CODE(patchBuffer, ("unreachable thunk")); +} + static void stringCharLoad(SpecializedThunkJIT& jit, VM* vm) { // load string @@ -554,7 +586,7 @@ MacroAssemblerCodeRef charCodeAtThunkGenerator(VM* vm) SpecializedThunkJIT jit(vm, 1); stringCharLoad(jit, vm); jit.returnInt32(SpecializedThunkJIT::regT0); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "charCodeAt"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "charCodeAt"); } MacroAssemblerCodeRef charAtThunkGenerator(VM* vm) @@ -563,7 +595,7 @@ MacroAssemblerCodeRef charAtThunkGenerator(VM* vm) stringCharLoad(jit, vm); charToString(jit, vm, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1); jit.returnJSCell(SpecializedThunkJIT::regT0); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "charAt"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "charAt"); } MacroAssemblerCodeRef fromCharCodeThunkGenerator(VM* vm) @@ -573,7 +605,28 @@ MacroAssemblerCodeRef fromCharCodeThunkGenerator(VM* vm) jit.loadInt32Argument(0, SpecializedThunkJIT::regT0); charToString(jit, vm, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1); jit.returnJSCell(SpecializedThunkJIT::regT0); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "fromCharCode"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "fromCharCode"); +} + +MacroAssemblerCodeRef clz32ThunkGenerator(VM* vm) +{ + SpecializedThunkJIT jit(vm, 1); + MacroAssembler::Jump nonIntArgJump; + jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntArgJump); + + SpecializedThunkJIT::Label convertedArgumentReentry(&jit); + jit.countLeadingZeros32(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1); + jit.returnInt32(SpecializedThunkJIT::regT1); + + if (jit.supportsFloatingPointTruncate()) { + nonIntArgJump.link(&jit); + jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); + jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(convertedArgumentReentry, &jit); + jit.appendFailure(jit.jump()); + } else + jit.appendFailure(nonIntArgJump); + + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "clz32"); } MacroAssemblerCodeRef sqrtThunkGenerator(VM* vm) @@ -585,25 +638,15 @@ MacroAssemblerCodeRef sqrtThunkGenerator(VM* vm) jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); jit.returnDouble(SpecializedThunkJIT::fpRegT0); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "sqrt"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "sqrt"); } #define UnaryDoubleOpWrapper(function) function##Wrapper enum MathThunkCallingConvention { }; typedef MathThunkCallingConvention(*MathThunk)(MathThunkCallingConvention); -extern "C" { - -double jsRound(double) REFERENCED_FROM_ASM; -double jsRound(double d) -{ - double integer = ceil(d); - return integer - (integer - d > 0.5); -} - -} -#if CPU(X86_64) && COMPILER(GCC) && (OS(DARWIN) || OS(LINUX)) +#if CPU(X86_64) && COMPILER(GCC_OR_CLANG) && (OS(DARWIN) || OS(LINUX)) #define defineUnaryDoubleOpWrapper(function) \ asm( \ @@ -611,7 +654,9 @@ double jsRound(double d) ".globl " SYMBOL_STRING(function##Thunk) "\n" \ HIDE_SYMBOL(function##Thunk) "\n" \ SYMBOL_STRING(function##Thunk) ":" "\n" \ + "pushq %rax\n" \ "call " GLOBAL_REFERENCE(function) "\n" \ + "popq %rcx\n" \ "ret\n" \ );\ extern "C" { \ @@ -619,7 +664,7 @@ double jsRound(double d) } \ static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk; -#elif CPU(X86) && COMPILER(GCC) && OS(LINUX) && defined(__PIC__) +#elif CPU(X86) && COMPILER(GCC_OR_CLANG) && OS(LINUX) && defined(__PIC__) #define defineUnaryDoubleOpWrapper(function) \ asm( \ ".text\n" \ @@ -643,19 +688,19 @@ double jsRound(double d) } \ static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk; -#elif CPU(X86) && COMPILER(GCC) && (OS(DARWIN) || OS(LINUX)) +#elif CPU(X86) && COMPILER(GCC_OR_CLANG) && (OS(DARWIN) || OS(LINUX)) #define defineUnaryDoubleOpWrapper(function) \ asm( \ ".text\n" \ ".globl " SYMBOL_STRING(function##Thunk) "\n" \ HIDE_SYMBOL(function##Thunk) "\n" \ SYMBOL_STRING(function##Thunk) ":" "\n" \ - "subl $8, %esp\n" \ + "subl $20, %esp\n" \ "movsd %xmm0, (%esp) \n" \ "call " GLOBAL_REFERENCE(function) "\n" \ "fstpl (%esp) \n" \ "movsd (%esp), %xmm0 \n" \ - "addl $8, %esp\n" \ + "addl $20, %esp\n" \ "ret\n" \ );\ extern "C" { \ @@ -663,7 +708,7 @@ double jsRound(double d) } \ static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk; -#elif CPU(ARM_THUMB2) && COMPILER(GCC) && PLATFORM(IOS) +#elif CPU(ARM_THUMB2) && COMPILER(GCC_OR_CLANG) && PLATFORM(IOS) #define defineUnaryDoubleOpWrapper(function) \ asm( \ @@ -696,12 +741,39 @@ double jsRound(double d) HIDE_SYMBOL(function##Thunk) "\n" \ SYMBOL_STRING(function##Thunk) ":" "\n" \ "b " GLOBAL_REFERENCE(function) "\n" \ + ".previous" \ ); \ extern "C" { \ MathThunkCallingConvention function##Thunk(MathThunkCallingConvention); \ } \ static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk; +#elif CPU(X86) && COMPILER(MSVC) && OS(WINDOWS) + +// MSVC does not accept floor, etc, to be called directly from inline assembly, so we need to wrap these functions. +static double (_cdecl *floorFunction)(double) = floor; +static double (_cdecl *ceilFunction)(double) = ceil; +static double (_cdecl *truncFunction)(double) = trunc; +static double (_cdecl *expFunction)(double) = exp; +static double (_cdecl *logFunction)(double) = log; +static double (_cdecl *jsRoundFunction)(double) = jsRound; + +#define defineUnaryDoubleOpWrapper(function) \ + extern "C" __declspec(naked) MathThunkCallingConvention function##Thunk(MathThunkCallingConvention) \ + { \ + __asm \ + { \ + __asm sub esp, 20 \ + __asm movsd mmword ptr [esp], xmm0 \ + __asm call function##Function \ + __asm fstp qword ptr [esp] \ + __asm movsd xmm0, mmword ptr [esp] \ + __asm add esp, 20 \ + __asm ret \ + } \ + } \ + static MathThunk UnaryDoubleOpWrapper(function) = &function##Thunk; + #else #define defineUnaryDoubleOpWrapper(function) \ @@ -713,10 +785,8 @@ defineUnaryDoubleOpWrapper(exp); defineUnaryDoubleOpWrapper(log); defineUnaryDoubleOpWrapper(floor); defineUnaryDoubleOpWrapper(ceil); +defineUnaryDoubleOpWrapper(trunc); -static const double oneConstant = 1.0; -static const double negativeHalfConstant = -0.5; -static const double zeroConstant = 0.0; static const double halfConstant = 0.5; MacroAssemblerCodeRef floorThunkGenerator(VM* vm) @@ -729,18 +799,21 @@ MacroAssemblerCodeRef floorThunkGenerator(VM* vm) jit.returnInt32(SpecializedThunkJIT::regT0); nonIntJump.link(&jit); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); -#if CPU(ARM64) - SpecializedThunkJIT::JumpList doubleResult; - jit.floorDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); - jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1); - jit.returnInt32(SpecializedThunkJIT::regT0); - doubleResult.link(&jit); - jit.returnDouble(SpecializedThunkJIT::fpRegT0); -#else + + if (jit.supportsFloatingPointRounding()) { + SpecializedThunkJIT::JumpList doubleResult; + jit.floorDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); + jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1); + jit.returnInt32(SpecializedThunkJIT::regT0); + doubleResult.link(&jit); + jit.returnDouble(SpecializedThunkJIT::fpRegT0); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "floor"); + } + SpecializedThunkJIT::Jump intResult; SpecializedThunkJIT::JumpList doubleResult; if (jit.supportsFloatingPointTruncate()) { - jit.loadDouble(&zeroConstant, SpecializedThunkJIT::fpRegT1); + jit.moveZeroToDouble(SpecializedThunkJIT::fpRegT1); doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); SpecializedThunkJIT::JumpList slowPath; // Handle the negative doubles in the slow path for now. @@ -756,8 +829,7 @@ MacroAssemblerCodeRef floorThunkGenerator(VM* vm) jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT0); -#endif // CPU(ARM64) - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "floor"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "floor"); } MacroAssemblerCodeRef ceilThunkGenerator(VM* vm) @@ -770,17 +842,40 @@ MacroAssemblerCodeRef ceilThunkGenerator(VM* vm) jit.returnInt32(SpecializedThunkJIT::regT0); nonIntJump.link(&jit); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); -#if CPU(ARM64) - jit.ceilDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); -#else - jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(ceil)); -#endif // CPU(ARM64) + if (jit.supportsFloatingPointRounding()) + jit.ceilDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); + else + jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(ceil)); + SpecializedThunkJIT::JumpList doubleResult; jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT0); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "ceil"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "ceil"); +} + +MacroAssemblerCodeRef truncThunkGenerator(VM* vm) +{ + SpecializedThunkJIT jit(vm, 1); + if (!UnaryDoubleOpWrapper(trunc) || !jit.supportsFloatingPoint()) + return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm)); + MacroAssembler::Jump nonIntJump; + jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump); + jit.returnInt32(SpecializedThunkJIT::regT0); + nonIntJump.link(&jit); + jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); + if (jit.supportsFloatingPointRounding()) + jit.roundTowardZeroDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); + else + jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(trunc)); + + SpecializedThunkJIT::JumpList doubleResult; + jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1); + jit.returnInt32(SpecializedThunkJIT::regT0); + doubleResult.link(&jit); + jit.returnDouble(SpecializedThunkJIT::fpRegT0); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "trunc"); } MacroAssemblerCodeRef roundThunkGenerator(VM* vm) @@ -796,12 +891,12 @@ MacroAssemblerCodeRef roundThunkGenerator(VM* vm) SpecializedThunkJIT::Jump intResult; SpecializedThunkJIT::JumpList doubleResult; if (jit.supportsFloatingPointTruncate()) { - jit.loadDouble(&zeroConstant, SpecializedThunkJIT::fpRegT1); + jit.moveZeroToDouble(SpecializedThunkJIT::fpRegT1); doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); SpecializedThunkJIT::JumpList slowPath; // Handle the negative doubles in the slow path for now. slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); - jit.loadDouble(&halfConstant, SpecializedThunkJIT::fpRegT1); + jit.loadDouble(MacroAssembler::TrustedImmPtr(&halfConstant), SpecializedThunkJIT::fpRegT1); jit.addDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0)); intResult = jit.jump(); @@ -814,7 +909,7 @@ MacroAssemblerCodeRef roundThunkGenerator(VM* vm) jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT0); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "round"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "round"); } MacroAssemblerCodeRef expThunkGenerator(VM* vm) @@ -827,7 +922,7 @@ MacroAssemblerCodeRef expThunkGenerator(VM* vm) jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(exp)); jit.returnDouble(SpecializedThunkJIT::fpRegT0); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "exp"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "exp"); } MacroAssemblerCodeRef logThunkGenerator(VM* vm) @@ -840,7 +935,7 @@ MacroAssemblerCodeRef logThunkGenerator(VM* vm) jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(log)); jit.returnDouble(SpecializedThunkJIT::fpRegT0); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "log"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "log"); } MacroAssemblerCodeRef absThunkGenerator(VM* vm) @@ -848,71 +943,56 @@ MacroAssemblerCodeRef absThunkGenerator(VM* vm) SpecializedThunkJIT jit(vm, 1); if (!jit.supportsFloatingPointAbs()) return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm)); + +#if USE(JSVALUE64) + unsigned virtualRegisterIndex = CallFrame::argumentOffset(0); + jit.load64(AssemblyHelpers::addressFor(virtualRegisterIndex), GPRInfo::regT0); + MacroAssembler::Jump notInteger = jit.branch64(MacroAssembler::Below, GPRInfo::regT0, GPRInfo::tagTypeNumberRegister); + + // Abs Int32. + jit.rshift32(GPRInfo::regT0, MacroAssembler::TrustedImm32(31), GPRInfo::regT1); + jit.add32(GPRInfo::regT1, GPRInfo::regT0); + jit.xor32(GPRInfo::regT1, GPRInfo::regT0); + + // IntMin cannot be inverted. + MacroAssembler::Jump integerIsIntMin = jit.branchTest32(MacroAssembler::Signed, GPRInfo::regT0); + + // Box and finish. + jit.or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0); + MacroAssembler::Jump doneWithIntegers = jit.jump(); + + // Handle Doubles. + notInteger.link(&jit); + jit.appendFailure(jit.branchTest64(MacroAssembler::Zero, GPRInfo::regT0, GPRInfo::tagTypeNumberRegister)); + jit.unboxDoubleWithoutAssertions(GPRInfo::regT0, GPRInfo::regT0, FPRInfo::fpRegT0); + MacroAssembler::Label absFPR0Label = jit.label(); + jit.absDouble(FPRInfo::fpRegT0, FPRInfo::fpRegT1); + jit.boxDouble(FPRInfo::fpRegT1, GPRInfo::regT0); + + // Tail. + doneWithIntegers.link(&jit); + jit.returnJSValue(GPRInfo::regT0); + + // We know the value of regT0 is IntMin. We could load that value from memory but + // it is simpler to just convert it. + integerIsIntMin.link(&jit); + jit.convertInt32ToDouble(GPRInfo::regT0, FPRInfo::fpRegT0); + jit.jump().linkTo(absFPR0Label, &jit); +#else MacroAssembler::Jump nonIntJump; jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump); jit.rshift32(SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(31), SpecializedThunkJIT::regT1); jit.add32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0); jit.xor32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0); - jit.appendFailure(jit.branch32(MacroAssembler::Equal, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1 << 31))); + jit.appendFailure(jit.branchTest32(MacroAssembler::Signed, SpecializedThunkJIT::regT0)); jit.returnInt32(SpecializedThunkJIT::regT0); nonIntJump.link(&jit); // Shame about the double int conversion here. jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); jit.absDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); jit.returnDouble(SpecializedThunkJIT::fpRegT1); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "abs"); -} - -MacroAssemblerCodeRef powThunkGenerator(VM* vm) -{ - SpecializedThunkJIT jit(vm, 2); - if (!jit.supportsFloatingPoint()) - return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm)); - - jit.loadDouble(&oneConstant, SpecializedThunkJIT::fpRegT1); - jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); - MacroAssembler::Jump nonIntExponent; - jit.loadInt32Argument(1, SpecializedThunkJIT::regT0, nonIntExponent); - jit.appendFailure(jit.branch32(MacroAssembler::LessThan, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(0))); - - MacroAssembler::Jump exponentIsZero = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0); - MacroAssembler::Label startLoop(jit.label()); - - MacroAssembler::Jump exponentIsEven = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1)); - jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); - exponentIsEven.link(&jit); - jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); - jit.rshift32(MacroAssembler::TrustedImm32(1), SpecializedThunkJIT::regT0); - jit.branchTest32(MacroAssembler::NonZero, SpecializedThunkJIT::regT0).linkTo(startLoop, &jit); - - exponentIsZero.link(&jit); - - { - SpecializedThunkJIT::JumpList doubleResult; - jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0); - jit.returnInt32(SpecializedThunkJIT::regT0); - doubleResult.link(&jit); - jit.returnDouble(SpecializedThunkJIT::fpRegT1); - } - - if (jit.supportsFloatingPointSqrt()) { - nonIntExponent.link(&jit); - jit.loadDouble(&negativeHalfConstant, SpecializedThunkJIT::fpRegT3); - jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::regT0); - jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleLessThanOrEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); - jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::fpRegT3)); - jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); - jit.divDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); - - SpecializedThunkJIT::JumpList doubleResult; - jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0); - jit.returnInt32(SpecializedThunkJIT::regT0); - doubleResult.link(&jit); - jit.returnDouble(SpecializedThunkJIT::fpRegT1); - } else - jit.appendFailure(nonIntExponent); - - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "pow"); +#endif + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "abs"); } MacroAssemblerCodeRef imulThunkGenerator(VM* vm) @@ -931,8 +1011,7 @@ MacroAssemblerCodeRef imulThunkGenerator(VM* vm) nonIntArg0Jump.link(&jit); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(doneLoadingArg0, &jit); - jit.xor32(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT0); - jit.jump(doneLoadingArg0); + jit.appendFailure(jit.jump()); } else jit.appendFailure(nonIntArg0Jump); @@ -940,117 +1019,160 @@ MacroAssemblerCodeRef imulThunkGenerator(VM* vm) nonIntArg1Jump.link(&jit); jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT1); jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT1, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(doneLoadingArg1, &jit); - jit.xor32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT1); - jit.jump(doneLoadingArg1); + jit.appendFailure(jit.jump()); } else jit.appendFailure(nonIntArg1Jump); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "imul"); + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "imul"); +} + +MacroAssemblerCodeRef randomThunkGenerator(VM* vm) +{ + SpecializedThunkJIT jit(vm, 0); + if (!jit.supportsFloatingPoint()) + return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm)); + +#if USE(JSVALUE64) + jit.emitRandomThunk(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT3, SpecializedThunkJIT::fpRegT0); + jit.returnDouble(SpecializedThunkJIT::fpRegT0); + + return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "random"); +#else + return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm)); +#endif } -static MacroAssemblerCodeRef arrayIteratorNextThunkGenerator(VM* vm, ArrayIterationKind kind) +MacroAssemblerCodeRef boundThisNoArgsFunctionCallGenerator(VM* vm) { - typedef SpecializedThunkJIT::TrustedImm32 TrustedImm32; - typedef SpecializedThunkJIT::TrustedImmPtr TrustedImmPtr; - typedef SpecializedThunkJIT::Address Address; - typedef SpecializedThunkJIT::BaseIndex BaseIndex; - typedef SpecializedThunkJIT::Jump Jump; + CCallHelpers jit(vm); + + jit.emitFunctionPrologue(); - SpecializedThunkJIT jit(vm); - // Make sure we're being called on an array iterator, and load m_iteratedObject, and m_nextIndex into regT0 and regT1 respectively - jit.loadArgumentWithSpecificClass(JSArrayIterator::info(), SpecializedThunkJIT::ThisArgument, SpecializedThunkJIT::regT4, SpecializedThunkJIT::regT1); + // Set up our call frame. + jit.storePtr(CCallHelpers::TrustedImmPtr(nullptr), CCallHelpers::addressFor(CallFrameSlot::codeBlock)); + jit.store32(CCallHelpers::TrustedImm32(0), CCallHelpers::tagFor(CallFrameSlot::argumentCount)); - // Early exit if we don't have a thunk for this form of iteration - jit.appendFailure(jit.branch32(SpecializedThunkJIT::AboveOrEqual, Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfIterationKind()), TrustedImm32(ArrayIterateKeyValue))); + unsigned extraStackNeeded = 0; + if (unsigned stackMisalignment = sizeof(CallerFrameAndPC) % stackAlignmentBytes()) + extraStackNeeded = stackAlignmentBytes() - stackMisalignment; - jit.loadPtr(Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfIteratedObject()), SpecializedThunkJIT::regT0); + // We need to forward all of the arguments that we were passed. We aren't allowed to do a tail + // call here as far as I can tell. At least not so long as the generic path doesn't do a tail + // call, since that would be way too weird. - jit.load32(Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex()), SpecializedThunkJIT::regT1); + // The formula for the number of stack bytes needed given some number of parameters (including + // this) is: + // + // stackAlign((numParams + CallFrameHeaderSize) * sizeof(Register) - sizeof(CallerFrameAndPC)) + // + // Probably we want to write this as: + // + // stackAlign((numParams + (CallFrameHeaderSize - CallerFrameAndPCSize)) * sizeof(Register)) + // + // That's really all there is to this. We have all the registers we need to do it. - // Pull out the butterfly from iteratedObject - jit.loadPtr(Address(SpecializedThunkJIT::regT0, JSCell::structureOffset()), SpecializedThunkJIT::regT2); + jit.load32(CCallHelpers::payloadFor(CallFrameSlot::argumentCount), GPRInfo::regT1); + jit.add32(CCallHelpers::TrustedImm32(CallFrame::headerSizeInRegisters - CallerFrameAndPC::sizeInRegisters), GPRInfo::regT1, GPRInfo::regT2); + jit.lshift32(CCallHelpers::TrustedImm32(3), GPRInfo::regT2); + jit.add32(CCallHelpers::TrustedImm32(stackAlignmentBytes() - 1), GPRInfo::regT2); + jit.and32(CCallHelpers::TrustedImm32(-stackAlignmentBytes()), GPRInfo::regT2); - jit.load8(Address(SpecializedThunkJIT::regT2, Structure::indexingTypeOffset()), SpecializedThunkJIT::regT3); - jit.loadPtr(Address(SpecializedThunkJIT::regT0, JSObject::butterflyOffset()), SpecializedThunkJIT::regT2); - Jump nullButterfly = jit.branchTestPtr(SpecializedThunkJIT::Zero, SpecializedThunkJIT::regT2); + if (extraStackNeeded) + jit.add32(CCallHelpers::TrustedImm32(extraStackNeeded), GPRInfo::regT2); - Jump notDone = jit.branch32(SpecializedThunkJIT::Below, SpecializedThunkJIT::regT1, Address(SpecializedThunkJIT::regT2, Butterfly::offsetOfPublicLength())); - - nullButterfly.link(&jit); - - // Return the termination signal to indicate that we've finished - jit.move(TrustedImmPtr(vm->iterationTerminator.get()), SpecializedThunkJIT::regT0); - jit.returnJSCell(SpecializedThunkJIT::regT0); + // At this point regT1 has the actual argument count and regT2 has the amount of stack we will + // need. - notDone.link(&jit); + jit.subPtr(GPRInfo::regT2, CCallHelpers::stackPointerRegister); + + // Do basic callee frame setup, including 'this'. - if (kind == ArrayIterateKey) { - jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex())); - jit.returnInt32(SpecializedThunkJIT::regT1); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "array-iterator-next-key"); - - } - ASSERT(kind == ArrayIterateValue); + jit.loadCell(CCallHelpers::addressFor(CallFrameSlot::callee), GPRInfo::regT3); + + jit.store32(GPRInfo::regT1, CCallHelpers::calleeFramePayloadSlot(CallFrameSlot::argumentCount)); - // Okay, now we're returning a value so make sure we're inside the vector size - jit.appendFailure(jit.branch32(SpecializedThunkJIT::AboveOrEqual, SpecializedThunkJIT::regT1, Address(SpecializedThunkJIT::regT2, Butterfly::offsetOfVectorLength()))); + JSValueRegs valueRegs = JSValueRegs::withTwoAvailableRegs(GPRInfo::regT0, GPRInfo::regT2); + jit.loadValue(CCallHelpers::Address(GPRInfo::regT3, JSBoundFunction::offsetOfBoundThis()), valueRegs); + jit.storeValue(valueRegs, CCallHelpers::calleeArgumentSlot(0)); + + jit.loadPtr(CCallHelpers::Address(GPRInfo::regT3, JSBoundFunction::offsetOfTargetFunction()), GPRInfo::regT3); + jit.storeCell(GPRInfo::regT3, CCallHelpers::calleeFrameSlot(CallFrameSlot::callee)); - // So now we perform inline loads for int32, value/undecided, and double storage - Jump undecidedStorage = jit.branch32(SpecializedThunkJIT::Equal, SpecializedThunkJIT::regT3, TrustedImm32(ArrayWithUndecided)); - Jump notContiguousStorage = jit.branch32(SpecializedThunkJIT::NotEqual, SpecializedThunkJIT::regT3, TrustedImm32(ArrayWithContiguous)); + // OK, now we can start copying. This is a simple matter of copying parameters from the caller's + // frame to the callee's frame. Note that we know that regT1 (the argument count) must be at + // least 1. + jit.sub32(CCallHelpers::TrustedImm32(1), GPRInfo::regT1); + CCallHelpers::Jump done = jit.branchTest32(CCallHelpers::Zero, GPRInfo::regT1); - undecidedStorage.link(&jit); + CCallHelpers::Label loop = jit.label(); + jit.sub32(CCallHelpers::TrustedImm32(1), GPRInfo::regT1); + jit.loadValue(CCallHelpers::addressFor(virtualRegisterForArgument(1)).indexedBy(GPRInfo::regT1, CCallHelpers::TimesEight), valueRegs); + jit.storeValue(valueRegs, CCallHelpers::calleeArgumentSlot(1).indexedBy(GPRInfo::regT1, CCallHelpers::TimesEight)); + jit.branchTest32(CCallHelpers::NonZero, GPRInfo::regT1).linkTo(loop, &jit); - jit.loadPtr(Address(SpecializedThunkJIT::regT0, JSObject::butterflyOffset()), SpecializedThunkJIT::regT2); + done.link(&jit); -#if USE(JSVALUE64) - jit.load64(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight), SpecializedThunkJIT::regT0); - Jump notHole = jit.branchTest64(SpecializedThunkJIT::NonZero, SpecializedThunkJIT::regT0); - jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT0); - notHole.link(&jit); - jit.addPtr(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex())); - jit.returnJSValue(SpecializedThunkJIT::regT0); -#else - jit.load32(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight, JSValue::offsetOfTag()), SpecializedThunkJIT::regT3); - Jump notHole = jit.branch32(SpecializedThunkJIT::NotEqual, SpecializedThunkJIT::regT3, TrustedImm32(JSValue::EmptyValueTag)); - jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1); - jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT0); - jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex())); - jit.returnJSValue(SpecializedThunkJIT::regT0, JSInterfaceJIT::regT1); - notHole.link(&jit); - jit.load32(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight, JSValue::offsetOfPayload()), SpecializedThunkJIT::regT0); - jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex())); - jit.move(SpecializedThunkJIT::regT3, SpecializedThunkJIT::regT1); - jit.returnJSValue(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1); -#endif - notContiguousStorage.link(&jit); + jit.loadPtr( + CCallHelpers::Address(GPRInfo::regT3, JSFunction::offsetOfExecutable()), + GPRInfo::regT0); + jit.loadPtr( + CCallHelpers::Address( + GPRInfo::regT0, ExecutableBase::offsetOfJITCodeWithArityCheckFor(CodeForCall)), + GPRInfo::regT0); + CCallHelpers::Jump noCode = jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT0); - Jump notInt32Storage = jit.branch32(SpecializedThunkJIT::NotEqual, SpecializedThunkJIT::regT3, TrustedImm32(ArrayWithInt32)); - jit.loadPtr(Address(SpecializedThunkJIT::regT0, JSObject::butterflyOffset()), SpecializedThunkJIT::regT2); - jit.load32(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight, JSValue::offsetOfPayload()), SpecializedThunkJIT::regT0); - jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex())); - jit.returnInt32(SpecializedThunkJIT::regT0); - notInt32Storage.link(&jit); + emitPointerValidation(jit, GPRInfo::regT0); + jit.call(GPRInfo::regT0); - jit.appendFailure(jit.branch32(SpecializedThunkJIT::NotEqual, SpecializedThunkJIT::regT3, TrustedImm32(ArrayWithDouble))); - jit.loadPtr(Address(SpecializedThunkJIT::regT0, JSObject::butterflyOffset()), SpecializedThunkJIT::regT2); - jit.loadDouble(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight), SpecializedThunkJIT::fpRegT0); - jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex())); - jit.returnDouble(SpecializedThunkJIT::fpRegT0); + jit.emitFunctionEpilogue(); + jit.ret(); - return jit.finalize(vm->jitStubs->ctiNativeCall(vm), "array-iterator-next-value"); + LinkBuffer linkBuffer(*vm, jit, GLOBAL_THUNK_ID); + linkBuffer.link(noCode, CodeLocationLabel(vm->jitStubs->ctiNativeTailCallWithoutSavedTags(vm))); + return FINALIZE_CODE( + linkBuffer, ("Specialized thunk for bound function calls with no arguments")); } -MacroAssemblerCodeRef arrayIteratorNextKeyThunkGenerator(VM* vm) +#if ENABLE(WEBASSEMBLY) +MacroAssemblerCodeRef throwExceptionFromWasmThunkGenerator(VM* vm) { - return arrayIteratorNextThunkGenerator(vm, ArrayIterateKey); -} + CCallHelpers jit(vm); -MacroAssemblerCodeRef arrayIteratorNextValueThunkGenerator(VM* vm) -{ - return arrayIteratorNextThunkGenerator(vm, ArrayIterateValue); -} - + // The thing that jumps here must move ExceptionType into the argumentGPR1 and jump here. + // We're allowed to use temp registers here, but not callee saves. + { + RegisterSet usedRegisters = RegisterSet::stubUnavailableRegisters(); + usedRegisters.set(GPRInfo::argumentGPR1); + jit.copyCalleeSavesToVMEntryFrameCalleeSavesBuffer(usedRegisters); + } + + jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0); + CCallHelpers::Call call = jit.call(); + jit.jumpToExceptionHandler(); + + void (*throwWasmException)(ExecState*, Wasm::ExceptionType) = [] (ExecState* exec, Wasm::ExceptionType type) { + VM* vm = &exec->vm(); + NativeCallFrameTracer tracer(vm, exec); + + { + auto throwScope = DECLARE_THROW_SCOPE(*vm); + JSGlobalObject* globalObject = vm->topJSWebAssemblyInstance->globalObject(); + + JSWebAssemblyRuntimeError* error = JSWebAssemblyRuntimeError::create(exec, *vm, globalObject->WebAssemblyRuntimeErrorStructure(), Wasm::errorMessageForExceptionType(type)); + throwException(exec, throwScope, error); + } + + genericUnwind(vm, exec); + ASSERT(!!vm->callFrameForCatch); + }; + + LinkBuffer linkBuffer(*vm, jit, GLOBAL_THUNK_ID); + linkBuffer.link(call, throwWasmException); + return FINALIZE_CODE( + linkBuffer, ("Throw exception from Wasm")); } +#endif // ENABLE(WEBASSEMBLY) + +} // namespace JSC #endif // ENABLE(JIT) |