diff options
Diffstat (limited to 'erts/emulator/beam/jit/x86/beam_asm_module.cpp')
| -rw-r--r-- | erts/emulator/beam/jit/x86/beam_asm_module.cpp | 494 |
1 files changed, 170 insertions, 324 deletions
diff --git a/erts/emulator/beam/jit/x86/beam_asm_module.cpp b/erts/emulator/beam/jit/x86/beam_asm_module.cpp index bada1c8395..be01db5ac3 100644 --- a/erts/emulator/beam/jit/x86/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_module.cpp @@ -1,7 +1,7 @@ /* * %CopyrightBegin% * - * Copyright Ericsson AB 2020-2022. All Rights Reserved. + * Copyright Ericsson AB 2020-2023. All Rights Reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,11 +24,6 @@ #include "beam_asm.hpp" using namespace asmjit; -static std::string getAtom(Eterm atom) { - Atom *ap = atom_tab(atom_val(atom)); - return std::string((char *)ap->name, ap->len); -} - #ifdef BEAMASM_DUMP_SIZES # include <mutex> @@ -67,127 +62,31 @@ extern "C" void beamasm_dump_sizes() { } #endif -BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *ga, - Eterm mod, - unsigned num_labels) - : BeamAssembler(getAtom(mod)) { - this->ga = ga; - this->mod = mod; - - labels.reserve(num_labels + 1); - for (unsigned i = 1; i < num_labels; i++) { - Label lbl; - -#ifdef DEBUG - std::string lblName = "label_" + std::to_string(i); - lbl = a.newNamedLabel(lblName.data()); -#else - lbl = a.newLabel(); -#endif +ErtsCodePtr BeamModuleAssembler::getCode(BeamLabel label) { + ASSERT(label < rawLabels.size() + 1); + return (ErtsCodePtr)getCode(rawLabels[label]); +} - labels[i] = lbl; - } +ErtsCodePtr BeamModuleAssembler::getLambda(unsigned index) { + const auto &lambda = lambdas[index]; + return (ErtsCodePtr)getCode(lambda.trampoline); } BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *ga, Eterm mod, - unsigned num_labels, - unsigned num_functions) - : BeamModuleAssembler(ga, mod, num_labels) { - codeHeader = a.newLabel(); + int num_labels, + int num_functions, + const BeamFile *file) + : BeamModuleAssembler(ga, mod, num_labels, file) { + code_header = a.newLabel(); a.align(AlignMode::kCode, 8); - a.bind(codeHeader); + a.bind(code_header); embed_zeros(sizeof(BeamCodeHeader) + sizeof(ErtsCodeInfo *) * num_functions); - - floatMax = a.newLabel(); - a.align(AlignMode::kCode, 8); - a.bind(floatMax); - double max = DBL_MAX; - a.embed((char *)&max, sizeof(double)); - - floatSignMask = a.newLabel(); - a.align(AlignMode::kCode, 16); /* 128-bit aligned */ - a.bind(floatSignMask); - uint64_t signMask = 0x7FFFFFFFFFFFFFFFul; - a.embed((char *)&signMask, sizeof(double)); - - /* Shared trampoline for function_clause errors, which can't jump straight - * to `i_func_info_shared` due to size restrictions. */ - funcInfo = a.newLabel(); - a.align(AlignMode::kCode, 8); - a.bind(funcInfo); - abs_jmp(ga->get_i_func_info_shared()); - - /* Shared trampoline for yielding on function ingress. */ - funcYield = a.newLabel(); - a.align(AlignMode::kCode, 8); - a.bind(funcYield); - abs_jmp(ga->get_i_test_yield_shared()); - - /* Setup the early_nif/breakpoint trampoline. */ - genericBPTramp = a.newLabel(); - a.align(AlignMode::kCode, 16); - a.bind(genericBPTramp); - { - a.ret(); - - a.align(AlignMode::kCode, 16); - abs_jmp(ga->get_call_nif_early()); - - a.align(AlignMode::kCode, 16); - aligned_call(ga->get_generic_bp_local()); - a.ret(); - - a.align(AlignMode::kCode, 16); - ASSERT(a.offset() - code.labelOffsetFromBase(genericBPTramp) == 16 * 3); - aligned_call(ga->get_generic_bp_local()); - abs_jmp(ga->get_call_nif_early()); - } -} - -ErtsCodePtr BeamModuleAssembler::getCode(unsigned label) { - ASSERT(label < labels.size() + 1); - return (ErtsCodePtr)getCode(labels[label]); } -void BeamAssembler::embed_rodata(const char *labelName, - const char *buff, - size_t size) { - Label label = a.newNamedLabel(labelName); - - a.section(rodata); - a.bind(label); - a.embed(buff, size); - a.section(code.textSection()); -} - -void BeamAssembler::embed_bss(const char *labelName, size_t size) { - Label label = a.newNamedLabel(labelName); - - /* Reuse rodata section for now */ - a.section(rodata); - a.bind(label); - embed_zeros(size); - a.section(code.textSection()); -} - -void BeamAssembler::embed_zeros(size_t size) { - static constexpr size_t buf_size = 16384; - static const char zeros[buf_size] = {}; - - while (size >= buf_size) { - a.embed(zeros, buf_size); - size -= buf_size; - } - - if (size > 0) { - a.embed(zeros, size); - } -} - -Label BeamModuleAssembler::embed_vararg_rodata(const std::vector<ArgVal> &args, +Label BeamModuleAssembler::embed_vararg_rodata(const Span<ArgVal> &args, int y_offset) { Label label = a.newLabel(); @@ -206,27 +105,33 @@ Label BeamModuleAssembler::embed_vararg_rodata(const std::vector<ArgVal> &args, a.align(AlignMode::kData, 8); switch (arg.getType()) { - case TAG_x: - data.as_beam = make_loader_x_reg(arg.getValue()); + case ArgVal::XReg: { + auto index = arg.as<ArgXRegister>().get(); + data.as_beam = make_loader_x_reg(index); a.embed(&data.as_char, sizeof(data.as_beam)); - break; - case TAG_y: - data.as_beam = make_loader_y_reg(arg.getValue() + y_offset); + } break; + case ArgVal::YReg: { + auto index = arg.as<ArgYRegister>().get(); + data.as_beam = make_loader_y_reg(index + y_offset); a.embed(&data.as_char, sizeof(data.as_beam)); + } break; + case ArgVal::Literal: { + auto index = arg.as<ArgLiteral>().get(); + make_word_patch(literals[index].patches); + } break; + case ArgVal::Label: + a.embedLabel(resolve_beam_label(arg)); break; - case TAG_q: - make_word_patch(literals[arg.getValue()].patches); - break; - case TAG_f: - a.embedLabel(labels[arg.getValue()]); + case ArgVal::Immediate: + data.as_beam = arg.as<ArgImmed>().get(); + a.embed(&data.as_char, sizeof(data.as_beam)); break; - case TAG_i: - case TAG_u: - /* Tagged immediate or untagged word. */ - data.as_beam = arg.getValue(); + case ArgVal::Word: + data.as_beam = arg.as<ArgWord>().get(); a.embed(&data.as_char, sizeof(data.as_beam)); break; default: + erts_fprintf(stderr, "tag: %li\n", arg.getType()); ERTS_ASSERT(!"error"); } } @@ -236,15 +141,11 @@ Label BeamModuleAssembler::embed_vararg_rodata(const std::vector<ArgVal> &args, return label; } -static void i_emit_nyi(char *msg) { - erts_exit(ERTS_ERROR_EXIT, "NYI: %s\n", msg); -} - void BeamModuleAssembler::emit_i_nif_padding() { const size_t minimum_size = sizeof(UWord[BEAM_NATIVE_MIN_FUNC_SZ]); size_t prev_func_start, diff; - prev_func_start = code.labelOffsetFromBase(labels[functions.back() + 1]); + prev_func_start = code.labelOffsetFromBase(rawLabels[functions.back() + 1]); diff = a.offset() - prev_func_start; if (diff < minimum_size) { @@ -252,34 +153,86 @@ void BeamModuleAssembler::emit_i_nif_padding() { } } +void BeamGlobalAssembler::emit_i_breakpoint_trampoline_shared() { + constexpr ssize_t flag_offset = + sizeof(ErtsCodeInfo) + BEAM_ASM_FUNC_PROLOGUE_SIZE - + offsetof(ErtsCodeInfo, u.metadata.breakpoint_flag); + + Label bp_and_nif = a.newLabel(), bp_only = a.newLabel(), + nif_only = a.newLabel(); + + a.mov(RET, x86::qword_ptr(x86::rsp)); + a.movzx(RETd, x86::byte_ptr(RET, -flag_offset)); + + a.cmp(RETd, imm(ERTS_ASM_BP_FLAG_BP_NIF_CALL_NIF_EARLY)); + a.short_().je(bp_and_nif); + a.cmp(RETd, imm(ERTS_ASM_BP_FLAG_CALL_NIF_EARLY)); + a.short_().je(nif_only); + a.cmp(RETd, imm(ERTS_ASM_BP_FLAG_BP)); + a.short_().je(bp_only); + +#ifndef DEBUG + a.ret(); +#else + Label error = a.newLabel(); + + /* RET must be a valid breakpoint flag. */ + a.test(RETd, RETd); + a.short_().jnz(error); + a.ret(); + + a.bind(error); + a.ud2(); +#endif + + a.bind(bp_and_nif); + { + aligned_call(labels[generic_bp_local]); + /* FALL THROUGH */ + } + + a.bind(nif_only); + { + /* call_nif_early returns on its own, unlike generic_bp_local. */ + a.jmp(labels[call_nif_early]); + } + + a.bind(bp_only); + { + aligned_call(labels[generic_bp_local]); + a.ret(); + } +} + void BeamModuleAssembler::emit_i_breakpoint_trampoline() { /* This little prologue is used by nif loading and tracing to insert * alternative instructions. The call is filled with a relative call to a * trampoline in the module header and then the jmp target is zeroed so that * it effectively becomes a nop */ - byte flag = ERTS_ASM_BP_FLAG_NONE; Label next = a.newLabel(); a.short_().jmp(next); - /* We embed a zero byte here, which is used to flag whether to make an early - * nif call, call a breakpoint handler, or both. */ - a.embed(&flag, sizeof(flag)); - - if (genericBPTramp.isValid()) { - a.call(genericBPTramp); + if (code_header.isValid()) { + auto fragment = ga->get_i_breakpoint_trampoline_shared(); + aligned_call(resolve_fragment(fragment)); } else { /* NIF or BIF stub; we're not going to use this trampoline as-is, but * we need to reserve space for it. */ a.ud2(); + a.align(AlignMode::kCode, sizeof(UWord)); } - a.align(AlignMode::kCode, 8); + ASSERT(a.offset() % sizeof(UWord) == 0); a.bind(next); - ASSERT((a.offset() - code.labelOffsetFromBase(currLabel)) == + ASSERT((a.offset() - code.labelOffsetFromBase(current_label)) == BEAM_ASM_FUNC_PROLOGUE_SIZE); } +static void i_emit_nyi(char *msg) { + erts_exit(ERTS_ERROR_EXIT, "NYI: %s\n", msg); +} + void BeamModuleAssembler::emit_nyi(const char *msg) { emit_enter_runtime(); @@ -293,8 +246,7 @@ void BeamModuleAssembler::emit_nyi() { emit_nyi("<unspecified>"); } -bool BeamModuleAssembler::emit(unsigned specific_op, - const std::vector<ArgVal> &args) { +bool BeamModuleAssembler::emit(unsigned specific_op, const Span<ArgVal> &args) { comment(opc[specific_op].name); #ifdef BEAMASM_DUMP_SIZES @@ -309,18 +261,6 @@ bool BeamModuleAssembler::emit(unsigned specific_op, break; } - if (getOffset() == last_error_offset) { - /* - * The previous PC where an exception may occur is equal to the - * current offset, which is also the offset of the next - * instruction. If the next instruction happens to be a - * line instruction, the location for the exception will - * be that line instruction, which is probably wrong. - * To avoid that, bump the instruction offset. - */ - a.nop(); - } - #ifdef BEAMASM_DUMP_SIZES { std::lock_guard<std::mutex> lock(size_lock); @@ -341,81 +281,100 @@ void BeamGlobalAssembler::emit_i_func_info_shared() { /* Pop the ErtsCodeInfo address into ARG1 and mask out the offset added by * the call instruction. */ a.pop(ARG1); - a.and_(ARG1, ~0x7); + a.and_(ARG1, imm(~0x7)); - a.lea(ARG1, x86::qword_ptr(ARG1, offsetof(ErtsCodeInfo, mfa))); + a.add(ARG1, imm(offsetof(ErtsCodeInfo, mfa))); a.mov(x86::qword_ptr(c_p, offsetof(Process, freason)), EXC_FUNCTION_CLAUSE); a.mov(x86::qword_ptr(c_p, offsetof(Process, current)), ARG1); - a.jmp(labels[error_action_code]); + + mov_imm(ARG2, 0); + mov_imm(ARG4, 0); + a.jmp(labels[raise_exception_shared]); } -void BeamModuleAssembler::emit_i_func_info(const ArgVal &Label, - const ArgVal &Module, - const ArgVal &Function, - const ArgVal &Arity) { +void BeamModuleAssembler::emit_i_func_info(const ArgWord &Label, + const ArgAtom &Module, + const ArgAtom &Function, + const ArgWord &Arity) { ErtsCodeInfo info; - functions.push_back(Label.getValue()); + /* `op_i_func_info_IaaI` is used in various places in the emulator, so this + * label is always encoded as a word, even though the signature ought to + * be `op_i_func_info_LaaI`. */ + functions.push_back(Label.get()); - info.mfa.module = Module.getValue(); - info.mfa.function = Function.getValue(); - info.mfa.arity = Arity.getValue(); - info.u.gen_bp = NULL; + info.mfa.module = Module.get(); + info.mfa.function = Function.get(); + info.mfa.arity = Arity.get(); + info.gen_bp = NULL; comment("%T:%T/%d", info.mfa.module, info.mfa.function, info.mfa.arity); /* This is an ErtsCodeInfo structure that has a valid x86 opcode as its `op` - * field, which *calls* the funcInfo trampoline so we can trace it back to - * this particular function. + * field, which *calls* the `_i_func_info_shared` fragment so we can trace + * it back to this particular function. * - * We make a relative call to a trampoline in the module header because this - * needs to fit into a word, and an directy call to `i_func_info_shared` - * would be too large. */ - if (funcInfo.isValid()) { - a.call(funcInfo); - } else { - a.nop(); - } - - a.align(AlignMode::kCode, sizeof(UWord)); - a.embed(&info.u.gen_bp, sizeof(info.u.gen_bp)); + * We also use this field to store the current breakpoint flag so that we + * only have to modify a single branch target when changing breakpoints. */ + a.call(resolve_fragment(ga->get_i_func_info_shared())); + a.nop(); + a.nop(); + a.embedUInt8(ERTS_ASM_BP_FLAG_NONE); + + ASSERT(a.offset() % sizeof(UWord) == 0); + a.embed(&info.gen_bp, sizeof(info.gen_bp)); a.embed(&info.mfa, sizeof(info.mfa)); } -void BeamModuleAssembler::emit_label(const ArgVal &Label) { - currLabel = labels[Label.getValue()]; - a.bind(currLabel); +void BeamModuleAssembler::emit_label(const ArgLabel &Label) { + ASSERT(Label.isLabel()); + + current_label = rawLabels[Label.get()]; + a.bind(current_label); } -void BeamModuleAssembler::emit_aligned_label(const ArgVal &Label, - const ArgVal &Alignment) { - ASSERT(Alignment.getType() == ArgVal::u); - a.align(AlignMode::kCode, Alignment.getValue()); +void BeamModuleAssembler::emit_aligned_label(const ArgLabel &Label, + const ArgWord &Alignment) { + a.align(AlignMode::kCode, Alignment.get()); emit_label(Label); } void BeamModuleAssembler::emit_on_load() { - on_load = currLabel; + on_load = current_label; } void BeamModuleAssembler::emit_int_code_end() { /* This label is used to figure out the end of the last function */ - labels[labels.size() + 1] = a.newLabel(); - a.bind(labels[labels.size()]); + code_end = a.newLabel(); + a.bind(code_end); emit_nyi("int_code_end"); + + for (auto pair : _dispatchTable) { + a.bind(pair.second); + a.jmp(imm(pair.first)); + } } -void BeamModuleAssembler::emit_line(const ArgVal &) { - /* - * There is no need to align the line instruction. In the loaded - * code, the type of the pointer will be void* and that pointer - * will only be used in comparisons. - */ +void BeamModuleAssembler::emit_line(const ArgWord &Loc) { + /* There is no need to align the line instruction. In the loaded code, the + * type of the pointer will be void* and that pointer will only be used in + * comparisons. + * + * We only need to do something when there's a possibility of raising an + * exception at the very end of the preceding instruction (and thus + * pointing at the start of this one). If we were to do nothing, the error + * would erroneously refer to this instead of the preceding line. + * + * Since line addresses are taken _after_ line instructions we can avoid + * this by adding a nop when we detect this condition. */ + if (a.offset() == last_error_offset) { + a.nop(); + } } -void BeamModuleAssembler::emit_func_line(const ArgVal &Loc) { +void BeamModuleAssembler::emit_func_line(const ArgWord &Loc) { emit_line(Loc); } @@ -434,7 +393,7 @@ void BeamModuleAssembler::emit_i_generic_breakpoint() { emit_nyi("i_generic_breakpoint should never be called"); } -void BeamModuleAssembler::emit_trace_jump(const ArgVal &) { +void BeamModuleAssembler::emit_trace_jump(const ArgWord &) { emit_nyi("trace_jump should never be called"); } @@ -442,129 +401,6 @@ void BeamModuleAssembler::emit_call_error_handler() { emit_nyi("call_error_handler should never be called"); } -void BeamModuleAssembler::codegen(JitAllocator *allocator, - const void **executable_ptr, - void **writable_ptr, - const BeamCodeHeader *in_hdr, - const BeamCodeHeader **out_exec_hdr, - BeamCodeHeader **out_rw_hdr) { - const BeamCodeHeader *code_hdr_exec; - BeamCodeHeader *code_hdr_rw; - - codegen(allocator, executable_ptr, writable_ptr); - - { - auto offset = code.labelOffsetFromBase(codeHeader); - - auto base_exec = (const char *)(*executable_ptr); - code_hdr_exec = (const BeamCodeHeader *)&base_exec[offset]; - - auto base_rw = (const char *)(*writable_ptr); - code_hdr_rw = (BeamCodeHeader *)&base_rw[offset]; - } - - sys_memcpy(code_hdr_rw, in_hdr, sizeof(BeamCodeHeader)); - code_hdr_rw->on_load = getOnLoad(); - - for (unsigned i = 0; i < functions.size(); i++) { - ErtsCodeInfo *ci = (ErtsCodeInfo *)getCode(functions[i]); - code_hdr_rw->functions[i] = ci; - } - - char *module_end = (char *)code.baseAddress() + a.offset(); - code_hdr_rw->functions[functions.size()] = (ErtsCodeInfo *)module_end; - - *out_exec_hdr = code_hdr_exec; - *out_rw_hdr = code_hdr_rw; -} - -void BeamModuleAssembler::codegen(JitAllocator *allocator, - const void **executable_ptr, - void **writable_ptr) { - _codegen(allocator, executable_ptr, writable_ptr); - -#ifndef WIN32 - if (functions.size()) { - char *buff = (char *)erts_alloc(ERTS_ALC_T_TMP, 1024); - std::vector<AsmRange> ranges; - std::string name = getAtom(mod); - ranges.reserve(functions.size() + 2); - - /* Push info about the header */ - ranges.push_back({.start = (ErtsCodePtr)getBaseAddress(), - .stop = getCode(functions[0]), - .name = name + "::codeHeader"}); - - for (unsigned i = 0; i < functions.size(); i++) { - ErtsCodePtr start, stop; - const ErtsCodeInfo *ci; - int n; - - start = getCode(functions[i]); - ci = (const ErtsCodeInfo *)start; - - n = erts_snprintf(buff, - 1024, - "%T:%T/%d", - ci->mfa.module, - ci->mfa.function, - ci->mfa.arity); - stop = ((const char *)erts_codeinfo_to_code(ci)) + - BEAM_ASM_FUNC_PROLOGUE_SIZE; - - /* We use a different symbol for CodeInfo and the Prologue - in order for the perf disassembly to be better. */ - std::string name(buff, n); - ranges.push_back({.start = start, - .stop = stop, - .name = name + "-CodeInfoPrologue"}); - - /* The actual code */ - start = stop; - if (i + 1 < functions.size()) { - stop = getCode(functions[i + 1]); - } else { - stop = getCode(labels.size()); - } - - ranges.push_back({.start = start, .stop = stop, .name = name}); - } - - /* Push info about the footer */ - ranges.push_back( - {.start = ranges.back().stop, - .stop = (ErtsCodePtr)(code.baseAddress() + code.codeSize()), - .name = name + "::codeFooter"}); - - update_gdb_jit_info(name, ranges); - beamasm_update_perf_info(name, ranges); - erts_free(ERTS_ALC_T_TMP, buff); - } -#endif -} - -void BeamModuleAssembler::codegen(char *buff, size_t len) { - code.flatten(); - code.resolveUnresolvedLinks(); - ERTS_ASSERT(code.codeSize() <= len); - code.relocateToBase((uint64_t)buff); - code.copyFlattenedData(buff, - code.codeSize(), - CopySectionFlags::kPadSectionBuffer); -} - -BeamCodeHeader *BeamModuleAssembler::getCodeHeader() { - return (BeamCodeHeader *)getCode(codeHeader); -} - -const ErtsCodeInfo *BeamModuleAssembler::getOnLoad() { - if (on_load.isValid()) { - return erts_code_to_codeinfo((ErtsCodePtr)getCode(on_load)); - } else { - return 0; - } -} - unsigned BeamModuleAssembler::patchCatches(char *rw_base) { unsigned catch_no = BEAM_CATCHES_NIL; @@ -637,3 +473,13 @@ void BeamModuleAssembler::patchStrings(char *rw_base, *where = string_table + patch.val_offs; } } + +const Label &BeamModuleAssembler::resolve_fragment(void (*fragment)()) { + auto it = _dispatchTable.find(fragment); + + if (it == _dispatchTable.end()) { + it = _dispatchTable.emplace(fragment, a.newLabel()).first; + } + + return it->second; +} |