diff options
author | John Högberg <john@erlang.org> | 2023-04-05 15:44:43 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-04-05 15:44:43 +0200 |
commit | c101574adf0c562650e0baf2a8d14e8ab61d91a6 (patch) | |
tree | 3d984d0539c9851a038a2316889e749013d941b8 /erts | |
parent | 977b10afa168996b2f4e426ab3dd2b153597c692 (diff) | |
parent | 64514614f394fbd3c9307fc8f6a2b69429801092 (diff) | |
download | erlang-c101574adf0c562650e0baf2a8d14e8ab61d91a6.tar.gz |
Merge pull request #7036 from jhogberg/john/jit/share-more-multiarch-code
jit: Share more code between architectures
Diffstat (limited to 'erts')
-rw-r--r-- | erts/emulator/beam/beam_file.c | 5 | ||||
-rw-r--r-- | erts/emulator/beam/beam_types.c | 56 | ||||
-rw-r--r-- | erts/emulator/beam/beam_types.h | 31 | ||||
-rw-r--r-- | erts/emulator/beam/jit/arm/beam_asm.hpp | 398 | ||||
-rw-r--r-- | erts/emulator/beam/jit/arm/beam_asm_module.cpp | 95 | ||||
-rw-r--r-- | erts/emulator/beam/jit/beam_jit_common.cpp | 155 | ||||
-rw-r--r-- | erts/emulator/beam/jit/beam_jit_common.hpp | 460 | ||||
-rw-r--r-- | erts/emulator/beam/jit/beam_jit_main.cpp | 2 | ||||
-rw-r--r-- | erts/emulator/beam/jit/beam_jit_metadata.cpp | 26 | ||||
-rw-r--r-- | erts/emulator/beam/jit/beam_jit_types.hpp | 150 | ||||
-rw-r--r-- | erts/emulator/beam/jit/x86/beam_asm.hpp | 391 | ||||
-rw-r--r-- | erts/emulator/beam/jit/x86/beam_asm_module.cpp | 73 | ||||
-rw-r--r-- | erts/emulator/beam/jit/x86/instr_bs.cpp | 4 | ||||
-rw-r--r-- | erts/emulator/beam/jit/x86/instr_common.cpp | 3 |
14 files changed, 763 insertions, 1086 deletions
diff --git a/erts/emulator/beam/beam_file.c b/erts/emulator/beam/beam_file.c index b6c23d0554..e1215e8640 100644 --- a/erts/emulator/beam/beam_file.c +++ b/erts/emulator/beam/beam_file.c @@ -651,14 +651,13 @@ static int parse_type_chunk_otp_26(BeamFile *beam, BeamReader *p_reader) { for (i = 0; i < count; i++) { const byte *type_data; - int raw_types; int extra; LoadAssert(beamreader_read_bytes(p_reader, 2, &type_data)); - extra = beam_types_decode_type_otp_26(type_data, &raw_types, &types->entries[i]); + extra = beam_types_decode_type_otp_26(type_data, &types->entries[i]); LoadAssert(extra >= 0); LoadAssert(beamreader_read_bytes(p_reader, extra, &type_data)); - beam_types_decode_extra_otp_26(type_data, raw_types, &types->entries[i]); + beam_types_decode_extra_otp_26(type_data, &types->entries[i]); } /* The first entry MUST be the "any type." */ diff --git a/erts/emulator/beam/beam_types.c b/erts/emulator/beam/beam_types.c index 659e5b0d1a..876db9f47b 100644 --- a/erts/emulator/beam/beam_types.c +++ b/erts/emulator/beam/beam_types.c @@ -30,50 +30,65 @@ static Sint64 get_sint64(const byte *data); -int beam_types_decode_type_otp_26(const byte *data, int *raw_types_ptr, BeamType *out) { - int types; - int extra = 0; +int beam_types_decode_type_otp_26(const byte *data, BeamType *out) { + int flags, extra; - types = (Uint16)data[0] << 8 | (Uint16)data[1]; - if (types == BEAM_TYPE_NONE) { + flags = (Uint16)data[0] << 8 | (Uint16)data[1]; + if (flags == BEAM_TYPE_NONE) { return -1; } - /* The "extra data" aren't part of the type union proper. */ - out->type_union = types & BEAM_TYPE_ANY; + extra = 0; + + out->type_union = flags & BEAM_TYPE_ANY; + out->metadata_flags = flags & BEAM_TYPE_METADATA_MASK; + + if (out->metadata_flags & BEAM_TYPE_HAS_LOWER_BOUND) { + if (!(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER))) { + return -1; + } - if (types & BEAM_TYPE_HAS_LOWER_BOUND) { extra += 8; } - if (types & BEAM_TYPE_HAS_UPPER_BOUND) { + if (out->metadata_flags & BEAM_TYPE_HAS_UPPER_BOUND) { + if (!(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER))) { + return -1; + } + extra += 8; } - if (types & BEAM_TYPE_HAS_UNIT) { + if (out->metadata_flags & BEAM_TYPE_HAS_UNIT) { + if (!(out->type_union & BEAM_TYPE_BITSTRING)) { + return -1; + } + extra += 1; } - *raw_types_ptr = types; return extra; } -void beam_types_decode_extra_otp_26(const byte *data, int types, BeamType *out) { +void beam_types_decode_extra_otp_26(const byte *data, BeamType *out) { out->min = MAX_SMALL + 1; out->max = MIN_SMALL - 1; out->size_unit = 1; - if (types & BEAM_TYPE_HAS_LOWER_BOUND) { + if (out->metadata_flags & BEAM_TYPE_HAS_LOWER_BOUND) { + ASSERT(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)); out->min = get_sint64(data); data += 8; } - if (types & BEAM_TYPE_HAS_UPPER_BOUND) { + if (out->metadata_flags & BEAM_TYPE_HAS_UPPER_BOUND) { + ASSERT(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER)); out->max = get_sint64(data); data += 8; } - if (types & BEAM_TYPE_HAS_UNIT) { + if (out->metadata_flags & BEAM_TYPE_HAS_UNIT) { + ASSERT(out->type_union & BEAM_TYPE_BITSTRING); out->size_unit = data[0] + 1; } } @@ -97,9 +112,18 @@ int beam_types_decode_otp_25(const byte *data, Uint size, BeamType *out) { data += 8; out->max = get_sint64(data); - if (out->min > out->max) { + if (out->min <= out->max) { + if (!(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER))) { + return -1; + } + + out->metadata_flags = (BEAM_TYPE_HAS_LOWER_BOUND | + BEAM_TYPE_HAS_UPPER_BOUND); + } else { out->min = MAX_SMALL + 1; out->max = MIN_SMALL - 1; + + out->metadata_flags = 0; } out->size_unit = 1; diff --git a/erts/emulator/beam/beam_types.h b/erts/emulator/beam/beam_types.h index cd15f37b98..fe215a1340 100644 --- a/erts/emulator/beam/beam_types.h +++ b/erts/emulator/beam/beam_types.h @@ -63,32 +63,33 @@ #define BEAM_TYPE_HAS_UPPER_BOUND (1 << 14) #define BEAM_TYPE_HAS_UNIT (1 << 15) +#define BEAM_TYPE_METADATA_MASK (BEAM_TYPE_HAS_LOWER_BOUND | \ + BEAM_TYPE_HAS_UPPER_BOUND | \ + BEAM_TYPE_HAS_UNIT) + typedef struct { /** @brief A set of the possible types (atom, tuple, etc) this term may * be. When a single bit is set, the term will always be of that type. */ int type_union; - /** @brief Minimum and maximum values. Valid if at least one of - * IS_SSMALL(min) and IS_SSMALL(max) is true; otherwise the range is - * unknown. - * - * If and only if min < max, then the value is always a small within - * the range min trough max (including the endpoints). - * - * If and only if IS_SSMALL(min) && !IS_SSMALL(max) is true, then - * the value is greater than or equal to min. - * - * If and only if !IS_SSMALL(min) && IS_SSMALL(max), then the - * value is less than or equal to min. */ + /** @brief A set of metadata presence flags, BEAM_TYPE_HAS_XYZ. */ + int metadata_flags; + + /** @brief Minimum numerical value. Only valid when the + * BEAM_TYPE_HAS_LOWER_BOUND metadata flag is present. */ Sint64 min; + + /** @brief Maximum numerical value. Only valid when the + * BEAM_TYPE_HAS_UPPER_BOUND metadata flag is present. */ Sint64 max; - /** @brief Unit for bitstring size. */ + /** @brief Unit for bitstring size. Only valid when the BEAM_TYPE_HAS_UNIT + * metadata flag is present. */ byte size_unit; } BeamType; -int beam_types_decode_type_otp_26(const byte *data, int *raw_types_ptr, BeamType *out); -void beam_types_decode_extra_otp_26(const byte *data, int types, BeamType *out); +int beam_types_decode_type_otp_26(const byte *data, BeamType *out); +void beam_types_decode_extra_otp_26(const byte *data, BeamType *out); int beam_types_decode_otp_25(const byte *data, Uint size, BeamType *out); #endif diff --git a/erts/emulator/beam/jit/arm/beam_asm.hpp b/erts/emulator/beam/jit/arm/beam_asm.hpp index a6a3f3bb98..8ef609f4b2 100644 --- a/erts/emulator/beam/jit/arm/beam_asm.hpp +++ b/erts/emulator/beam/jit/arm/beam_asm.hpp @@ -60,18 +60,20 @@ extern "C" using namespace asmjit; -class BeamAssembler : public ErrorHandler { -protected: - /* Holds code and relocation information. */ - CodeHolder code; - - a64::Assembler a; - - FileLogger logger; +struct BeamAssembler : public BeamAssemblerCommon { + BeamAssembler() : BeamAssemblerCommon(a) { + Error err = code.attach(&a); + ERTS_ASSERT(!err && "Failed to attach codeHolder"); + } - Section *rodata = nullptr; + BeamAssembler(const std::string &log) : BeamAssembler() { + if (erts_jit_asm_dump) { + setLogger(log + ".asm"); + } + } - /* * * * * * * * * */ +protected: + a64::Assembler a; /* Points at x_reg_array inside an ErtsSchedulerRegisters struct, allowing * the aux_regs field to be addressed with an 8-bit displacement. */ @@ -224,27 +226,6 @@ protected: kNone = 0 }; -public: - static bool hasCpuFeature(uint32_t featureId); - - BeamAssembler(); - BeamAssembler(const std::string &log); - - ~BeamAssembler(); - - void *getBaseAddress(); - size_t getOffset(); - -protected: - void codegen(JitAllocator *allocator, - const void **executable_ptr, - void **writable_ptr); - - void *getCode(Label label); - byte *getCode(char *labelName); - - void handleError(Error err, const char *message, BaseEmitter *origin); - #ifdef JIT_HARD_DEBUG constexpr arm::Mem getInitialSPRef() const { int base = offsetof(ErtsSchedulerRegisters, initial_sp); @@ -823,125 +804,19 @@ protected: add(to, arm::GpX(mem.baseId()), offset); } } - -public: - void embed_rodata(const char *labelName, const char *buff, size_t size); - void embed_bss(const char *labelName, size_t size); - void embed_zeros(size_t size); - - void setLogger(std::string log); - void setLogger(FILE *log); - - void comment(const char *format) { - if (logger.file()) { - a.commentf("# %s", format); - } - } - - template<typename... Ts> - void comment(const char *format, Ts... args) { - if (logger.file()) { - char buff[1024]; - erts_snprintf(buff, sizeof(buff), format, args...); - a.commentf("# %s", buff); - } - } - - struct AsmRange { - ErtsCodePtr start; - ErtsCodePtr stop; - const std::string name; - - struct LineData { - ErtsCodePtr start; - const std::string file; - unsigned line; - }; - - const std::vector<LineData> lines; - }; - - void embed(void *data, uint32_t size) { - a.embed((char *)data, size); - } }; #include "beam_asm_global.hpp" -class BeamModuleAssembler : public BeamAssembler { - typedef unsigned BeamLabel; - - /* Map of BEAM label number to asmjit Label. These should not be used - * directly by most instructions because of displacement limits, use - * `resolve_beam_label` instead. */ - typedef std::unordered_map<BeamLabel, const Label> LabelMap; - LabelMap rawLabels; +class BeamModuleAssembler : public BeamAssembler, + public BeamModuleAssemblerCommon { + BeamGlobalAssembler *ga; /* Sequence number used to create unique named labels by * resolve_label(). Only used when assembly output has been * requested. */ long labelSeq = 0; - struct patch { - Label where; - uint64_t val_offs; - }; - - struct patch_catch { - struct patch patch; - Label handler; - }; - std::vector<struct patch_catch> catches; - - /* Map of import entry to patch labels and mfa */ - struct patch_import { - std::vector<struct patch> patches; - ErtsCodeMFA mfa; - }; - typedef std::unordered_map<unsigned, struct patch_import> ImportMap; - ImportMap imports; - - /* Map of fun entry to patch labels */ - struct patch_lambda { - std::vector<struct patch> patches; - Label trampoline; - }; - typedef std::unordered_map<unsigned, struct patch_lambda> LambdaMap; - LambdaMap lambdas; - - /* Map of literals to patch labels */ - struct patch_literal { - std::vector<struct patch> patches; - }; - typedef std::unordered_map<unsigned, struct patch_literal> LiteralMap; - LiteralMap literals; - - /* All string patches */ - std::vector<struct patch> strings; - - /* All functions that have been seen so far */ - std::vector<BeamLabel> functions; - - /* The BEAM file we've been loaded from, if any. */ - const BeamFile *beam; - - BeamGlobalAssembler *ga; - - /* Used by emit to populate the labelToMFA map */ - Label current_label; - - /* The offset of our BeamCodeHeader, if any. */ - Label code_header; - - /* The module's on_load function, if any. */ - Label on_load; - - /* The end of the last function. Note that the dispatch table, constants, - * and veneers may follow. */ - Label code_end; - - Eterm mod; - /* Save the last PC for an error. */ size_t last_error_offset = 0; @@ -1190,240 +1065,6 @@ public: void patchStrings(char *rw_base, const byte *string); protected: - const auto &getTypeEntry(const ArgSource &arg) const { - auto typeIndex = - arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; - ASSERT(typeIndex < beam->types.count); - return beam->types.entries[typeIndex]; - } - - auto getTypeUnion(const ArgSource &arg) const { - if (arg.isRegister()) { - return static_cast<BeamTypeId>(getTypeEntry(arg).type_union); - } - - Eterm constant = - arg.isImmed() - ? arg.as<ArgImmed>().get() - : beamfile_get_literal(beam, - arg.as<ArgLiteral>().get()); - - switch (tag_val_def(constant)) { - case ATOM_DEF: - return BeamTypeId::Atom; - case BINARY_DEF: - return BeamTypeId::Bitstring; - case FLOAT_DEF: - return BeamTypeId::Float; - case FUN_DEF: - return BeamTypeId::Fun; - case BIG_DEF: - case SMALL_DEF: - return BeamTypeId::Integer; - case LIST_DEF: - return BeamTypeId::Cons; - case MAP_DEF: - return BeamTypeId::Map; - case NIL_DEF: - return BeamTypeId::Nil; - case EXTERNAL_PID_DEF: - case PID_DEF: - return BeamTypeId::Pid; - case EXTERNAL_PORT_DEF: - case PORT_DEF: - return BeamTypeId::Port; - case EXTERNAL_REF_DEF: - case REF_DEF: - return BeamTypeId::Reference; - case TUPLE_DEF: - return BeamTypeId::Tuple; - default: - ERTS_ASSERT(!"tag_val_def error"); - } - } - - auto getClampedRange(const ArgSource &arg) const { - if (arg.isSmall()) { - Sint value = arg.as<ArgSmall>().getSigned(); - return std::make_pair(value, value); - } else { - const auto &entry = getTypeEntry(arg); - - if (entry.min <= entry.max) { - return std::make_pair(entry.min, entry.max); - } else if (IS_SSMALL(entry.min) && !IS_SSMALL(entry.max)) { - return std::make_pair(entry.min, MAX_SMALL); - } else if (!IS_SSMALL(entry.min) && IS_SSMALL(entry.max)) { - return std::make_pair(MIN_SMALL, entry.max); - } else { - return std::make_pair(MIN_SMALL, MAX_SMALL); - } - } - } - - int getSizeUnit(const ArgSource &arg) const { - const auto &entry = getTypeEntry(arg); - ASSERT(maybe_one_of<BeamTypeId::Bitstring>(arg)); - return entry.size_unit; - } - - bool hasLowerBound(const ArgSource &arg) const { - const auto &entry = getTypeEntry(arg); - return IS_SSMALL(entry.min) && !IS_SSMALL(entry.max); - } - - bool hasUpperBound(const ArgSource &arg) const { - const auto &entry = getTypeEntry(arg); - return !IS_SSMALL(entry.min) && IS_SSMALL(entry.max); - } - - bool always_small(const ArgSource &arg) const { - if (arg.isSmall()) { - return true; - } else if (!exact_type<BeamTypeId::Integer>(arg)) { - return false; - } - - const auto &entry = getTypeEntry(arg); - return entry.min <= entry.max; - } - - bool always_immediate(const ArgSource &arg) const { - return always_one_of<BeamTypeId::AlwaysImmediate>(arg) || - always_small(arg); - } - - bool always_same_types(const ArgSource &lhs, const ArgSource &rhs) const { - using integral = std::underlying_type_t<BeamTypeId>; - auto lhs_types = static_cast<integral>(getTypeUnion(lhs)); - auto rhs_types = static_cast<integral>(getTypeUnion(rhs)); - - /* We can only be certain that the types are the same when there's - * one possible type. For example, if one is a number and the other - * is an integer, they could differ if the former is a float. */ - if ((lhs_types & (lhs_types - 1)) == 0) { - return lhs_types == rhs_types; - } - - return false; - } - - template<BeamTypeId... Types> - bool never_one_of(const ArgSource &arg) const { - using integral = std::underlying_type_t<BeamTypeId>; - auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value()); - auto type_union = static_cast<integral>(getTypeUnion(arg)); - return static_cast<BeamTypeId>(type_union & types) == BeamTypeId::None; - } - - template<BeamTypeId... Types> - bool maybe_one_of(const ArgSource &arg) const { - return !never_one_of<Types...>(arg); - } - - template<BeamTypeId... Types> - bool always_one_of(const ArgSource &arg) const { - /* Providing a single type to this function is not an error per se, but - * `exact_type` provides a bit more error checking for that use-case, - * so we want to encourage its use. */ - static_assert(!BeamTypeIdUnion<Types...>::is_single_typed(), - "always_one_of expects a union of several primitive " - "types, use exact_type instead"); - - using integral = std::underlying_type_t<BeamTypeId>; - auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value()); - auto type_union = static_cast<integral>(getTypeUnion(arg)); - return type_union == (type_union & types); - } - - template<BeamTypeId Type> - bool exact_type(const ArgSource &arg) const { - /* Rejects `exact_type<BeamTypeId::List>(...)` and similar, as it's - * almost always an error to exactly match a union of several types. - * - * On the off chance that you _do_ need to match a union exactly, use - * `masked_types<T>(arg) == T` instead. */ - static_assert(BeamTypeIdUnion<Type>::is_single_typed(), - "exact_type expects exactly one primitive type, use " - "always_one_of instead"); - - using integral = std::underlying_type_t<BeamTypeId>; - auto type_union = static_cast<integral>(getTypeUnion(arg)); - return type_union == (type_union & static_cast<integral>(Type)); - } - - template<BeamTypeId Mask> - BeamTypeId masked_types(const ArgSource &arg) const { - static_assert((Mask != BeamTypeId::AlwaysBoxed && - Mask != BeamTypeId::AlwaysImmediate), - "using masked_types with AlwaysBoxed or AlwaysImmediate " - "is almost always an error, use exact_type, " - "maybe_one_of, or never_one_of instead"); - static_assert((Mask == BeamTypeId::MaybeBoxed || - Mask == BeamTypeId::MaybeImmediate || - Mask == BeamTypeId::AlwaysBoxed || - Mask == BeamTypeId::AlwaysImmediate), - "masked_types expects a mask type like MaybeBoxed or " - "MaybeImmediate, use exact_type, maybe_one_of, or" - "never_one_of instead"); - - using integral = std::underlying_type_t<BeamTypeId>; - auto mask = static_cast<integral>(Mask); - auto type_union = static_cast<integral>(getTypeUnion(arg)); - return static_cast<BeamTypeId>(type_union & mask); - } - - bool is_sum_small_if_args_are_small(const ArgSource &LHS, - const ArgSource &RHS) { - Sint min, max; - auto [min1, max1] = getClampedRange(LHS); - auto [min2, max2] = getClampedRange(RHS); - min = min1 + min2; - max = max1 + max2; - return IS_SSMALL(min) && IS_SSMALL(max); - } - - bool is_diff_small_if_args_are_small(const ArgSource &LHS, - const ArgSource &RHS) { - Sint min, max; - auto [min1, max1] = getClampedRange(LHS); - auto [min2, max2] = getClampedRange(RHS); - min = min1 - max2; - max = max1 - min2; - return IS_SSMALL(min) && IS_SSMALL(max); - } - - bool is_product_small_if_args_are_small(const ArgSource &LHS, - const ArgSource &RHS) { - auto [min1, max1] = getClampedRange(LHS); - auto [min2, max2] = getClampedRange(RHS); - auto mag1 = std::max(std::abs(min1), std::abs(max1)); - auto mag2 = std::max(std::abs(min2), std::abs(max2)); - - /* - * mag1 * mag2 <= MAX_SMALL - * mag1 <= MAX_SMALL / mag2 (when mag2 != 0) - */ - ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL); - return mag2 == 0 || mag1 <= MAX_SMALL / mag2; - } - - bool is_bsl_small(const ArgSource &LHS, const ArgSource &RHS) { - if (!(always_small(LHS) && always_small(RHS))) { - return false; - } else { - auto [min1, max1] = getClampedRange(LHS); - auto [min2, max2] = getClampedRange(RHS); - - if (min1 < 0 || max1 == 0 || min2 < 0) { - return false; - } - - return max2 < Support::clz(max1) - _TAG_IMMED1_SIZE; - } - } - - /* Helpers */ void emit_gc_test(const ArgWord &Stack, const ArgWord &Heap, const ArgWord &Live); @@ -1971,10 +1612,9 @@ protected: } }; -void beamasm_metadata_update( - std::string module_name, - ErtsCodePtr base_address, - size_t code_size, - const std::vector<BeamAssembler::AsmRange> &ranges); +void beamasm_metadata_update(std::string module_name, + ErtsCodePtr base_address, + size_t code_size, + const std::vector<AsmRange> &ranges); void beamasm_metadata_early_init(); void beamasm_metadata_late_init(); diff --git a/erts/emulator/beam/jit/arm/beam_asm_module.cpp b/erts/emulator/beam/jit/arm/beam_asm_module.cpp index c368aeb47e..57f5746a8c 100644 --- a/erts/emulator/beam/jit/arm/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/arm/beam_asm_module.cpp @@ -475,80 +475,6 @@ void BeamModuleAssembler::emit_call_error_handler() { emit_nyi("call_error_handler should never be called"); } -unsigned BeamModuleAssembler::patchCatches(char *rw_base) { - unsigned catch_no = BEAM_CATCHES_NIL; - - for (const auto &c : catches) { - const auto &patch = c.patch; - ErtsCodePtr handler; - - handler = (ErtsCodePtr)getCode(c.handler); - catch_no = beam_catches_cons(handler, catch_no, nullptr); - - /* Patch the `mov` instruction with the catch tag */ - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (Eterm *)&rw_base[offset]; - - ASSERT(LLONG_MAX == *where); - Eterm catch_term = make_catch(catch_no); - - /* With the current tag scheme, more than 33 million - * catches can exist at once. */ - ERTS_ASSERT(catch_term >> 31 == 0); - - *where = catch_term; - } - - return catch_no; -} - -void BeamModuleAssembler::patchImport(char *rw_base, - unsigned index, - const Export *import) { - for (const auto &patch : imports[index].patches) { - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (Eterm *)&rw_base[offset]; - - ASSERT(LLONG_MAX == *where); - *where = reinterpret_cast<Eterm>(import) + patch.val_offs; - } -} - -void BeamModuleAssembler::patchLambda(char *rw_base, - unsigned index, - const ErlFunEntry *fe) { - for (const auto &patch : lambdas[index].patches) { - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (Eterm *)&rw_base[offset]; - - ASSERT(LLONG_MAX == *where); - *where = reinterpret_cast<Eterm>(fe) + patch.val_offs; - } -} - -void BeamModuleAssembler::patchLiteral(char *rw_base, - unsigned index, - Eterm lit) { - for (const auto &patch : literals[index].patches) { - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (Eterm *)&rw_base[offset]; - - ASSERT(LLONG_MAX == *where); - *where = lit + patch.val_offs; - } -} - -void BeamModuleAssembler::patchStrings(char *rw_base, - const byte *string_table) { - for (const auto &patch : strings) { - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (const byte **)&rw_base[offset]; - - ASSERT(LLONG_MAX == (Eterm)*where); - *where = string_table + patch.val_offs; - } -} - const Label &BeamModuleAssembler::resolve_beam_label(const ArgLabel &Lbl, enum Displacement disp) { ASSERT(Lbl.isLabel()); @@ -804,30 +730,37 @@ void BeamModuleAssembler::emit_constant(const Constant &constant) { } else if (value.isLabel()) { a.embedLabel(rawLabels.at(value.as<ArgLabel>().get())); } else { - a.embedUInt64(LLONG_MAX); - switch (value.getType()) { case ArgVal::BytePtr: - strings.push_back({anchor, value.as<ArgBytePtr>().get()}); + strings.push_back({anchor, 0, value.as<ArgBytePtr>().get()}); + a.embedUInt64(LLONG_MAX); break; case ArgVal::Catch: { auto handler = rawLabels[value.as<ArgCatch>().get()]; - catches.push_back({{anchor, 0}, handler}); + catches.push_back({{anchor, 0, 0}, handler}); + + /* Catches are limited to 32 bits, but since we don't want to load + * 32-bit argument values due to displacement limits, we'll store + * this as a 64-bit value with the upper bits cleared. */ + a.embedUInt64(INT_MAX); break; } case ArgVal::Export: { auto index = value.as<ArgExport>().get(); - imports[index].patches.push_back({anchor, 0}); + imports[index].patches.push_back({anchor, 0, 0}); + a.embedUInt64(LLONG_MAX); break; } case ArgVal::FunEntry: { auto index = value.as<ArgLambda>().get(); - lambdas[index].patches.push_back({anchor, 0}); + lambdas[index].patches.push_back({anchor, 0, 0}); + a.embedUInt64(LLONG_MAX); break; } case ArgVal::Literal: { auto index = value.as<ArgLiteral>().get(); - literals[index].patches.push_back({anchor, 0}); + literals[index].patches.push_back({anchor, 0, 0}); + a.embedUInt64(LLONG_MAX); break; } default: diff --git a/erts/emulator/beam/jit/beam_jit_common.cpp b/erts/emulator/beam/jit/beam_jit_common.cpp index f27897b80b..a0bb07ce56 100644 --- a/erts/emulator/beam/jit/beam_jit_common.cpp +++ b/erts/emulator/beam/jit/beam_jit_common.cpp @@ -42,7 +42,8 @@ static std::string getAtom(Eterm atom) { return std::string((char *)ap->name, ap->len); } -BeamAssembler::BeamAssembler() : code() { +BeamAssemblerCommon::BeamAssemblerCommon(BaseAssembler &assembler_) + : assembler(assembler_), code() { /* Setup with default code info */ Error err = code.init(Environment::host()); ERTS_ASSERT(!err && "Failed to init codeHolder"); @@ -54,41 +55,32 @@ BeamAssembler::BeamAssembler() : code() { 8); ERTS_ASSERT(!err && "Failed to create .rodata section"); - err = code.attach(&a); - - ERTS_ASSERT(!err && "Failed to attach codeHolder"); #ifdef DEBUG - a.addDiagnosticOptions(DiagnosticOptions::kValidateAssembler); + assembler.addDiagnosticOptions(DiagnosticOptions::kValidateAssembler); #endif - a.addEncodingOptions(EncodingOptions::kOptimizeForSize | - EncodingOptions::kOptimizedAlign); + assembler.addEncodingOptions(EncodingOptions::kOptimizeForSize | + EncodingOptions::kOptimizedAlign); code.setErrorHandler(this); } -BeamAssembler::BeamAssembler(const std::string &log) : BeamAssembler() { - if (erts_jit_asm_dump) { - setLogger(log + ".asm"); - } -} - -BeamAssembler::~BeamAssembler() { +BeamAssemblerCommon::~BeamAssemblerCommon() { if (logger.file()) { fclose(logger.file()); } } -void *BeamAssembler::getBaseAddress() { +void *BeamAssemblerCommon::getBaseAddress() { ASSERT(code.hasBaseAddress()); return (void *)code.baseAddress(); } -size_t BeamAssembler::getOffset() { - return a.offset(); +size_t BeamAssemblerCommon::getOffset() { + return assembler.offset(); } -void BeamAssembler::codegen(JitAllocator *allocator, - const void **executable_ptr, - void **writable_ptr) { +void BeamAssemblerCommon::codegen(JitAllocator *allocator, + const void **executable_ptr, + void **writable_ptr) { Error err; err = code.flatten(); @@ -137,18 +129,18 @@ void BeamAssembler::codegen(JitAllocator *allocator, #endif } -void *BeamAssembler::getCode(Label label) { +void *BeamAssemblerCommon::getCode(Label label) { ASSERT(label.isValid()); return (char *)getBaseAddress() + code.labelOffsetFromBase(label); } -byte *BeamAssembler::getCode(char *labelName) { +byte *BeamAssemblerCommon::getCode(char *labelName) { return (byte *)getCode(code.labelByName(labelName, strlen(labelName))); } -void BeamAssembler::handleError(Error err, - const char *message, - BaseEmitter *origin) { +void BeamAssemblerCommon::handleError(Error err, + const char *message, + BaseEmitter *origin) { comment(message); if (logger.file() != NULL) { @@ -158,42 +150,42 @@ void BeamAssembler::handleError(Error err, ASSERT(0 && "Failed to encode instruction"); } -void BeamAssembler::embed_rodata(const char *labelName, - const char *buff, - size_t size) { - Label label = a.newNamedLabel(labelName); +void BeamAssemblerCommon::embed_rodata(const char *labelName, + const char *buff, + size_t size) { + Label label = assembler.newNamedLabel(labelName); - a.section(rodata); - a.bind(label); - a.embed(buff, size); - a.section(code.textSection()); + assembler.section(rodata); + assembler.bind(label); + assembler.embed(buff, size); + assembler.section(code.textSection()); } -void BeamAssembler::embed_bss(const char *labelName, size_t size) { - Label label = a.newNamedLabel(labelName); +void BeamAssemblerCommon::embed_bss(const char *labelName, size_t size) { + Label label = assembler.newNamedLabel(labelName); /* Reuse rodata section for now */ - a.section(rodata); - a.bind(label); + assembler.section(rodata); + assembler.bind(label); embed_zeros(size); - a.section(code.textSection()); + assembler.section(code.textSection()); } -void BeamAssembler::embed_zeros(size_t size) { +void BeamAssemblerCommon::embed_zeros(size_t size) { static constexpr size_t buf_size = 16384; static const char zeros[buf_size] = {}; while (size >= buf_size) { - a.embed(zeros, buf_size); + assembler.embed(zeros, buf_size); size -= buf_size; } if (size > 0) { - a.embed(zeros, size); + assembler.embed(zeros, size); } } -void BeamAssembler::setLogger(std::string log) { +void BeamAssemblerCommon::setLogger(const std::string &log) { FILE *f = fopen(log.data(), "w+"); /* FIXME: Don't crash when loading multiple modules with the same name. @@ -206,7 +198,7 @@ void BeamAssembler::setLogger(std::string log) { setLogger(f); } -void BeamAssembler::setLogger(FILE *log) { +void BeamAssemblerCommon::setLogger(FILE *log) { logger.setFile(log); logger.setIndentation(FormatIndentationGroup::kCode, 4); code.setLogger(&logger); @@ -274,7 +266,8 @@ BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *_ga, Eterm _mod, int num_labels, const BeamFile *file) - : BeamAssembler(getAtom(_mod)), beam(file), ga(_ga), mod(_mod) { + : BeamAssembler(getAtom(_mod)), BeamModuleAssemblerCommon(file, _mod), + ga(_ga) { rawLabels.reserve(num_labels + 1); if (logger.file() && beam) { @@ -497,6 +490,80 @@ const ErtsCodeInfo *BeamModuleAssembler::getOnLoad() { } } +unsigned BeamModuleAssembler::patchCatches(char *rw_base) { + unsigned catch_no = BEAM_CATCHES_NIL; + + for (const auto &c : catches) { + const auto &patch = c.patch; + ErtsCodePtr handler; + + handler = (ErtsCodePtr)getCode(c.handler); + catch_no = beam_catches_cons(handler, catch_no, nullptr); + + /* Patch the `mov` instruction with the catch tag */ + auto offset = code.labelOffsetFromBase(patch.where); + auto where = (int32_t *)&rw_base[offset + patch.ptr_offs]; + + ASSERT(INT_MAX == *where); + Eterm catch_term = make_catch(catch_no); + + /* With the current tag scheme, more than 33 million + * catches can exist at once. */ + ERTS_ASSERT(catch_term >> 31 == 0); + + *where = catch_term; + } + + return catch_no; +} + +void BeamModuleAssembler::patchImport(char *rw_base, + unsigned index, + const Export *import) { + for (const auto &patch : imports[index].patches) { + auto offset = code.labelOffsetFromBase(patch.where); + auto where = (Eterm *)&rw_base[offset + patch.ptr_offs]; + + ASSERT(LLONG_MAX == *where); + *where = reinterpret_cast<Eterm>(import) + patch.val_offs; + } +} + +void BeamModuleAssembler::patchLambda(char *rw_base, + unsigned index, + const ErlFunEntry *fe) { + for (const auto &patch : lambdas[index].patches) { + auto offset = code.labelOffsetFromBase(patch.where); + auto where = (Eterm *)&rw_base[offset + patch.ptr_offs]; + + ASSERT(LLONG_MAX == *where); + *where = reinterpret_cast<Eterm>(fe) + patch.val_offs; + } +} + +void BeamModuleAssembler::patchLiteral(char *rw_base, + unsigned index, + Eterm lit) { + for (const auto &patch : literals[index].patches) { + auto offset = code.labelOffsetFromBase(patch.where); + auto where = (Eterm *)&rw_base[offset + patch.ptr_offs]; + + ASSERT(LLONG_MAX == *where); + *where = lit + patch.val_offs; + } +} + +void BeamModuleAssembler::patchStrings(char *rw_base, + const byte *string_table) { + for (const auto &patch : strings) { + auto offset = code.labelOffsetFromBase(patch.where); + auto where = (const byte **)&rw_base[offset + patch.ptr_offs]; + + ASSERT(LLONG_MAX == (Eterm)*where); + *where = string_table + patch.val_offs; + } +} + /* ** */ #if defined(DEBUG) && defined(JIT_HARD_DEBUG) diff --git a/erts/emulator/beam/jit/beam_jit_common.hpp b/erts/emulator/beam/jit/beam_jit_common.hpp index b78ca5341d..c7b9f0ade0 100644 --- a/erts/emulator/beam/jit/beam_jit_common.hpp +++ b/erts/emulator/beam/jit/beam_jit_common.hpp @@ -21,6 +21,10 @@ #ifndef __BEAM_JIT_COMMON_HPP__ #define __BEAM_JIT_COMMON_HPP__ +#ifndef ASMJIT_ASMJIT_H_INCLUDED +# include <asmjit/asmjit.hpp> +#endif + #include <string> #include <vector> #include <unordered_map> @@ -36,102 +40,390 @@ extern "C" #include "bif.h" #include "erl_vm.h" #include "global.h" +#include "big.h" #include "beam_file.h" #include "beam_types.h" } -/* Type-safe wrapper around the definitions in beam_types.h. We've redefined it - * as an `enum class` to force the usage of our helpers, which lets us check - * for common usage errors at compile time. */ -enum class BeamTypeId : int { - None = BEAM_TYPE_NONE, - - Atom = BEAM_TYPE_ATOM, - Bitstring = BEAM_TYPE_BITSTRING, - BsMatchState = BEAM_TYPE_BS_MATCHSTATE, - Cons = BEAM_TYPE_CONS, - Float = BEAM_TYPE_FLOAT, - Fun = BEAM_TYPE_FUN, - Integer = BEAM_TYPE_INTEGER, - Map = BEAM_TYPE_MAP, - Nil = BEAM_TYPE_NIL, - Pid = BEAM_TYPE_PID, - Port = BEAM_TYPE_PORT, - Reference = BEAM_TYPE_REFERENCE, - Tuple = BEAM_TYPE_TUPLE, - - Any = BEAM_TYPE_ANY, - - Identifier = Pid | Port | Reference, - List = Cons | Nil, - Number = Float | Integer, - - /** @brief Types that can be boxed, including those that may also be - * immediates (e.g. pids, integers). */ - MaybeBoxed = Bitstring | BsMatchState | Float | Fun | Integer | Map | Pid | - Port | Reference | Tuple, - /** @brief Types that can be immediates, including those that may also be - * boxed (e.g. pids, integers). */ - MaybeImmediate = Atom | Integer | Nil | Pid | Port, - - /** @brief Types that are _always_ boxed. */ - AlwaysBoxed = MaybeBoxed & ~(Cons | MaybeImmediate), - /** @brief Types that are _always_ immediates. */ - AlwaysImmediate = MaybeImmediate & ~(Cons | MaybeBoxed), +/* On Windows, the min and max macros may be defined. */ +#undef min +#undef max + +#include "beam_jit_args.hpp" +#include "beam_jit_types.hpp" + +using namespace asmjit; + +struct AsmRange { + ErtsCodePtr start; + ErtsCodePtr stop; + const std::string name; + + struct LineData { + ErtsCodePtr start; + const std::string file; + unsigned line; + }; + + const std::vector<LineData> lines; }; -template<BeamTypeId... T> -struct BeamTypeIdUnion; +/* This is a partial class for `BeamAssembler`, containing various fields and + * helper functions that are common to all architectures so that we won't have + * to redefine them all the time. */ +class BeamAssemblerCommon : public ErrorHandler { + BaseAssembler &assembler; + +protected: + CodeHolder code; + FileLogger logger; + Section *rodata; -template<> -struct BeamTypeIdUnion<> { - static constexpr BeamTypeId value() { - return BeamTypeId::None; + static bool hasCpuFeature(uint32_t featureId); + + BeamAssemblerCommon(BaseAssembler &assembler); + ~BeamAssemblerCommon(); + + void codegen(JitAllocator *allocator, + const void **executable_ptr, + void **writable_ptr); + + void comment(const char *format) { + if (logger.file()) { + assembler.commentf("# %s", format); + } } -}; -template<BeamTypeId T, BeamTypeId... Rest> -struct BeamTypeIdUnion<T, Rest...> : BeamTypeIdUnion<Rest...> { - using integral = std::underlying_type_t<BeamTypeId>; - using super = BeamTypeIdUnion<Rest...>; - - /* Overlapping type specifications are redundant at best and a subtle error - * at worst. We've had several bugs where `Integer | MaybeBoxed` was used - * instead of `Integer | AlwaysBoxed` or similar, and erroneously drew the - * conclusion that the value is always an integer when not boxed, when it - * could also be a pid or port. */ - static constexpr bool no_overlap = - (static_cast<integral>(super::value()) & - static_cast<integral>(T)) == BEAM_TYPE_NONE; - static constexpr bool no_boxed_overlap = - no_overlap || (super::value() != BeamTypeId::MaybeBoxed && - T != BeamTypeId::MaybeBoxed); - static constexpr bool no_immed_overlap = - no_overlap || (super::value() != BeamTypeId::MaybeImmediate && - T != BeamTypeId::MaybeImmediate); - - static_assert(no_boxed_overlap, - "types must not overlap, did you mean to use " - "BeamTypeId::AlwaysBoxed here?"); - static_assert(no_immed_overlap, - "types must not overlap, did you mean to use " - "BeamTypeId::AlwaysImmediate here?"); - static_assert(no_overlap || no_boxed_overlap || no_immed_overlap, - "types must not overlap"); - - static constexpr bool is_single_typed() { - constexpr auto V = static_cast<integral>(value()); - return (static_cast<integral>(V) & (static_cast<integral>(V) - 1)) == - BEAM_TYPE_NONE; - } - - static constexpr BeamTypeId value() { - return static_cast<BeamTypeId>(static_cast<integral>(super::value()) | - static_cast<integral>(T)); + template<typename... Ts> + void comment(const char *format, Ts... args) { + if (logger.file()) { + char buff[1024]; + erts_snprintf(buff, sizeof(buff), format, args...); + assembler.commentf("# %s", buff); + } } + + void *getCode(Label label); + byte *getCode(char *labelName); + + void embed(void *data, uint32_t size) { + assembler.embed((char *)data, size); + } + + void handleError(Error err, const char *message, BaseEmitter *origin); + + void setLogger(const std::string &log); + void setLogger(FILE *log); + +public: + void embed_rodata(const char *labelName, const char *buff, size_t size); + void embed_bss(const char *labelName, size_t size); + void embed_zeros(size_t size); + + void *getBaseAddress(); + size_t getOffset(); }; -#include "beam_jit_args.hpp" +struct BeamModuleAssemblerCommon { + typedef unsigned BeamLabel; + + /* The BEAM file we've been loaded from, if any. */ + const BeamFile *beam; + Eterm mod; + + /* Map of label number to asmjit Label */ + typedef std::unordered_map<BeamLabel, const Label> LabelMap; + LabelMap rawLabels; + + struct patch { + Label where; + size_t ptr_offs; + size_t val_offs; + }; + + struct patch_catch { + struct patch patch; + Label handler; + }; + std::vector<struct patch_catch> catches; + + /* Map of import entry to patch labels and mfa */ + struct patch_import { + std::vector<struct patch> patches; + ErtsCodeMFA mfa; + }; + typedef std::unordered_map<unsigned, struct patch_import> ImportMap; + ImportMap imports; + + /* Map of fun entry to trampoline labels and patches */ + struct patch_lambda { + std::vector<struct patch> patches; + Label trampoline; + }; + typedef std::unordered_map<unsigned, struct patch_lambda> LambdaMap; + LambdaMap lambdas; + + /* Map of literals to patch labels */ + struct patch_literal { + std::vector<struct patch> patches; + }; + typedef std::unordered_map<unsigned, struct patch_literal> LiteralMap; + LiteralMap literals; + + /* All string patches */ + std::vector<struct patch> strings; + + /* All functions that have been seen so far */ + std::vector<BeamLabel> functions; + + /* Used by emit to populate the labelToMFA map */ + Label current_label; + + /* The offset of our BeamCodeHeader, if any. */ + Label code_header; + + /* The module's on_load function, if any. */ + Label on_load; + + /* The end of the last function. Note that the dispatch table, constants, + * and veneers may follow. */ + Label code_end; + + BeamModuleAssemblerCommon(const BeamFile *beam_, Eterm mod_) + : beam(beam_), mod(mod_) { + } + + /* Helpers */ + const auto &getTypeEntry(const ArgSource &arg) const { + auto typeIndex = + arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; + ASSERT(typeIndex < beam->types.count); + return static_cast<BeamArgType &>(beam->types.entries[typeIndex]); + } + + auto getTypeUnion(const ArgSource &arg) const { + if (arg.isRegister()) { + return getTypeEntry(arg).type(); + } + + Eterm constant = + arg.isImmed() + ? arg.as<ArgImmed>().get() + : beamfile_get_literal(beam, + arg.as<ArgLiteral>().get()); + + switch (tag_val_def(constant)) { + case ATOM_DEF: + return BeamTypeId::Atom; + case BINARY_DEF: + return BeamTypeId::Bitstring; + case FLOAT_DEF: + return BeamTypeId::Float; + case FUN_DEF: + return BeamTypeId::Fun; + case BIG_DEF: + case SMALL_DEF: + return BeamTypeId::Integer; + case LIST_DEF: + return BeamTypeId::Cons; + case MAP_DEF: + return BeamTypeId::Map; + case NIL_DEF: + return BeamTypeId::Nil; + case EXTERNAL_PID_DEF: + case PID_DEF: + return BeamTypeId::Pid; + case EXTERNAL_PORT_DEF: + case PORT_DEF: + return BeamTypeId::Port; + case EXTERNAL_REF_DEF: + case REF_DEF: + return BeamTypeId::Reference; + case TUPLE_DEF: + return BeamTypeId::Tuple; + default: + ERTS_ASSERT(!"tag_val_def error"); + } + } + + auto getClampedRange(const ArgSource &arg) const { + if (arg.isSmall()) { + Sint value = arg.as<ArgSmall>().getSigned(); + return std::make_pair(value, value); + } else { + const auto &entry = getTypeEntry(arg); + + auto min = entry.hasLowerBound() ? entry.min() : MIN_SMALL; + auto max = entry.hasUpperBound() ? entry.max() : MAX_SMALL; + + return std::make_pair(min, max); + } + } + + bool always_small(const ArgSource &arg) const { + if (arg.isSmall()) { + return true; + } else if (!exact_type<BeamTypeId::Integer>(arg)) { + return false; + } + + const auto &entry = getTypeEntry(arg); + + if (entry.hasLowerBound() && entry.hasUpperBound()) { + return IS_SSMALL(entry.min()) && IS_SSMALL(entry.max()); + } + + return false; + } + + bool always_immediate(const ArgSource &arg) const { + return always_one_of<BeamTypeId::AlwaysImmediate>(arg) || + always_small(arg); + } + + bool always_same_types(const ArgSource &lhs, const ArgSource &rhs) const { + using integral = std::underlying_type_t<BeamTypeId>; + auto lhs_types = static_cast<integral>(getTypeUnion(lhs)); + auto rhs_types = static_cast<integral>(getTypeUnion(rhs)); + + /* We can only be certain that the types are the same when there's + * one possible type. For example, if one is a number and the other + * is an integer, they could differ if the former is a float. */ + if ((lhs_types & (lhs_types - 1)) == 0) { + return lhs_types == rhs_types; + } + + return false; + } + + template<BeamTypeId... Types> + bool never_one_of(const ArgSource &arg) const { + using integral = std::underlying_type_t<BeamTypeId>; + auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value()); + auto type_union = static_cast<integral>(getTypeUnion(arg)); + return static_cast<BeamTypeId>(type_union & types) == BeamTypeId::None; + } + + template<BeamTypeId... Types> + bool maybe_one_of(const ArgSource &arg) const { + return !never_one_of<Types...>(arg); + } + + template<BeamTypeId... Types> + bool always_one_of(const ArgSource &arg) const { + /* Providing a single type to this function is not an error per se, but + * `exact_type` provides a bit more error checking for that use-case, + * so we want to encourage its use. */ + static_assert(!BeamTypeIdUnion<Types...>::is_single_typed(), + "always_one_of expects a union of several primitive " + "types, use exact_type instead"); + + using integral = std::underlying_type_t<BeamTypeId>; + auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value()); + auto type_union = static_cast<integral>(getTypeUnion(arg)); + return type_union == (type_union & types); + } + + template<BeamTypeId Type> + bool exact_type(const ArgSource &arg) const { + /* Rejects `exact_type<BeamTypeId::List>(...)` and similar, as it's + * almost always an error to exactly match a union of several types. + * + * On the off chance that you _do_ need to match a union exactly, use + * `masked_types<T>(arg) == T` instead. */ + static_assert(BeamTypeIdUnion<Type>::is_single_typed(), + "exact_type expects exactly one primitive type, use " + "always_one_of instead"); + + using integral = std::underlying_type_t<BeamTypeId>; + auto type_union = static_cast<integral>(getTypeUnion(arg)); + return type_union == (type_union & static_cast<integral>(Type)); + } + + template<BeamTypeId Mask> + BeamTypeId masked_types(const ArgSource &arg) const { + static_assert((Mask != BeamTypeId::AlwaysBoxed && + Mask != BeamTypeId::AlwaysImmediate), + "using masked_types with AlwaysBoxed or AlwaysImmediate " + "is almost always an error, use exact_type, " + "maybe_one_of, or never_one_of instead"); + static_assert((Mask == BeamTypeId::MaybeBoxed || + Mask == BeamTypeId::MaybeImmediate || + Mask == BeamTypeId::AlwaysBoxed || + Mask == BeamTypeId::AlwaysImmediate), + "masked_types expects a mask type like MaybeBoxed or " + "MaybeImmediate, use exact_type, maybe_one_of, or" + "never_one_of instead"); + + using integral = std::underlying_type_t<BeamTypeId>; + auto mask = static_cast<integral>(Mask); + auto type_union = static_cast<integral>(getTypeUnion(arg)); + return static_cast<BeamTypeId>(type_union & mask); + } + + bool is_sum_small_if_args_are_small(const ArgSource &LHS, + const ArgSource &RHS) const { + Sint min, max; + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + min = min1 + min2; + max = max1 + max2; + return IS_SSMALL(min) && IS_SSMALL(max); + } + + bool is_diff_small_if_args_are_small(const ArgSource &LHS, + const ArgSource &RHS) const { + Sint min, max; + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + min = min1 - max2; + max = max1 - min2; + return IS_SSMALL(min) && IS_SSMALL(max); + } + + bool is_product_small_if_args_are_small(const ArgSource &LHS, + const ArgSource &RHS) const { + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + auto mag1 = std::max(std::abs(min1), std::abs(max1)); + auto mag2 = std::max(std::abs(min2), std::abs(max2)); + + /* + * mag1 * mag2 <= MAX_SMALL + * mag1 <= MAX_SMALL / mag2 (when mag2 != 0) + */ + ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL); + return mag2 == 0 || mag1 <= MAX_SMALL / mag2; + } + + bool is_bsl_small(const ArgSource &LHS, const ArgSource &RHS) const { + if (!(always_small(LHS) && always_small(RHS))) { + return false; + } else { + auto [min1, max1] = getClampedRange(LHS); + auto [min2, max2] = getClampedRange(RHS); + + if (min1 < 0 || max1 == 0 || min2 < 0) { + return false; + } + + return max2 < asmjit::Support::clz(max1) - _TAG_IMMED1_SIZE; + } + } + + int getSizeUnit(const ArgSource &arg) const { + auto entry = getTypeEntry(arg); + return entry.hasUnit() ? entry.unit() : 1; + } + + bool hasLowerBound(const ArgSource &arg) const { + return getTypeEntry(arg).hasLowerBound(); + } + + bool hasUpperBound(const ArgSource &arg) const { + return getTypeEntry(arg).hasUpperBound(); + } +}; /* This is a view into a contiguous container (like an array or `std::vector`), * letting us reuse the existing argument array in `beamasm_emit` while keeping diff --git a/erts/emulator/beam/jit/beam_jit_main.cpp b/erts/emulator/beam/jit/beam_jit_main.cpp index 8dc457b46a..7cd8dbf2aa 100644 --- a/erts/emulator/beam/jit/beam_jit_main.cpp +++ b/erts/emulator/beam/jit/beam_jit_main.cpp @@ -364,7 +364,7 @@ void beamasm_init() { beamasm_metadata_late_init(); } -bool BeamAssembler::hasCpuFeature(uint32_t featureId) { +bool BeamAssemblerCommon::hasCpuFeature(uint32_t featureId) { return cpuinfo.hasFeature(featureId); } diff --git a/erts/emulator/beam/jit/beam_jit_metadata.cpp b/erts/emulator/beam/jit/beam_jit_metadata.cpp index 68b59babd4..780fd97c97 100644 --- a/erts/emulator/beam/jit/beam_jit_metadata.cpp +++ b/erts/emulator/beam/jit/beam_jit_metadata.cpp @@ -155,11 +155,10 @@ static void beamasm_init_late_gdb() { erts_mtx_unlock(&__jit_debug_descriptor.mutex); } -static void beamasm_update_gdb_info( - std::string module_name, - ErtsCodePtr base_address, - size_t code_size, - const std::vector<BeamAssembler::AsmRange> &ranges) { +static void beamasm_update_gdb_info(std::string module_name, + ErtsCodePtr base_address, + size_t code_size, + const std::vector<AsmRange> &ranges) { Sint symfile_size = sizeof(struct debug_info) + module_name.size() + 1; for (const auto &range : ranges) { @@ -336,7 +335,7 @@ public: return true; } - void update(const std::vector<BeamAssembler::AsmRange> &ranges) { + void update(const std::vector<AsmRange> &ranges) { struct JitCodeLoadRecord { RecordHeader header; Uint32 pid; @@ -353,7 +352,7 @@ public: record.pid = getpid(); record.tid = erts_thr_self(); - for (const BeamAssembler::AsmRange &range : ranges) { + for (const AsmRange &range : ranges) { /* Line entries must be written first, if present. */ if (!range.lines.empty()) { struct JitCodeDebugEntry { @@ -449,7 +448,7 @@ public: return true; } - void update(const std::vector<BeamAssembler::AsmRange> &ranges) { + void update(const std::vector<AsmRange> &ranges) { for (const auto &range : ranges) { char *start = (char *)range.start, *stop = (char *)range.stop; ptrdiff_t size = stop - start; @@ -491,7 +490,7 @@ public: ERTS_LOCK_FLAGS_CATEGORY_GENERIC); } - void update(const std::vector<BeamAssembler::AsmRange> &ranges) { + void update(const std::vector<AsmRange> &ranges) { if (modes) { erts_mtx_lock(&mutex); # ifdef HAVE_LINUX_PERF_DUMP_SUPPORT @@ -526,11 +525,10 @@ void beamasm_metadata_late_init() { #endif } -void beamasm_metadata_update( - std::string module_name, - ErtsCodePtr base_address, - size_t code_size, - const std::vector<BeamAssembler::AsmRange> &ranges) { +void beamasm_metadata_update(std::string module_name, + ErtsCodePtr base_address, + size_t code_size, + const std::vector<AsmRange> &ranges) { #ifdef HAVE_LINUX_PERF_SUPPORT perf.update(ranges); #endif diff --git a/erts/emulator/beam/jit/beam_jit_types.hpp b/erts/emulator/beam/jit/beam_jit_types.hpp new file mode 100644 index 0000000000..74edc66496 --- /dev/null +++ b/erts/emulator/beam/jit/beam_jit_types.hpp @@ -0,0 +1,150 @@ +/* + * %CopyrightBegin% + * + * Copyright Ericsson AB 2023. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * %CopyrightEnd% + */ + +#ifndef __BEAM_JIT_TYPES_HPP__ +#define __BEAM_JIT_TYPES_HPP__ + +/* Type-safe wrapper around the definitions in beam_types.h. We've redefined it + * as an `enum class` to force the usage of our helpers, which lets us check + * for common usage errors at compile time. */ +enum class BeamTypeId : int { + None = BEAM_TYPE_NONE, + + Atom = BEAM_TYPE_ATOM, + Bitstring = BEAM_TYPE_BITSTRING, + BsMatchState = BEAM_TYPE_BS_MATCHSTATE, + Cons = BEAM_TYPE_CONS, + Float = BEAM_TYPE_FLOAT, + Fun = BEAM_TYPE_FUN, + Integer = BEAM_TYPE_INTEGER, + Map = BEAM_TYPE_MAP, + Nil = BEAM_TYPE_NIL, + Pid = BEAM_TYPE_PID, + Port = BEAM_TYPE_PORT, + Reference = BEAM_TYPE_REFERENCE, + Tuple = BEAM_TYPE_TUPLE, + + Any = BEAM_TYPE_ANY, + + Identifier = Pid | Port | Reference, + List = Cons | Nil, + Number = Float | Integer, + + /** @brief Types that can be boxed, including those that may also be + * immediates (e.g. pids, integers). */ + MaybeBoxed = Bitstring | BsMatchState | Float | Fun | Integer | Map | Pid | + Port | Reference | Tuple, + /** @brief Types that can be immediates, including those that may also be + * boxed (e.g. pids, integers). */ + MaybeImmediate = Atom | Integer | Nil | Pid | Port, + + /** @brief Types that are _always_ boxed. */ + AlwaysBoxed = MaybeBoxed & ~(Cons | MaybeImmediate), + /** @brief Types that are _always_ immediates. */ + AlwaysImmediate = MaybeImmediate & ~(Cons | MaybeBoxed), +}; + +template<BeamTypeId... T> +struct BeamTypeIdUnion; + +template<> +struct BeamTypeIdUnion<> { + static constexpr BeamTypeId value() { + return BeamTypeId::None; + } +}; + +template<BeamTypeId T, BeamTypeId... Rest> +struct BeamTypeIdUnion<T, Rest...> : BeamTypeIdUnion<Rest...> { + using integral = std::underlying_type_t<BeamTypeId>; + using super = BeamTypeIdUnion<Rest...>; + + /* Overlapping type specifications are redundant at best and a subtle error + * at worst. We've had several bugs where `Integer | MaybeBoxed` was used + * instead of `Integer | AlwaysBoxed` or similar, and erroneously drew the + * conclusion that the value is always an integer when not boxed, when it + * could also be a pid or port. */ + static constexpr bool no_overlap = + (static_cast<integral>(super::value()) & + static_cast<integral>(T)) == BEAM_TYPE_NONE; + static constexpr bool no_boxed_overlap = + no_overlap || (super::value() != BeamTypeId::MaybeBoxed && + T != BeamTypeId::MaybeBoxed); + static constexpr bool no_immed_overlap = + no_overlap || (super::value() != BeamTypeId::MaybeImmediate && + T != BeamTypeId::MaybeImmediate); + + static_assert(no_boxed_overlap, + "types must not overlap, did you mean to use " + "BeamTypeId::AlwaysBoxed here?"); + static_assert(no_immed_overlap, + "types must not overlap, did you mean to use " + "BeamTypeId::AlwaysImmediate here?"); + static_assert(no_overlap || no_boxed_overlap || no_immed_overlap, + "types must not overlap"); + + static constexpr bool is_single_typed() { + constexpr auto V = static_cast<integral>(value()); + return (static_cast<integral>(V) & (static_cast<integral>(V) - 1)) == + BEAM_TYPE_NONE; + } + + static constexpr BeamTypeId value() { + return static_cast<BeamTypeId>(static_cast<integral>(super::value()) | + static_cast<integral>(T)); + } +}; + +struct BeamArgType : public BeamType { + BeamTypeId type() const { + return static_cast<BeamTypeId>(BeamType::type_union); + } + + bool hasLowerBound() const { + return metadata_flags & BEAM_TYPE_HAS_LOWER_BOUND; + } + + bool hasUpperBound() const { + return metadata_flags & BEAM_TYPE_HAS_UPPER_BOUND; + } + + bool hasUnit() const { + return metadata_flags & BEAM_TYPE_HAS_UNIT; + } + + auto max() const { + ASSERT(hasUpperBound()); + return BeamType::max; + } + + auto min() const { + ASSERT(hasLowerBound()); + return BeamType::min; + } + + auto unit() const { + ASSERT(hasUnit()); + return BeamType::size_unit; + } +}; + +static_assert(std::is_standard_layout<BeamArgType>::value); + +#endif diff --git a/erts/emulator/beam/jit/x86/beam_asm.hpp b/erts/emulator/beam/jit/x86/beam_asm.hpp index 2501f3a3c4..c7f085ee62 100644 --- a/erts/emulator/beam/jit/x86/beam_asm.hpp +++ b/erts/emulator/beam/jit/x86/beam_asm.hpp @@ -52,18 +52,20 @@ extern "C" using namespace asmjit; -class BeamAssembler : public ErrorHandler { -protected: - /* Holds code and relocation information. */ - CodeHolder code; - - /* TODO: Want to change this to x86::Builder in order to be able to patch - * the correct I into the code after code generation */ - x86::Assembler a; +struct BeamAssembler : public BeamAssemblerCommon { + BeamAssembler() : BeamAssemblerCommon(a) { + Error err = code.attach(&a); + ERTS_ASSERT(!err && "Failed to attach codeHolder"); + } - FileLogger logger; + BeamAssembler(const std::string &log) : BeamAssembler() { + if (erts_jit_asm_dump) { + setLogger(log + ".asm"); + } + } - Section *rodata = nullptr; +protected: + x86::Assembler a; /* * * * * * * * * */ @@ -170,27 +172,6 @@ protected: enum Distance { dShort, dLong }; -public: - static bool hasCpuFeature(uint32_t featureId); - - BeamAssembler(); - BeamAssembler(const std::string &log); - - ~BeamAssembler(); - - void *getBaseAddress(); - size_t getOffset(); - -protected: - void codegen(JitAllocator *allocator, - const void **executable_ptr, - void **writable_ptr); - - void *getCode(Label label); - byte *getCode(char *labelName); - - void handleError(Error err, const char *message, BaseEmitter *origin); - constexpr x86::Mem getRuntimeStackRef() const { int base = offsetof(ErtsSchedulerRegisters, aux_regs.d.runtime_stack); @@ -1039,113 +1020,14 @@ protected: ASSERT(count == 0); (void)count; } - -public: - void embed_rodata(const char *labelName, const char *buff, size_t size); - void embed_bss(const char *labelName, size_t size); - void embed_zeros(size_t size); - - void setLogger(std::string log); - void setLogger(FILE *log); - - void comment(const char *format) { - if (logger.file()) { - a.commentf("# %s", format); - } - } - - template<typename... Ts> - void comment(const char *format, Ts... args) { - if (logger.file()) { - char buff[1024]; - erts_snprintf(buff, sizeof(buff), format, args...); - a.commentf("# %s", buff); - } - } - - struct AsmRange { - ErtsCodePtr start; - ErtsCodePtr stop; - const std::string name; - - struct LineData { - ErtsCodePtr start; - const std::string file; - unsigned line; - }; - - const std::vector<LineData> lines; - }; }; #include "beam_asm_global.hpp" -class BeamModuleAssembler : public BeamAssembler { - typedef unsigned BeamLabel; - - /* Map of label number to asmjit Label */ - typedef std::unordered_map<BeamLabel, const Label> LabelMap; - LabelMap rawLabels; - - struct patch { - Label where; - int64_t ptr_offs; - int64_t val_offs; - }; - - struct patch_catch { - struct patch patch; - Label handler; - }; - std::vector<struct patch_catch> catches; - - /* Map of import entry to patch labels and mfa */ - struct patch_import { - std::vector<struct patch> patches; - ErtsCodeMFA mfa; - }; - typedef std::unordered_map<unsigned, struct patch_import> ImportMap; - ImportMap imports; - - /* Map of fun entry to trampoline labels and patches */ - struct patch_lambda { - std::vector<struct patch> patches; - Label trampoline; - }; - typedef std::unordered_map<unsigned, struct patch_lambda> LambdaMap; - LambdaMap lambdas; - - /* Map of literals to patch labels */ - struct patch_literal { - std::vector<struct patch> patches; - }; - typedef std::unordered_map<unsigned, struct patch_literal> LiteralMap; - LiteralMap literals; - - /* All string patches */ - std::vector<struct patch> strings; - - /* All functions that have been seen so far */ - std::vector<BeamLabel> functions; - - /* The BEAM file we've been loaded from, if any. */ - const BeamFile *beam; - +class BeamModuleAssembler : public BeamAssembler, + public BeamModuleAssemblerCommon { BeamGlobalAssembler *ga; - Label code_header; - - /* Used by emit to populate the labelToMFA map */ - Label current_label; - - /* The module's on_load function, if any. */ - Label on_load; - - /* The end of the last function. */ - Label code_end; - - Eterm mod; - /* Save the last PC for an error. */ size_t last_error_offset = 0; @@ -1302,240 +1184,6 @@ public: void patchStrings(char *rw_base, const byte *string); protected: - const auto &getTypeEntry(const ArgSource &arg) const { - auto typeIndex = - arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0; - ASSERT(typeIndex < beam->types.count); - return beam->types.entries[typeIndex]; - } - - auto getTypeUnion(const ArgSource &arg) const { - if (arg.isRegister()) { - return static_cast<BeamTypeId>(getTypeEntry(arg).type_union); - } - - Eterm constant = - arg.isImmed() - ? arg.as<ArgImmed>().get() - : beamfile_get_literal(beam, - arg.as<ArgLiteral>().get()); - - switch (tag_val_def(constant)) { - case ATOM_DEF: - return BeamTypeId::Atom; - case BINARY_DEF: - return BeamTypeId::Bitstring; - case FLOAT_DEF: - return BeamTypeId::Float; - case FUN_DEF: - return BeamTypeId::Fun; - case BIG_DEF: - case SMALL_DEF: - return BeamTypeId::Integer; - case LIST_DEF: - return BeamTypeId::Cons; - case MAP_DEF: - return BeamTypeId::Map; - case NIL_DEF: - return BeamTypeId::Nil; - case EXTERNAL_PID_DEF: - case PID_DEF: - return BeamTypeId::Pid; - case EXTERNAL_PORT_DEF: - case PORT_DEF: - return BeamTypeId::Port; - case EXTERNAL_REF_DEF: - case REF_DEF: - return BeamTypeId::Reference; - case TUPLE_DEF: - return BeamTypeId::Tuple; - default: - ERTS_ASSERT(!"tag_val_def error"); - } - } - - auto getClampedRange(const ArgSource &arg) const { - if (arg.isSmall()) { - Sint value = arg.as<ArgSmall>().getSigned(); - return std::make_pair(value, value); - } else { - const auto &entry = getTypeEntry(arg); - - if (entry.min <= entry.max) { - return std::make_pair(entry.min, entry.max); - } else if (IS_SSMALL(entry.min) && !IS_SSMALL(entry.max)) { - return std::make_pair(entry.min, MAX_SMALL); - } else if (!IS_SSMALL(entry.min) && IS_SSMALL(entry.max)) { - return std::make_pair(MIN_SMALL, entry.max); - } else { - return std::make_pair(MIN_SMALL, MAX_SMALL); - } - } - } - - int getSizeUnit(const ArgSource &arg) const { - const auto &entry = getTypeEntry(arg); - ASSERT(maybe_one_of<BeamTypeId::Bitstring>(arg)); - return entry.size_unit; - } - - bool hasLowerBound(const ArgSource &arg) const { - const auto &entry = getTypeEntry(arg); - return IS_SSMALL(entry.min) && !IS_SSMALL(entry.max); - } - - bool hasUpperBound(const ArgSource &arg) const { - const auto &entry = getTypeEntry(arg); - return !IS_SSMALL(entry.min) && IS_SSMALL(entry.max); - } - - bool always_small(const ArgSource &arg) const { - if (arg.isSmall()) { - return true; - } else if (!exact_type<BeamTypeId::Integer>(arg)) { - return false; - } - - const auto &entry = getTypeEntry(arg); - return entry.min <= entry.max; - } - - bool always_immediate(const ArgSource &arg) const { - return always_one_of<BeamTypeId::AlwaysImmediate>(arg) || - always_small(arg); - } - - bool always_same_types(const ArgSource &lhs, const ArgSource &rhs) const { - using integral = std::underlying_type_t<BeamTypeId>; - auto lhs_types = static_cast<integral>(getTypeUnion(lhs)); - auto rhs_types = static_cast<integral>(getTypeUnion(rhs)); - - /* We can only be certain that the types are the same when there's - * one possible type. For example, if one is a number and the other - * is an integer, they could differ if the former is a float. */ - if ((lhs_types & (lhs_types - 1)) == 0) { - return lhs_types == rhs_types; - } - - return false; - } - - template<BeamTypeId... Types> - bool never_one_of(const ArgSource &arg) const { - using integral = std::underlying_type_t<BeamTypeId>; - auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value()); - auto type_union = static_cast<integral>(getTypeUnion(arg)); - return static_cast<BeamTypeId>(type_union & types) == BeamTypeId::None; - } - - template<BeamTypeId... Types> - bool maybe_one_of(const ArgSource &arg) const { - return !never_one_of<Types...>(arg); - } - - template<BeamTypeId... Types> - bool always_one_of(const ArgSource &arg) const { - /* Providing a single type to this function is not an error per se, but - * `exact_type` provides a bit more error checking for that use-case, - * so we want to encourage its use. */ - static_assert(!BeamTypeIdUnion<Types...>::is_single_typed(), - "always_one_of expects a union of several primitive " - "types, use exact_type instead"); - - using integral = std::underlying_type_t<BeamTypeId>; - auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value()); - auto type_union = static_cast<integral>(getTypeUnion(arg)); - return type_union == (type_union & types); - } - - template<BeamTypeId Type> - bool exact_type(const ArgSource &arg) const { - /* Rejects `exact_type<BeamTypeId::List>(...)` and similar, as it's - * almost always an error to exactly match a union of several types. - * - * On the off chance that you _do_ need to match a union exactly, use - * `masked_types<T>(arg) == T` instead. */ - static_assert(BeamTypeIdUnion<Type>::is_single_typed(), - "exact_type expects exactly one primitive type, use " - "always_one_of instead"); - - using integral = std::underlying_type_t<BeamTypeId>; - auto type_union = static_cast<integral>(getTypeUnion(arg)); - return type_union == (type_union & static_cast<integral>(Type)); - } - - template<BeamTypeId Mask> - BeamTypeId masked_types(const ArgSource &arg) const { - static_assert((Mask != BeamTypeId::AlwaysBoxed && - Mask != BeamTypeId::AlwaysImmediate), - "using masked_types with AlwaysBoxed or AlwaysImmediate " - "is almost always an error, use exact_type, " - "maybe_one_of, or never_one_of instead"); - static_assert((Mask == BeamTypeId::MaybeBoxed || - Mask == BeamTypeId::MaybeImmediate || - Mask == BeamTypeId::AlwaysBoxed || - Mask == BeamTypeId::AlwaysImmediate), - "masked_types expects a mask type like MaybeBoxed or " - "MaybeImmediate, use exact_type, maybe_one_of, or" - "never_one_of instead"); - - using integral = std::underlying_type_t<BeamTypeId>; - auto mask = static_cast<integral>(Mask); - auto type_union = static_cast<integral>(getTypeUnion(arg)); - return static_cast<BeamTypeId>(type_union & mask); - } - - bool is_sum_small_if_args_are_small(const ArgSource &LHS, - const ArgSource &RHS) { - Sint min, max; - auto [min1, max1] = getClampedRange(LHS); - auto [min2, max2] = getClampedRange(RHS); - min = min1 + min2; - max = max1 + max2; - return IS_SSMALL(min) && IS_SSMALL(max); - } - - bool is_diff_small_if_args_are_small(const ArgSource &LHS, - const ArgSource &RHS) { - Sint min, max; - auto [min1, max1] = getClampedRange(LHS); - auto [min2, max2] = getClampedRange(RHS); - min = min1 - max2; - max = max1 - min2; - return IS_SSMALL(min) && IS_SSMALL(max); - } - - bool is_product_small_if_args_are_small(const ArgSource &LHS, - const ArgSource &RHS) { - auto [min1, max1] = getClampedRange(LHS); - auto [min2, max2] = getClampedRange(RHS); - auto mag1 = std::max(std::abs(min1), std::abs(max1)); - auto mag2 = std::max(std::abs(min2), std::abs(max2)); - - /* - * mag1 * mag2 <= MAX_SMALL - * mag1 <= MAX_SMALL / mag2 (when mag2 != 0) - */ - ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL); - return mag2 == 0 || mag1 <= MAX_SMALL / mag2; - } - - bool is_bsl_small(const ArgSource &LHS, const ArgSource &RHS) { - if (!(always_small(LHS) && always_small(RHS))) { - return false; - } else { - auto [min1, max1] = getClampedRange(LHS); - auto [min2, max2] = getClampedRange(RHS); - - if (min1 < 0 || max1 == 0 || min2 < 0) { - return false; - } - - return max2 < Support::clz(max1) - _TAG_IMMED1_SIZE; - } - } - - /* Helpers */ void emit_gc_test(const ArgWord &Stack, const ArgWord &Heap, const ArgWord &Live); @@ -1722,7 +1370,7 @@ protected: void make_move_patch(x86::Gp to, std::vector<struct patch> &patches, - int64_t offset = 0) { + size_t offset = 0) { const int MOV_IMM64_PAYLOAD_OFFSET = 2; Label lbl = a.newLabel(); @@ -1906,10 +1554,9 @@ protected: } }; -void beamasm_metadata_update( - std::string module_name, - ErtsCodePtr base_address, - size_t code_size, - const std::vector<BeamAssembler::AsmRange> &ranges); +void beamasm_metadata_update(std::string module_name, + ErtsCodePtr base_address, + size_t code_size, + const std::vector<AsmRange> &ranges); void beamasm_metadata_early_init(); void beamasm_metadata_late_init(); diff --git a/erts/emulator/beam/jit/x86/beam_asm_module.cpp b/erts/emulator/beam/jit/x86/beam_asm_module.cpp index f50398367e..bc8a11e15e 100644 --- a/erts/emulator/beam/jit/x86/beam_asm_module.cpp +++ b/erts/emulator/beam/jit/x86/beam_asm_module.cpp @@ -403,79 +403,6 @@ void BeamModuleAssembler::emit_call_error_handler() { emit_nyi("call_error_handler should never be called"); } -unsigned BeamModuleAssembler::patchCatches(char *rw_base) { - unsigned catch_no = BEAM_CATCHES_NIL; - - for (const auto &c : catches) { - const auto &patch = c.patch; - ErtsCodePtr handler; - - handler = (ErtsCodePtr)getCode(c.handler); - catch_no = beam_catches_cons(handler, catch_no, nullptr); - - /* Patch the `mov` instruction with the catch tag */ - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (unsigned *)&rw_base[offset + patch.ptr_offs]; - - ASSERT(0x7fffffff == *where); - Eterm catch_term = make_catch(catch_no); - - /* With the current tag scheme, more than 33 million - * catches can exist at once. */ - ERTS_ASSERT(catch_term >> 31 == 0); - *where = (unsigned)catch_term; - } - - return catch_no; -} - -void BeamModuleAssembler::patchImport(char *rw_base, - unsigned index, - const Export *import) { - for (const auto &patch : imports[index].patches) { - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (Eterm *)&rw_base[offset + patch.ptr_offs]; - - ASSERT(LLONG_MAX == *where); - *where = reinterpret_cast<Eterm>(import) + patch.val_offs; - } -} - -void BeamModuleAssembler::patchLambda(char *rw_base, - unsigned index, - const ErlFunEntry *fe) { - for (const auto &patch : lambdas[index].patches) { - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (Eterm *)&rw_base[offset + patch.ptr_offs]; - - ASSERT(LLONG_MAX == *where); - *where = reinterpret_cast<Eterm>(fe) + patch.val_offs; - } -} - -void BeamModuleAssembler::patchLiteral(char *rw_base, - unsigned index, - Eterm lit) { - for (const auto &patch : literals[index].patches) { - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (Eterm *)&rw_base[offset + patch.ptr_offs]; - - ASSERT(LLONG_MAX == *where); - *where = lit + patch.val_offs; - } -} - -void BeamModuleAssembler::patchStrings(char *rw_base, - const byte *string_table) { - for (const auto &patch : strings) { - auto offset = code.labelOffsetFromBase(patch.where); - auto where = (const byte **)&rw_base[offset + 2]; - - ASSERT(LLONG_MAX == (Eterm)*where); - *where = string_table + patch.val_offs; - } -} - const Label &BeamModuleAssembler::resolve_fragment(void (*fragment)()) { auto it = _dispatchTable.find(fragment); diff --git a/erts/emulator/beam/jit/x86/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp index 2293ec96fb..36e95df57c 100644 --- a/erts/emulator/beam/jit/x86/instr_bs.cpp +++ b/erts/emulator/beam/jit/x86/instr_bs.cpp @@ -1366,7 +1366,7 @@ void BeamModuleAssembler::emit_bs_get_utf8(const ArgRegister &Ctx, a.bind(multi_byte); - if (BeamAssembler::hasCpuFeature(CpuFeatures::X86::kBMI2)) { + if (hasCpuFeature(CpuFeatures::X86::kBMI2)) { /* This CPU supports the PEXT and SHRX instructions. */ safe_fragment_call(ga->get_bs_get_utf8_shared()); a.short_().jmp(check); @@ -1375,7 +1375,7 @@ void BeamModuleAssembler::emit_bs_get_utf8(const ArgRegister &Ctx, /* Take care of unaligned binaries and binaries with less than 32 * bits left. */ a.bind(fallback); - if (BeamAssembler::hasCpuFeature(CpuFeatures::X86::kBMI2)) { + if (hasCpuFeature(CpuFeatures::X86::kBMI2)) { /* This CPU supports the PEXT and SHRX instructions. */ safe_fragment_call(ga->get_bs_get_utf8_short_shared()); } else { diff --git a/erts/emulator/beam/jit/x86/instr_common.cpp b/erts/emulator/beam/jit/x86/instr_common.cpp index 19149b8e27..99e67c40b2 100644 --- a/erts/emulator/beam/jit/x86/instr_common.cpp +++ b/erts/emulator/beam/jit/x86/instr_common.cpp @@ -2307,8 +2307,7 @@ void BeamModuleAssembler::emit_catch(const ArgYRegister &CatchTag, * with the tagged catch */ a.bind(patch_addr); - a.mov(RETd, imm(0x7fffffff)); - + a.mov(RETd, imm(INT_MAX)); mov_arg(CatchTag, RET); /* Offset = 1 for `mov` payload */ |