summaryrefslogtreecommitdiff
path: root/erts
diff options
context:
space:
mode:
authorJohn Högberg <john@erlang.org>2023-04-05 15:44:43 +0200
committerGitHub <noreply@github.com>2023-04-05 15:44:43 +0200
commitc101574adf0c562650e0baf2a8d14e8ab61d91a6 (patch)
tree3d984d0539c9851a038a2316889e749013d941b8 /erts
parent977b10afa168996b2f4e426ab3dd2b153597c692 (diff)
parent64514614f394fbd3c9307fc8f6a2b69429801092 (diff)
downloaderlang-c101574adf0c562650e0baf2a8d14e8ab61d91a6.tar.gz
Merge pull request #7036 from jhogberg/john/jit/share-more-multiarch-code
jit: Share more code between architectures
Diffstat (limited to 'erts')
-rw-r--r--erts/emulator/beam/beam_file.c5
-rw-r--r--erts/emulator/beam/beam_types.c56
-rw-r--r--erts/emulator/beam/beam_types.h31
-rw-r--r--erts/emulator/beam/jit/arm/beam_asm.hpp398
-rw-r--r--erts/emulator/beam/jit/arm/beam_asm_module.cpp95
-rw-r--r--erts/emulator/beam/jit/beam_jit_common.cpp155
-rw-r--r--erts/emulator/beam/jit/beam_jit_common.hpp460
-rw-r--r--erts/emulator/beam/jit/beam_jit_main.cpp2
-rw-r--r--erts/emulator/beam/jit/beam_jit_metadata.cpp26
-rw-r--r--erts/emulator/beam/jit/beam_jit_types.hpp150
-rw-r--r--erts/emulator/beam/jit/x86/beam_asm.hpp391
-rw-r--r--erts/emulator/beam/jit/x86/beam_asm_module.cpp73
-rw-r--r--erts/emulator/beam/jit/x86/instr_bs.cpp4
-rw-r--r--erts/emulator/beam/jit/x86/instr_common.cpp3
14 files changed, 763 insertions, 1086 deletions
diff --git a/erts/emulator/beam/beam_file.c b/erts/emulator/beam/beam_file.c
index b6c23d0554..e1215e8640 100644
--- a/erts/emulator/beam/beam_file.c
+++ b/erts/emulator/beam/beam_file.c
@@ -651,14 +651,13 @@ static int parse_type_chunk_otp_26(BeamFile *beam, BeamReader *p_reader) {
for (i = 0; i < count; i++) {
const byte *type_data;
- int raw_types;
int extra;
LoadAssert(beamreader_read_bytes(p_reader, 2, &type_data));
- extra = beam_types_decode_type_otp_26(type_data, &raw_types, &types->entries[i]);
+ extra = beam_types_decode_type_otp_26(type_data, &types->entries[i]);
LoadAssert(extra >= 0);
LoadAssert(beamreader_read_bytes(p_reader, extra, &type_data));
- beam_types_decode_extra_otp_26(type_data, raw_types, &types->entries[i]);
+ beam_types_decode_extra_otp_26(type_data, &types->entries[i]);
}
/* The first entry MUST be the "any type." */
diff --git a/erts/emulator/beam/beam_types.c b/erts/emulator/beam/beam_types.c
index 659e5b0d1a..876db9f47b 100644
--- a/erts/emulator/beam/beam_types.c
+++ b/erts/emulator/beam/beam_types.c
@@ -30,50 +30,65 @@
static Sint64 get_sint64(const byte *data);
-int beam_types_decode_type_otp_26(const byte *data, int *raw_types_ptr, BeamType *out) {
- int types;
- int extra = 0;
+int beam_types_decode_type_otp_26(const byte *data, BeamType *out) {
+ int flags, extra;
- types = (Uint16)data[0] << 8 | (Uint16)data[1];
- if (types == BEAM_TYPE_NONE) {
+ flags = (Uint16)data[0] << 8 | (Uint16)data[1];
+ if (flags == BEAM_TYPE_NONE) {
return -1;
}
- /* The "extra data" aren't part of the type union proper. */
- out->type_union = types & BEAM_TYPE_ANY;
+ extra = 0;
+
+ out->type_union = flags & BEAM_TYPE_ANY;
+ out->metadata_flags = flags & BEAM_TYPE_METADATA_MASK;
+
+ if (out->metadata_flags & BEAM_TYPE_HAS_LOWER_BOUND) {
+ if (!(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER))) {
+ return -1;
+ }
- if (types & BEAM_TYPE_HAS_LOWER_BOUND) {
extra += 8;
}
- if (types & BEAM_TYPE_HAS_UPPER_BOUND) {
+ if (out->metadata_flags & BEAM_TYPE_HAS_UPPER_BOUND) {
+ if (!(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER))) {
+ return -1;
+ }
+
extra += 8;
}
- if (types & BEAM_TYPE_HAS_UNIT) {
+ if (out->metadata_flags & BEAM_TYPE_HAS_UNIT) {
+ if (!(out->type_union & BEAM_TYPE_BITSTRING)) {
+ return -1;
+ }
+
extra += 1;
}
- *raw_types_ptr = types;
return extra;
}
-void beam_types_decode_extra_otp_26(const byte *data, int types, BeamType *out) {
+void beam_types_decode_extra_otp_26(const byte *data, BeamType *out) {
out->min = MAX_SMALL + 1;
out->max = MIN_SMALL - 1;
out->size_unit = 1;
- if (types & BEAM_TYPE_HAS_LOWER_BOUND) {
+ if (out->metadata_flags & BEAM_TYPE_HAS_LOWER_BOUND) {
+ ASSERT(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER));
out->min = get_sint64(data);
data += 8;
}
- if (types & BEAM_TYPE_HAS_UPPER_BOUND) {
+ if (out->metadata_flags & BEAM_TYPE_HAS_UPPER_BOUND) {
+ ASSERT(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER));
out->max = get_sint64(data);
data += 8;
}
- if (types & BEAM_TYPE_HAS_UNIT) {
+ if (out->metadata_flags & BEAM_TYPE_HAS_UNIT) {
+ ASSERT(out->type_union & BEAM_TYPE_BITSTRING);
out->size_unit = data[0] + 1;
}
}
@@ -97,9 +112,18 @@ int beam_types_decode_otp_25(const byte *data, Uint size, BeamType *out) {
data += 8;
out->max = get_sint64(data);
- if (out->min > out->max) {
+ if (out->min <= out->max) {
+ if (!(out->type_union & (BEAM_TYPE_FLOAT | BEAM_TYPE_INTEGER))) {
+ return -1;
+ }
+
+ out->metadata_flags = (BEAM_TYPE_HAS_LOWER_BOUND |
+ BEAM_TYPE_HAS_UPPER_BOUND);
+ } else {
out->min = MAX_SMALL + 1;
out->max = MIN_SMALL - 1;
+
+ out->metadata_flags = 0;
}
out->size_unit = 1;
diff --git a/erts/emulator/beam/beam_types.h b/erts/emulator/beam/beam_types.h
index cd15f37b98..fe215a1340 100644
--- a/erts/emulator/beam/beam_types.h
+++ b/erts/emulator/beam/beam_types.h
@@ -63,32 +63,33 @@
#define BEAM_TYPE_HAS_UPPER_BOUND (1 << 14)
#define BEAM_TYPE_HAS_UNIT (1 << 15)
+#define BEAM_TYPE_METADATA_MASK (BEAM_TYPE_HAS_LOWER_BOUND | \
+ BEAM_TYPE_HAS_UPPER_BOUND | \
+ BEAM_TYPE_HAS_UNIT)
+
typedef struct {
/** @brief A set of the possible types (atom, tuple, etc) this term may
* be. When a single bit is set, the term will always be of that type. */
int type_union;
- /** @brief Minimum and maximum values. Valid if at least one of
- * IS_SSMALL(min) and IS_SSMALL(max) is true; otherwise the range is
- * unknown.
- *
- * If and only if min < max, then the value is always a small within
- * the range min trough max (including the endpoints).
- *
- * If and only if IS_SSMALL(min) && !IS_SSMALL(max) is true, then
- * the value is greater than or equal to min.
- *
- * If and only if !IS_SSMALL(min) && IS_SSMALL(max), then the
- * value is less than or equal to min. */
+ /** @brief A set of metadata presence flags, BEAM_TYPE_HAS_XYZ. */
+ int metadata_flags;
+
+ /** @brief Minimum numerical value. Only valid when the
+ * BEAM_TYPE_HAS_LOWER_BOUND metadata flag is present. */
Sint64 min;
+
+ /** @brief Maximum numerical value. Only valid when the
+ * BEAM_TYPE_HAS_UPPER_BOUND metadata flag is present. */
Sint64 max;
- /** @brief Unit for bitstring size. */
+ /** @brief Unit for bitstring size. Only valid when the BEAM_TYPE_HAS_UNIT
+ * metadata flag is present. */
byte size_unit;
} BeamType;
-int beam_types_decode_type_otp_26(const byte *data, int *raw_types_ptr, BeamType *out);
-void beam_types_decode_extra_otp_26(const byte *data, int types, BeamType *out);
+int beam_types_decode_type_otp_26(const byte *data, BeamType *out);
+void beam_types_decode_extra_otp_26(const byte *data, BeamType *out);
int beam_types_decode_otp_25(const byte *data, Uint size, BeamType *out);
#endif
diff --git a/erts/emulator/beam/jit/arm/beam_asm.hpp b/erts/emulator/beam/jit/arm/beam_asm.hpp
index a6a3f3bb98..8ef609f4b2 100644
--- a/erts/emulator/beam/jit/arm/beam_asm.hpp
+++ b/erts/emulator/beam/jit/arm/beam_asm.hpp
@@ -60,18 +60,20 @@ extern "C"
using namespace asmjit;
-class BeamAssembler : public ErrorHandler {
-protected:
- /* Holds code and relocation information. */
- CodeHolder code;
-
- a64::Assembler a;
-
- FileLogger logger;
+struct BeamAssembler : public BeamAssemblerCommon {
+ BeamAssembler() : BeamAssemblerCommon(a) {
+ Error err = code.attach(&a);
+ ERTS_ASSERT(!err && "Failed to attach codeHolder");
+ }
- Section *rodata = nullptr;
+ BeamAssembler(const std::string &log) : BeamAssembler() {
+ if (erts_jit_asm_dump) {
+ setLogger(log + ".asm");
+ }
+ }
- /* * * * * * * * * */
+protected:
+ a64::Assembler a;
/* Points at x_reg_array inside an ErtsSchedulerRegisters struct, allowing
* the aux_regs field to be addressed with an 8-bit displacement. */
@@ -224,27 +226,6 @@ protected:
kNone = 0
};
-public:
- static bool hasCpuFeature(uint32_t featureId);
-
- BeamAssembler();
- BeamAssembler(const std::string &log);
-
- ~BeamAssembler();
-
- void *getBaseAddress();
- size_t getOffset();
-
-protected:
- void codegen(JitAllocator *allocator,
- const void **executable_ptr,
- void **writable_ptr);
-
- void *getCode(Label label);
- byte *getCode(char *labelName);
-
- void handleError(Error err, const char *message, BaseEmitter *origin);
-
#ifdef JIT_HARD_DEBUG
constexpr arm::Mem getInitialSPRef() const {
int base = offsetof(ErtsSchedulerRegisters, initial_sp);
@@ -823,125 +804,19 @@ protected:
add(to, arm::GpX(mem.baseId()), offset);
}
}
-
-public:
- void embed_rodata(const char *labelName, const char *buff, size_t size);
- void embed_bss(const char *labelName, size_t size);
- void embed_zeros(size_t size);
-
- void setLogger(std::string log);
- void setLogger(FILE *log);
-
- void comment(const char *format) {
- if (logger.file()) {
- a.commentf("# %s", format);
- }
- }
-
- template<typename... Ts>
- void comment(const char *format, Ts... args) {
- if (logger.file()) {
- char buff[1024];
- erts_snprintf(buff, sizeof(buff), format, args...);
- a.commentf("# %s", buff);
- }
- }
-
- struct AsmRange {
- ErtsCodePtr start;
- ErtsCodePtr stop;
- const std::string name;
-
- struct LineData {
- ErtsCodePtr start;
- const std::string file;
- unsigned line;
- };
-
- const std::vector<LineData> lines;
- };
-
- void embed(void *data, uint32_t size) {
- a.embed((char *)data, size);
- }
};
#include "beam_asm_global.hpp"
-class BeamModuleAssembler : public BeamAssembler {
- typedef unsigned BeamLabel;
-
- /* Map of BEAM label number to asmjit Label. These should not be used
- * directly by most instructions because of displacement limits, use
- * `resolve_beam_label` instead. */
- typedef std::unordered_map<BeamLabel, const Label> LabelMap;
- LabelMap rawLabels;
+class BeamModuleAssembler : public BeamAssembler,
+ public BeamModuleAssemblerCommon {
+ BeamGlobalAssembler *ga;
/* Sequence number used to create unique named labels by
* resolve_label(). Only used when assembly output has been
* requested. */
long labelSeq = 0;
- struct patch {
- Label where;
- uint64_t val_offs;
- };
-
- struct patch_catch {
- struct patch patch;
- Label handler;
- };
- std::vector<struct patch_catch> catches;
-
- /* Map of import entry to patch labels and mfa */
- struct patch_import {
- std::vector<struct patch> patches;
- ErtsCodeMFA mfa;
- };
- typedef std::unordered_map<unsigned, struct patch_import> ImportMap;
- ImportMap imports;
-
- /* Map of fun entry to patch labels */
- struct patch_lambda {
- std::vector<struct patch> patches;
- Label trampoline;
- };
- typedef std::unordered_map<unsigned, struct patch_lambda> LambdaMap;
- LambdaMap lambdas;
-
- /* Map of literals to patch labels */
- struct patch_literal {
- std::vector<struct patch> patches;
- };
- typedef std::unordered_map<unsigned, struct patch_literal> LiteralMap;
- LiteralMap literals;
-
- /* All string patches */
- std::vector<struct patch> strings;
-
- /* All functions that have been seen so far */
- std::vector<BeamLabel> functions;
-
- /* The BEAM file we've been loaded from, if any. */
- const BeamFile *beam;
-
- BeamGlobalAssembler *ga;
-
- /* Used by emit to populate the labelToMFA map */
- Label current_label;
-
- /* The offset of our BeamCodeHeader, if any. */
- Label code_header;
-
- /* The module's on_load function, if any. */
- Label on_load;
-
- /* The end of the last function. Note that the dispatch table, constants,
- * and veneers may follow. */
- Label code_end;
-
- Eterm mod;
-
/* Save the last PC for an error. */
size_t last_error_offset = 0;
@@ -1190,240 +1065,6 @@ public:
void patchStrings(char *rw_base, const byte *string);
protected:
- const auto &getTypeEntry(const ArgSource &arg) const {
- auto typeIndex =
- arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0;
- ASSERT(typeIndex < beam->types.count);
- return beam->types.entries[typeIndex];
- }
-
- auto getTypeUnion(const ArgSource &arg) const {
- if (arg.isRegister()) {
- return static_cast<BeamTypeId>(getTypeEntry(arg).type_union);
- }
-
- Eterm constant =
- arg.isImmed()
- ? arg.as<ArgImmed>().get()
- : beamfile_get_literal(beam,
- arg.as<ArgLiteral>().get());
-
- switch (tag_val_def(constant)) {
- case ATOM_DEF:
- return BeamTypeId::Atom;
- case BINARY_DEF:
- return BeamTypeId::Bitstring;
- case FLOAT_DEF:
- return BeamTypeId::Float;
- case FUN_DEF:
- return BeamTypeId::Fun;
- case BIG_DEF:
- case SMALL_DEF:
- return BeamTypeId::Integer;
- case LIST_DEF:
- return BeamTypeId::Cons;
- case MAP_DEF:
- return BeamTypeId::Map;
- case NIL_DEF:
- return BeamTypeId::Nil;
- case EXTERNAL_PID_DEF:
- case PID_DEF:
- return BeamTypeId::Pid;
- case EXTERNAL_PORT_DEF:
- case PORT_DEF:
- return BeamTypeId::Port;
- case EXTERNAL_REF_DEF:
- case REF_DEF:
- return BeamTypeId::Reference;
- case TUPLE_DEF:
- return BeamTypeId::Tuple;
- default:
- ERTS_ASSERT(!"tag_val_def error");
- }
- }
-
- auto getClampedRange(const ArgSource &arg) const {
- if (arg.isSmall()) {
- Sint value = arg.as<ArgSmall>().getSigned();
- return std::make_pair(value, value);
- } else {
- const auto &entry = getTypeEntry(arg);
-
- if (entry.min <= entry.max) {
- return std::make_pair(entry.min, entry.max);
- } else if (IS_SSMALL(entry.min) && !IS_SSMALL(entry.max)) {
- return std::make_pair(entry.min, MAX_SMALL);
- } else if (!IS_SSMALL(entry.min) && IS_SSMALL(entry.max)) {
- return std::make_pair(MIN_SMALL, entry.max);
- } else {
- return std::make_pair(MIN_SMALL, MAX_SMALL);
- }
- }
- }
-
- int getSizeUnit(const ArgSource &arg) const {
- const auto &entry = getTypeEntry(arg);
- ASSERT(maybe_one_of<BeamTypeId::Bitstring>(arg));
- return entry.size_unit;
- }
-
- bool hasLowerBound(const ArgSource &arg) const {
- const auto &entry = getTypeEntry(arg);
- return IS_SSMALL(entry.min) && !IS_SSMALL(entry.max);
- }
-
- bool hasUpperBound(const ArgSource &arg) const {
- const auto &entry = getTypeEntry(arg);
- return !IS_SSMALL(entry.min) && IS_SSMALL(entry.max);
- }
-
- bool always_small(const ArgSource &arg) const {
- if (arg.isSmall()) {
- return true;
- } else if (!exact_type<BeamTypeId::Integer>(arg)) {
- return false;
- }
-
- const auto &entry = getTypeEntry(arg);
- return entry.min <= entry.max;
- }
-
- bool always_immediate(const ArgSource &arg) const {
- return always_one_of<BeamTypeId::AlwaysImmediate>(arg) ||
- always_small(arg);
- }
-
- bool always_same_types(const ArgSource &lhs, const ArgSource &rhs) const {
- using integral = std::underlying_type_t<BeamTypeId>;
- auto lhs_types = static_cast<integral>(getTypeUnion(lhs));
- auto rhs_types = static_cast<integral>(getTypeUnion(rhs));
-
- /* We can only be certain that the types are the same when there's
- * one possible type. For example, if one is a number and the other
- * is an integer, they could differ if the former is a float. */
- if ((lhs_types & (lhs_types - 1)) == 0) {
- return lhs_types == rhs_types;
- }
-
- return false;
- }
-
- template<BeamTypeId... Types>
- bool never_one_of(const ArgSource &arg) const {
- using integral = std::underlying_type_t<BeamTypeId>;
- auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value());
- auto type_union = static_cast<integral>(getTypeUnion(arg));
- return static_cast<BeamTypeId>(type_union & types) == BeamTypeId::None;
- }
-
- template<BeamTypeId... Types>
- bool maybe_one_of(const ArgSource &arg) const {
- return !never_one_of<Types...>(arg);
- }
-
- template<BeamTypeId... Types>
- bool always_one_of(const ArgSource &arg) const {
- /* Providing a single type to this function is not an error per se, but
- * `exact_type` provides a bit more error checking for that use-case,
- * so we want to encourage its use. */
- static_assert(!BeamTypeIdUnion<Types...>::is_single_typed(),
- "always_one_of expects a union of several primitive "
- "types, use exact_type instead");
-
- using integral = std::underlying_type_t<BeamTypeId>;
- auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value());
- auto type_union = static_cast<integral>(getTypeUnion(arg));
- return type_union == (type_union & types);
- }
-
- template<BeamTypeId Type>
- bool exact_type(const ArgSource &arg) const {
- /* Rejects `exact_type<BeamTypeId::List>(...)` and similar, as it's
- * almost always an error to exactly match a union of several types.
- *
- * On the off chance that you _do_ need to match a union exactly, use
- * `masked_types<T>(arg) == T` instead. */
- static_assert(BeamTypeIdUnion<Type>::is_single_typed(),
- "exact_type expects exactly one primitive type, use "
- "always_one_of instead");
-
- using integral = std::underlying_type_t<BeamTypeId>;
- auto type_union = static_cast<integral>(getTypeUnion(arg));
- return type_union == (type_union & static_cast<integral>(Type));
- }
-
- template<BeamTypeId Mask>
- BeamTypeId masked_types(const ArgSource &arg) const {
- static_assert((Mask != BeamTypeId::AlwaysBoxed &&
- Mask != BeamTypeId::AlwaysImmediate),
- "using masked_types with AlwaysBoxed or AlwaysImmediate "
- "is almost always an error, use exact_type, "
- "maybe_one_of, or never_one_of instead");
- static_assert((Mask == BeamTypeId::MaybeBoxed ||
- Mask == BeamTypeId::MaybeImmediate ||
- Mask == BeamTypeId::AlwaysBoxed ||
- Mask == BeamTypeId::AlwaysImmediate),
- "masked_types expects a mask type like MaybeBoxed or "
- "MaybeImmediate, use exact_type, maybe_one_of, or"
- "never_one_of instead");
-
- using integral = std::underlying_type_t<BeamTypeId>;
- auto mask = static_cast<integral>(Mask);
- auto type_union = static_cast<integral>(getTypeUnion(arg));
- return static_cast<BeamTypeId>(type_union & mask);
- }
-
- bool is_sum_small_if_args_are_small(const ArgSource &LHS,
- const ArgSource &RHS) {
- Sint min, max;
- auto [min1, max1] = getClampedRange(LHS);
- auto [min2, max2] = getClampedRange(RHS);
- min = min1 + min2;
- max = max1 + max2;
- return IS_SSMALL(min) && IS_SSMALL(max);
- }
-
- bool is_diff_small_if_args_are_small(const ArgSource &LHS,
- const ArgSource &RHS) {
- Sint min, max;
- auto [min1, max1] = getClampedRange(LHS);
- auto [min2, max2] = getClampedRange(RHS);
- min = min1 - max2;
- max = max1 - min2;
- return IS_SSMALL(min) && IS_SSMALL(max);
- }
-
- bool is_product_small_if_args_are_small(const ArgSource &LHS,
- const ArgSource &RHS) {
- auto [min1, max1] = getClampedRange(LHS);
- auto [min2, max2] = getClampedRange(RHS);
- auto mag1 = std::max(std::abs(min1), std::abs(max1));
- auto mag2 = std::max(std::abs(min2), std::abs(max2));
-
- /*
- * mag1 * mag2 <= MAX_SMALL
- * mag1 <= MAX_SMALL / mag2 (when mag2 != 0)
- */
- ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL);
- return mag2 == 0 || mag1 <= MAX_SMALL / mag2;
- }
-
- bool is_bsl_small(const ArgSource &LHS, const ArgSource &RHS) {
- if (!(always_small(LHS) && always_small(RHS))) {
- return false;
- } else {
- auto [min1, max1] = getClampedRange(LHS);
- auto [min2, max2] = getClampedRange(RHS);
-
- if (min1 < 0 || max1 == 0 || min2 < 0) {
- return false;
- }
-
- return max2 < Support::clz(max1) - _TAG_IMMED1_SIZE;
- }
- }
-
- /* Helpers */
void emit_gc_test(const ArgWord &Stack,
const ArgWord &Heap,
const ArgWord &Live);
@@ -1971,10 +1612,9 @@ protected:
}
};
-void beamasm_metadata_update(
- std::string module_name,
- ErtsCodePtr base_address,
- size_t code_size,
- const std::vector<BeamAssembler::AsmRange> &ranges);
+void beamasm_metadata_update(std::string module_name,
+ ErtsCodePtr base_address,
+ size_t code_size,
+ const std::vector<AsmRange> &ranges);
void beamasm_metadata_early_init();
void beamasm_metadata_late_init();
diff --git a/erts/emulator/beam/jit/arm/beam_asm_module.cpp b/erts/emulator/beam/jit/arm/beam_asm_module.cpp
index c368aeb47e..57f5746a8c 100644
--- a/erts/emulator/beam/jit/arm/beam_asm_module.cpp
+++ b/erts/emulator/beam/jit/arm/beam_asm_module.cpp
@@ -475,80 +475,6 @@ void BeamModuleAssembler::emit_call_error_handler() {
emit_nyi("call_error_handler should never be called");
}
-unsigned BeamModuleAssembler::patchCatches(char *rw_base) {
- unsigned catch_no = BEAM_CATCHES_NIL;
-
- for (const auto &c : catches) {
- const auto &patch = c.patch;
- ErtsCodePtr handler;
-
- handler = (ErtsCodePtr)getCode(c.handler);
- catch_no = beam_catches_cons(handler, catch_no, nullptr);
-
- /* Patch the `mov` instruction with the catch tag */
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (Eterm *)&rw_base[offset];
-
- ASSERT(LLONG_MAX == *where);
- Eterm catch_term = make_catch(catch_no);
-
- /* With the current tag scheme, more than 33 million
- * catches can exist at once. */
- ERTS_ASSERT(catch_term >> 31 == 0);
-
- *where = catch_term;
- }
-
- return catch_no;
-}
-
-void BeamModuleAssembler::patchImport(char *rw_base,
- unsigned index,
- const Export *import) {
- for (const auto &patch : imports[index].patches) {
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (Eterm *)&rw_base[offset];
-
- ASSERT(LLONG_MAX == *where);
- *where = reinterpret_cast<Eterm>(import) + patch.val_offs;
- }
-}
-
-void BeamModuleAssembler::patchLambda(char *rw_base,
- unsigned index,
- const ErlFunEntry *fe) {
- for (const auto &patch : lambdas[index].patches) {
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (Eterm *)&rw_base[offset];
-
- ASSERT(LLONG_MAX == *where);
- *where = reinterpret_cast<Eterm>(fe) + patch.val_offs;
- }
-}
-
-void BeamModuleAssembler::patchLiteral(char *rw_base,
- unsigned index,
- Eterm lit) {
- for (const auto &patch : literals[index].patches) {
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (Eterm *)&rw_base[offset];
-
- ASSERT(LLONG_MAX == *where);
- *where = lit + patch.val_offs;
- }
-}
-
-void BeamModuleAssembler::patchStrings(char *rw_base,
- const byte *string_table) {
- for (const auto &patch : strings) {
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (const byte **)&rw_base[offset];
-
- ASSERT(LLONG_MAX == (Eterm)*where);
- *where = string_table + patch.val_offs;
- }
-}
-
const Label &BeamModuleAssembler::resolve_beam_label(const ArgLabel &Lbl,
enum Displacement disp) {
ASSERT(Lbl.isLabel());
@@ -804,30 +730,37 @@ void BeamModuleAssembler::emit_constant(const Constant &constant) {
} else if (value.isLabel()) {
a.embedLabel(rawLabels.at(value.as<ArgLabel>().get()));
} else {
- a.embedUInt64(LLONG_MAX);
-
switch (value.getType()) {
case ArgVal::BytePtr:
- strings.push_back({anchor, value.as<ArgBytePtr>().get()});
+ strings.push_back({anchor, 0, value.as<ArgBytePtr>().get()});
+ a.embedUInt64(LLONG_MAX);
break;
case ArgVal::Catch: {
auto handler = rawLabels[value.as<ArgCatch>().get()];
- catches.push_back({{anchor, 0}, handler});
+ catches.push_back({{anchor, 0, 0}, handler});
+
+ /* Catches are limited to 32 bits, but since we don't want to load
+ * 32-bit argument values due to displacement limits, we'll store
+ * this as a 64-bit value with the upper bits cleared. */
+ a.embedUInt64(INT_MAX);
break;
}
case ArgVal::Export: {
auto index = value.as<ArgExport>().get();
- imports[index].patches.push_back({anchor, 0});
+ imports[index].patches.push_back({anchor, 0, 0});
+ a.embedUInt64(LLONG_MAX);
break;
}
case ArgVal::FunEntry: {
auto index = value.as<ArgLambda>().get();
- lambdas[index].patches.push_back({anchor, 0});
+ lambdas[index].patches.push_back({anchor, 0, 0});
+ a.embedUInt64(LLONG_MAX);
break;
}
case ArgVal::Literal: {
auto index = value.as<ArgLiteral>().get();
- literals[index].patches.push_back({anchor, 0});
+ literals[index].patches.push_back({anchor, 0, 0});
+ a.embedUInt64(LLONG_MAX);
break;
}
default:
diff --git a/erts/emulator/beam/jit/beam_jit_common.cpp b/erts/emulator/beam/jit/beam_jit_common.cpp
index f27897b80b..a0bb07ce56 100644
--- a/erts/emulator/beam/jit/beam_jit_common.cpp
+++ b/erts/emulator/beam/jit/beam_jit_common.cpp
@@ -42,7 +42,8 @@ static std::string getAtom(Eterm atom) {
return std::string((char *)ap->name, ap->len);
}
-BeamAssembler::BeamAssembler() : code() {
+BeamAssemblerCommon::BeamAssemblerCommon(BaseAssembler &assembler_)
+ : assembler(assembler_), code() {
/* Setup with default code info */
Error err = code.init(Environment::host());
ERTS_ASSERT(!err && "Failed to init codeHolder");
@@ -54,41 +55,32 @@ BeamAssembler::BeamAssembler() : code() {
8);
ERTS_ASSERT(!err && "Failed to create .rodata section");
- err = code.attach(&a);
-
- ERTS_ASSERT(!err && "Failed to attach codeHolder");
#ifdef DEBUG
- a.addDiagnosticOptions(DiagnosticOptions::kValidateAssembler);
+ assembler.addDiagnosticOptions(DiagnosticOptions::kValidateAssembler);
#endif
- a.addEncodingOptions(EncodingOptions::kOptimizeForSize |
- EncodingOptions::kOptimizedAlign);
+ assembler.addEncodingOptions(EncodingOptions::kOptimizeForSize |
+ EncodingOptions::kOptimizedAlign);
code.setErrorHandler(this);
}
-BeamAssembler::BeamAssembler(const std::string &log) : BeamAssembler() {
- if (erts_jit_asm_dump) {
- setLogger(log + ".asm");
- }
-}
-
-BeamAssembler::~BeamAssembler() {
+BeamAssemblerCommon::~BeamAssemblerCommon() {
if (logger.file()) {
fclose(logger.file());
}
}
-void *BeamAssembler::getBaseAddress() {
+void *BeamAssemblerCommon::getBaseAddress() {
ASSERT(code.hasBaseAddress());
return (void *)code.baseAddress();
}
-size_t BeamAssembler::getOffset() {
- return a.offset();
+size_t BeamAssemblerCommon::getOffset() {
+ return assembler.offset();
}
-void BeamAssembler::codegen(JitAllocator *allocator,
- const void **executable_ptr,
- void **writable_ptr) {
+void BeamAssemblerCommon::codegen(JitAllocator *allocator,
+ const void **executable_ptr,
+ void **writable_ptr) {
Error err;
err = code.flatten();
@@ -137,18 +129,18 @@ void BeamAssembler::codegen(JitAllocator *allocator,
#endif
}
-void *BeamAssembler::getCode(Label label) {
+void *BeamAssemblerCommon::getCode(Label label) {
ASSERT(label.isValid());
return (char *)getBaseAddress() + code.labelOffsetFromBase(label);
}
-byte *BeamAssembler::getCode(char *labelName) {
+byte *BeamAssemblerCommon::getCode(char *labelName) {
return (byte *)getCode(code.labelByName(labelName, strlen(labelName)));
}
-void BeamAssembler::handleError(Error err,
- const char *message,
- BaseEmitter *origin) {
+void BeamAssemblerCommon::handleError(Error err,
+ const char *message,
+ BaseEmitter *origin) {
comment(message);
if (logger.file() != NULL) {
@@ -158,42 +150,42 @@ void BeamAssembler::handleError(Error err,
ASSERT(0 && "Failed to encode instruction");
}
-void BeamAssembler::embed_rodata(const char *labelName,
- const char *buff,
- size_t size) {
- Label label = a.newNamedLabel(labelName);
+void BeamAssemblerCommon::embed_rodata(const char *labelName,
+ const char *buff,
+ size_t size) {
+ Label label = assembler.newNamedLabel(labelName);
- a.section(rodata);
- a.bind(label);
- a.embed(buff, size);
- a.section(code.textSection());
+ assembler.section(rodata);
+ assembler.bind(label);
+ assembler.embed(buff, size);
+ assembler.section(code.textSection());
}
-void BeamAssembler::embed_bss(const char *labelName, size_t size) {
- Label label = a.newNamedLabel(labelName);
+void BeamAssemblerCommon::embed_bss(const char *labelName, size_t size) {
+ Label label = assembler.newNamedLabel(labelName);
/* Reuse rodata section for now */
- a.section(rodata);
- a.bind(label);
+ assembler.section(rodata);
+ assembler.bind(label);
embed_zeros(size);
- a.section(code.textSection());
+ assembler.section(code.textSection());
}
-void BeamAssembler::embed_zeros(size_t size) {
+void BeamAssemblerCommon::embed_zeros(size_t size) {
static constexpr size_t buf_size = 16384;
static const char zeros[buf_size] = {};
while (size >= buf_size) {
- a.embed(zeros, buf_size);
+ assembler.embed(zeros, buf_size);
size -= buf_size;
}
if (size > 0) {
- a.embed(zeros, size);
+ assembler.embed(zeros, size);
}
}
-void BeamAssembler::setLogger(std::string log) {
+void BeamAssemblerCommon::setLogger(const std::string &log) {
FILE *f = fopen(log.data(), "w+");
/* FIXME: Don't crash when loading multiple modules with the same name.
@@ -206,7 +198,7 @@ void BeamAssembler::setLogger(std::string log) {
setLogger(f);
}
-void BeamAssembler::setLogger(FILE *log) {
+void BeamAssemblerCommon::setLogger(FILE *log) {
logger.setFile(log);
logger.setIndentation(FormatIndentationGroup::kCode, 4);
code.setLogger(&logger);
@@ -274,7 +266,8 @@ BeamModuleAssembler::BeamModuleAssembler(BeamGlobalAssembler *_ga,
Eterm _mod,
int num_labels,
const BeamFile *file)
- : BeamAssembler(getAtom(_mod)), beam(file), ga(_ga), mod(_mod) {
+ : BeamAssembler(getAtom(_mod)), BeamModuleAssemblerCommon(file, _mod),
+ ga(_ga) {
rawLabels.reserve(num_labels + 1);
if (logger.file() && beam) {
@@ -497,6 +490,80 @@ const ErtsCodeInfo *BeamModuleAssembler::getOnLoad() {
}
}
+unsigned BeamModuleAssembler::patchCatches(char *rw_base) {
+ unsigned catch_no = BEAM_CATCHES_NIL;
+
+ for (const auto &c : catches) {
+ const auto &patch = c.patch;
+ ErtsCodePtr handler;
+
+ handler = (ErtsCodePtr)getCode(c.handler);
+ catch_no = beam_catches_cons(handler, catch_no, nullptr);
+
+ /* Patch the `mov` instruction with the catch tag */
+ auto offset = code.labelOffsetFromBase(patch.where);
+ auto where = (int32_t *)&rw_base[offset + patch.ptr_offs];
+
+ ASSERT(INT_MAX == *where);
+ Eterm catch_term = make_catch(catch_no);
+
+ /* With the current tag scheme, more than 33 million
+ * catches can exist at once. */
+ ERTS_ASSERT(catch_term >> 31 == 0);
+
+ *where = catch_term;
+ }
+
+ return catch_no;
+}
+
+void BeamModuleAssembler::patchImport(char *rw_base,
+ unsigned index,
+ const Export *import) {
+ for (const auto &patch : imports[index].patches) {
+ auto offset = code.labelOffsetFromBase(patch.where);
+ auto where = (Eterm *)&rw_base[offset + patch.ptr_offs];
+
+ ASSERT(LLONG_MAX == *where);
+ *where = reinterpret_cast<Eterm>(import) + patch.val_offs;
+ }
+}
+
+void BeamModuleAssembler::patchLambda(char *rw_base,
+ unsigned index,
+ const ErlFunEntry *fe) {
+ for (const auto &patch : lambdas[index].patches) {
+ auto offset = code.labelOffsetFromBase(patch.where);
+ auto where = (Eterm *)&rw_base[offset + patch.ptr_offs];
+
+ ASSERT(LLONG_MAX == *where);
+ *where = reinterpret_cast<Eterm>(fe) + patch.val_offs;
+ }
+}
+
+void BeamModuleAssembler::patchLiteral(char *rw_base,
+ unsigned index,
+ Eterm lit) {
+ for (const auto &patch : literals[index].patches) {
+ auto offset = code.labelOffsetFromBase(patch.where);
+ auto where = (Eterm *)&rw_base[offset + patch.ptr_offs];
+
+ ASSERT(LLONG_MAX == *where);
+ *where = lit + patch.val_offs;
+ }
+}
+
+void BeamModuleAssembler::patchStrings(char *rw_base,
+ const byte *string_table) {
+ for (const auto &patch : strings) {
+ auto offset = code.labelOffsetFromBase(patch.where);
+ auto where = (const byte **)&rw_base[offset + patch.ptr_offs];
+
+ ASSERT(LLONG_MAX == (Eterm)*where);
+ *where = string_table + patch.val_offs;
+ }
+}
+
/* ** */
#if defined(DEBUG) && defined(JIT_HARD_DEBUG)
diff --git a/erts/emulator/beam/jit/beam_jit_common.hpp b/erts/emulator/beam/jit/beam_jit_common.hpp
index b78ca5341d..c7b9f0ade0 100644
--- a/erts/emulator/beam/jit/beam_jit_common.hpp
+++ b/erts/emulator/beam/jit/beam_jit_common.hpp
@@ -21,6 +21,10 @@
#ifndef __BEAM_JIT_COMMON_HPP__
#define __BEAM_JIT_COMMON_HPP__
+#ifndef ASMJIT_ASMJIT_H_INCLUDED
+# include <asmjit/asmjit.hpp>
+#endif
+
#include <string>
#include <vector>
#include <unordered_map>
@@ -36,102 +40,390 @@ extern "C"
#include "bif.h"
#include "erl_vm.h"
#include "global.h"
+#include "big.h"
#include "beam_file.h"
#include "beam_types.h"
}
-/* Type-safe wrapper around the definitions in beam_types.h. We've redefined it
- * as an `enum class` to force the usage of our helpers, which lets us check
- * for common usage errors at compile time. */
-enum class BeamTypeId : int {
- None = BEAM_TYPE_NONE,
-
- Atom = BEAM_TYPE_ATOM,
- Bitstring = BEAM_TYPE_BITSTRING,
- BsMatchState = BEAM_TYPE_BS_MATCHSTATE,
- Cons = BEAM_TYPE_CONS,
- Float = BEAM_TYPE_FLOAT,
- Fun = BEAM_TYPE_FUN,
- Integer = BEAM_TYPE_INTEGER,
- Map = BEAM_TYPE_MAP,
- Nil = BEAM_TYPE_NIL,
- Pid = BEAM_TYPE_PID,
- Port = BEAM_TYPE_PORT,
- Reference = BEAM_TYPE_REFERENCE,
- Tuple = BEAM_TYPE_TUPLE,
-
- Any = BEAM_TYPE_ANY,
-
- Identifier = Pid | Port | Reference,
- List = Cons | Nil,
- Number = Float | Integer,
-
- /** @brief Types that can be boxed, including those that may also be
- * immediates (e.g. pids, integers). */
- MaybeBoxed = Bitstring | BsMatchState | Float | Fun | Integer | Map | Pid |
- Port | Reference | Tuple,
- /** @brief Types that can be immediates, including those that may also be
- * boxed (e.g. pids, integers). */
- MaybeImmediate = Atom | Integer | Nil | Pid | Port,
-
- /** @brief Types that are _always_ boxed. */
- AlwaysBoxed = MaybeBoxed & ~(Cons | MaybeImmediate),
- /** @brief Types that are _always_ immediates. */
- AlwaysImmediate = MaybeImmediate & ~(Cons | MaybeBoxed),
+/* On Windows, the min and max macros may be defined. */
+#undef min
+#undef max
+
+#include "beam_jit_args.hpp"
+#include "beam_jit_types.hpp"
+
+using namespace asmjit;
+
+struct AsmRange {
+ ErtsCodePtr start;
+ ErtsCodePtr stop;
+ const std::string name;
+
+ struct LineData {
+ ErtsCodePtr start;
+ const std::string file;
+ unsigned line;
+ };
+
+ const std::vector<LineData> lines;
};
-template<BeamTypeId... T>
-struct BeamTypeIdUnion;
+/* This is a partial class for `BeamAssembler`, containing various fields and
+ * helper functions that are common to all architectures so that we won't have
+ * to redefine them all the time. */
+class BeamAssemblerCommon : public ErrorHandler {
+ BaseAssembler &assembler;
+
+protected:
+ CodeHolder code;
+ FileLogger logger;
+ Section *rodata;
-template<>
-struct BeamTypeIdUnion<> {
- static constexpr BeamTypeId value() {
- return BeamTypeId::None;
+ static bool hasCpuFeature(uint32_t featureId);
+
+ BeamAssemblerCommon(BaseAssembler &assembler);
+ ~BeamAssemblerCommon();
+
+ void codegen(JitAllocator *allocator,
+ const void **executable_ptr,
+ void **writable_ptr);
+
+ void comment(const char *format) {
+ if (logger.file()) {
+ assembler.commentf("# %s", format);
+ }
}
-};
-template<BeamTypeId T, BeamTypeId... Rest>
-struct BeamTypeIdUnion<T, Rest...> : BeamTypeIdUnion<Rest...> {
- using integral = std::underlying_type_t<BeamTypeId>;
- using super = BeamTypeIdUnion<Rest...>;
-
- /* Overlapping type specifications are redundant at best and a subtle error
- * at worst. We've had several bugs where `Integer | MaybeBoxed` was used
- * instead of `Integer | AlwaysBoxed` or similar, and erroneously drew the
- * conclusion that the value is always an integer when not boxed, when it
- * could also be a pid or port. */
- static constexpr bool no_overlap =
- (static_cast<integral>(super::value()) &
- static_cast<integral>(T)) == BEAM_TYPE_NONE;
- static constexpr bool no_boxed_overlap =
- no_overlap || (super::value() != BeamTypeId::MaybeBoxed &&
- T != BeamTypeId::MaybeBoxed);
- static constexpr bool no_immed_overlap =
- no_overlap || (super::value() != BeamTypeId::MaybeImmediate &&
- T != BeamTypeId::MaybeImmediate);
-
- static_assert(no_boxed_overlap,
- "types must not overlap, did you mean to use "
- "BeamTypeId::AlwaysBoxed here?");
- static_assert(no_immed_overlap,
- "types must not overlap, did you mean to use "
- "BeamTypeId::AlwaysImmediate here?");
- static_assert(no_overlap || no_boxed_overlap || no_immed_overlap,
- "types must not overlap");
-
- static constexpr bool is_single_typed() {
- constexpr auto V = static_cast<integral>(value());
- return (static_cast<integral>(V) & (static_cast<integral>(V) - 1)) ==
- BEAM_TYPE_NONE;
- }
-
- static constexpr BeamTypeId value() {
- return static_cast<BeamTypeId>(static_cast<integral>(super::value()) |
- static_cast<integral>(T));
+ template<typename... Ts>
+ void comment(const char *format, Ts... args) {
+ if (logger.file()) {
+ char buff[1024];
+ erts_snprintf(buff, sizeof(buff), format, args...);
+ assembler.commentf("# %s", buff);
+ }
}
+
+ void *getCode(Label label);
+ byte *getCode(char *labelName);
+
+ void embed(void *data, uint32_t size) {
+ assembler.embed((char *)data, size);
+ }
+
+ void handleError(Error err, const char *message, BaseEmitter *origin);
+
+ void setLogger(const std::string &log);
+ void setLogger(FILE *log);
+
+public:
+ void embed_rodata(const char *labelName, const char *buff, size_t size);
+ void embed_bss(const char *labelName, size_t size);
+ void embed_zeros(size_t size);
+
+ void *getBaseAddress();
+ size_t getOffset();
};
-#include "beam_jit_args.hpp"
+struct BeamModuleAssemblerCommon {
+ typedef unsigned BeamLabel;
+
+ /* The BEAM file we've been loaded from, if any. */
+ const BeamFile *beam;
+ Eterm mod;
+
+ /* Map of label number to asmjit Label */
+ typedef std::unordered_map<BeamLabel, const Label> LabelMap;
+ LabelMap rawLabels;
+
+ struct patch {
+ Label where;
+ size_t ptr_offs;
+ size_t val_offs;
+ };
+
+ struct patch_catch {
+ struct patch patch;
+ Label handler;
+ };
+ std::vector<struct patch_catch> catches;
+
+ /* Map of import entry to patch labels and mfa */
+ struct patch_import {
+ std::vector<struct patch> patches;
+ ErtsCodeMFA mfa;
+ };
+ typedef std::unordered_map<unsigned, struct patch_import> ImportMap;
+ ImportMap imports;
+
+ /* Map of fun entry to trampoline labels and patches */
+ struct patch_lambda {
+ std::vector<struct patch> patches;
+ Label trampoline;
+ };
+ typedef std::unordered_map<unsigned, struct patch_lambda> LambdaMap;
+ LambdaMap lambdas;
+
+ /* Map of literals to patch labels */
+ struct patch_literal {
+ std::vector<struct patch> patches;
+ };
+ typedef std::unordered_map<unsigned, struct patch_literal> LiteralMap;
+ LiteralMap literals;
+
+ /* All string patches */
+ std::vector<struct patch> strings;
+
+ /* All functions that have been seen so far */
+ std::vector<BeamLabel> functions;
+
+ /* Used by emit to populate the labelToMFA map */
+ Label current_label;
+
+ /* The offset of our BeamCodeHeader, if any. */
+ Label code_header;
+
+ /* The module's on_load function, if any. */
+ Label on_load;
+
+ /* The end of the last function. Note that the dispatch table, constants,
+ * and veneers may follow. */
+ Label code_end;
+
+ BeamModuleAssemblerCommon(const BeamFile *beam_, Eterm mod_)
+ : beam(beam_), mod(mod_) {
+ }
+
+ /* Helpers */
+ const auto &getTypeEntry(const ArgSource &arg) const {
+ auto typeIndex =
+ arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0;
+ ASSERT(typeIndex < beam->types.count);
+ return static_cast<BeamArgType &>(beam->types.entries[typeIndex]);
+ }
+
+ auto getTypeUnion(const ArgSource &arg) const {
+ if (arg.isRegister()) {
+ return getTypeEntry(arg).type();
+ }
+
+ Eterm constant =
+ arg.isImmed()
+ ? arg.as<ArgImmed>().get()
+ : beamfile_get_literal(beam,
+ arg.as<ArgLiteral>().get());
+
+ switch (tag_val_def(constant)) {
+ case ATOM_DEF:
+ return BeamTypeId::Atom;
+ case BINARY_DEF:
+ return BeamTypeId::Bitstring;
+ case FLOAT_DEF:
+ return BeamTypeId::Float;
+ case FUN_DEF:
+ return BeamTypeId::Fun;
+ case BIG_DEF:
+ case SMALL_DEF:
+ return BeamTypeId::Integer;
+ case LIST_DEF:
+ return BeamTypeId::Cons;
+ case MAP_DEF:
+ return BeamTypeId::Map;
+ case NIL_DEF:
+ return BeamTypeId::Nil;
+ case EXTERNAL_PID_DEF:
+ case PID_DEF:
+ return BeamTypeId::Pid;
+ case EXTERNAL_PORT_DEF:
+ case PORT_DEF:
+ return BeamTypeId::Port;
+ case EXTERNAL_REF_DEF:
+ case REF_DEF:
+ return BeamTypeId::Reference;
+ case TUPLE_DEF:
+ return BeamTypeId::Tuple;
+ default:
+ ERTS_ASSERT(!"tag_val_def error");
+ }
+ }
+
+ auto getClampedRange(const ArgSource &arg) const {
+ if (arg.isSmall()) {
+ Sint value = arg.as<ArgSmall>().getSigned();
+ return std::make_pair(value, value);
+ } else {
+ const auto &entry = getTypeEntry(arg);
+
+ auto min = entry.hasLowerBound() ? entry.min() : MIN_SMALL;
+ auto max = entry.hasUpperBound() ? entry.max() : MAX_SMALL;
+
+ return std::make_pair(min, max);
+ }
+ }
+
+ bool always_small(const ArgSource &arg) const {
+ if (arg.isSmall()) {
+ return true;
+ } else if (!exact_type<BeamTypeId::Integer>(arg)) {
+ return false;
+ }
+
+ const auto &entry = getTypeEntry(arg);
+
+ if (entry.hasLowerBound() && entry.hasUpperBound()) {
+ return IS_SSMALL(entry.min()) && IS_SSMALL(entry.max());
+ }
+
+ return false;
+ }
+
+ bool always_immediate(const ArgSource &arg) const {
+ return always_one_of<BeamTypeId::AlwaysImmediate>(arg) ||
+ always_small(arg);
+ }
+
+ bool always_same_types(const ArgSource &lhs, const ArgSource &rhs) const {
+ using integral = std::underlying_type_t<BeamTypeId>;
+ auto lhs_types = static_cast<integral>(getTypeUnion(lhs));
+ auto rhs_types = static_cast<integral>(getTypeUnion(rhs));
+
+ /* We can only be certain that the types are the same when there's
+ * one possible type. For example, if one is a number and the other
+ * is an integer, they could differ if the former is a float. */
+ if ((lhs_types & (lhs_types - 1)) == 0) {
+ return lhs_types == rhs_types;
+ }
+
+ return false;
+ }
+
+ template<BeamTypeId... Types>
+ bool never_one_of(const ArgSource &arg) const {
+ using integral = std::underlying_type_t<BeamTypeId>;
+ auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value());
+ auto type_union = static_cast<integral>(getTypeUnion(arg));
+ return static_cast<BeamTypeId>(type_union & types) == BeamTypeId::None;
+ }
+
+ template<BeamTypeId... Types>
+ bool maybe_one_of(const ArgSource &arg) const {
+ return !never_one_of<Types...>(arg);
+ }
+
+ template<BeamTypeId... Types>
+ bool always_one_of(const ArgSource &arg) const {
+ /* Providing a single type to this function is not an error per se, but
+ * `exact_type` provides a bit more error checking for that use-case,
+ * so we want to encourage its use. */
+ static_assert(!BeamTypeIdUnion<Types...>::is_single_typed(),
+ "always_one_of expects a union of several primitive "
+ "types, use exact_type instead");
+
+ using integral = std::underlying_type_t<BeamTypeId>;
+ auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value());
+ auto type_union = static_cast<integral>(getTypeUnion(arg));
+ return type_union == (type_union & types);
+ }
+
+ template<BeamTypeId Type>
+ bool exact_type(const ArgSource &arg) const {
+ /* Rejects `exact_type<BeamTypeId::List>(...)` and similar, as it's
+ * almost always an error to exactly match a union of several types.
+ *
+ * On the off chance that you _do_ need to match a union exactly, use
+ * `masked_types<T>(arg) == T` instead. */
+ static_assert(BeamTypeIdUnion<Type>::is_single_typed(),
+ "exact_type expects exactly one primitive type, use "
+ "always_one_of instead");
+
+ using integral = std::underlying_type_t<BeamTypeId>;
+ auto type_union = static_cast<integral>(getTypeUnion(arg));
+ return type_union == (type_union & static_cast<integral>(Type));
+ }
+
+ template<BeamTypeId Mask>
+ BeamTypeId masked_types(const ArgSource &arg) const {
+ static_assert((Mask != BeamTypeId::AlwaysBoxed &&
+ Mask != BeamTypeId::AlwaysImmediate),
+ "using masked_types with AlwaysBoxed or AlwaysImmediate "
+ "is almost always an error, use exact_type, "
+ "maybe_one_of, or never_one_of instead");
+ static_assert((Mask == BeamTypeId::MaybeBoxed ||
+ Mask == BeamTypeId::MaybeImmediate ||
+ Mask == BeamTypeId::AlwaysBoxed ||
+ Mask == BeamTypeId::AlwaysImmediate),
+ "masked_types expects a mask type like MaybeBoxed or "
+ "MaybeImmediate, use exact_type, maybe_one_of, or"
+ "never_one_of instead");
+
+ using integral = std::underlying_type_t<BeamTypeId>;
+ auto mask = static_cast<integral>(Mask);
+ auto type_union = static_cast<integral>(getTypeUnion(arg));
+ return static_cast<BeamTypeId>(type_union & mask);
+ }
+
+ bool is_sum_small_if_args_are_small(const ArgSource &LHS,
+ const ArgSource &RHS) const {
+ Sint min, max;
+ auto [min1, max1] = getClampedRange(LHS);
+ auto [min2, max2] = getClampedRange(RHS);
+ min = min1 + min2;
+ max = max1 + max2;
+ return IS_SSMALL(min) && IS_SSMALL(max);
+ }
+
+ bool is_diff_small_if_args_are_small(const ArgSource &LHS,
+ const ArgSource &RHS) const {
+ Sint min, max;
+ auto [min1, max1] = getClampedRange(LHS);
+ auto [min2, max2] = getClampedRange(RHS);
+ min = min1 - max2;
+ max = max1 - min2;
+ return IS_SSMALL(min) && IS_SSMALL(max);
+ }
+
+ bool is_product_small_if_args_are_small(const ArgSource &LHS,
+ const ArgSource &RHS) const {
+ auto [min1, max1] = getClampedRange(LHS);
+ auto [min2, max2] = getClampedRange(RHS);
+ auto mag1 = std::max(std::abs(min1), std::abs(max1));
+ auto mag2 = std::max(std::abs(min2), std::abs(max2));
+
+ /*
+ * mag1 * mag2 <= MAX_SMALL
+ * mag1 <= MAX_SMALL / mag2 (when mag2 != 0)
+ */
+ ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL);
+ return mag2 == 0 || mag1 <= MAX_SMALL / mag2;
+ }
+
+ bool is_bsl_small(const ArgSource &LHS, const ArgSource &RHS) const {
+ if (!(always_small(LHS) && always_small(RHS))) {
+ return false;
+ } else {
+ auto [min1, max1] = getClampedRange(LHS);
+ auto [min2, max2] = getClampedRange(RHS);
+
+ if (min1 < 0 || max1 == 0 || min2 < 0) {
+ return false;
+ }
+
+ return max2 < asmjit::Support::clz(max1) - _TAG_IMMED1_SIZE;
+ }
+ }
+
+ int getSizeUnit(const ArgSource &arg) const {
+ auto entry = getTypeEntry(arg);
+ return entry.hasUnit() ? entry.unit() : 1;
+ }
+
+ bool hasLowerBound(const ArgSource &arg) const {
+ return getTypeEntry(arg).hasLowerBound();
+ }
+
+ bool hasUpperBound(const ArgSource &arg) const {
+ return getTypeEntry(arg).hasUpperBound();
+ }
+};
/* This is a view into a contiguous container (like an array or `std::vector`),
* letting us reuse the existing argument array in `beamasm_emit` while keeping
diff --git a/erts/emulator/beam/jit/beam_jit_main.cpp b/erts/emulator/beam/jit/beam_jit_main.cpp
index 8dc457b46a..7cd8dbf2aa 100644
--- a/erts/emulator/beam/jit/beam_jit_main.cpp
+++ b/erts/emulator/beam/jit/beam_jit_main.cpp
@@ -364,7 +364,7 @@ void beamasm_init() {
beamasm_metadata_late_init();
}
-bool BeamAssembler::hasCpuFeature(uint32_t featureId) {
+bool BeamAssemblerCommon::hasCpuFeature(uint32_t featureId) {
return cpuinfo.hasFeature(featureId);
}
diff --git a/erts/emulator/beam/jit/beam_jit_metadata.cpp b/erts/emulator/beam/jit/beam_jit_metadata.cpp
index 68b59babd4..780fd97c97 100644
--- a/erts/emulator/beam/jit/beam_jit_metadata.cpp
+++ b/erts/emulator/beam/jit/beam_jit_metadata.cpp
@@ -155,11 +155,10 @@ static void beamasm_init_late_gdb() {
erts_mtx_unlock(&__jit_debug_descriptor.mutex);
}
-static void beamasm_update_gdb_info(
- std::string module_name,
- ErtsCodePtr base_address,
- size_t code_size,
- const std::vector<BeamAssembler::AsmRange> &ranges) {
+static void beamasm_update_gdb_info(std::string module_name,
+ ErtsCodePtr base_address,
+ size_t code_size,
+ const std::vector<AsmRange> &ranges) {
Sint symfile_size = sizeof(struct debug_info) + module_name.size() + 1;
for (const auto &range : ranges) {
@@ -336,7 +335,7 @@ public:
return true;
}
- void update(const std::vector<BeamAssembler::AsmRange> &ranges) {
+ void update(const std::vector<AsmRange> &ranges) {
struct JitCodeLoadRecord {
RecordHeader header;
Uint32 pid;
@@ -353,7 +352,7 @@ public:
record.pid = getpid();
record.tid = erts_thr_self();
- for (const BeamAssembler::AsmRange &range : ranges) {
+ for (const AsmRange &range : ranges) {
/* Line entries must be written first, if present. */
if (!range.lines.empty()) {
struct JitCodeDebugEntry {
@@ -449,7 +448,7 @@ public:
return true;
}
- void update(const std::vector<BeamAssembler::AsmRange> &ranges) {
+ void update(const std::vector<AsmRange> &ranges) {
for (const auto &range : ranges) {
char *start = (char *)range.start, *stop = (char *)range.stop;
ptrdiff_t size = stop - start;
@@ -491,7 +490,7 @@ public:
ERTS_LOCK_FLAGS_CATEGORY_GENERIC);
}
- void update(const std::vector<BeamAssembler::AsmRange> &ranges) {
+ void update(const std::vector<AsmRange> &ranges) {
if (modes) {
erts_mtx_lock(&mutex);
# ifdef HAVE_LINUX_PERF_DUMP_SUPPORT
@@ -526,11 +525,10 @@ void beamasm_metadata_late_init() {
#endif
}
-void beamasm_metadata_update(
- std::string module_name,
- ErtsCodePtr base_address,
- size_t code_size,
- const std::vector<BeamAssembler::AsmRange> &ranges) {
+void beamasm_metadata_update(std::string module_name,
+ ErtsCodePtr base_address,
+ size_t code_size,
+ const std::vector<AsmRange> &ranges) {
#ifdef HAVE_LINUX_PERF_SUPPORT
perf.update(ranges);
#endif
diff --git a/erts/emulator/beam/jit/beam_jit_types.hpp b/erts/emulator/beam/jit/beam_jit_types.hpp
new file mode 100644
index 0000000000..74edc66496
--- /dev/null
+++ b/erts/emulator/beam/jit/beam_jit_types.hpp
@@ -0,0 +1,150 @@
+/*
+ * %CopyrightBegin%
+ *
+ * Copyright Ericsson AB 2023. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * %CopyrightEnd%
+ */
+
+#ifndef __BEAM_JIT_TYPES_HPP__
+#define __BEAM_JIT_TYPES_HPP__
+
+/* Type-safe wrapper around the definitions in beam_types.h. We've redefined it
+ * as an `enum class` to force the usage of our helpers, which lets us check
+ * for common usage errors at compile time. */
+enum class BeamTypeId : int {
+ None = BEAM_TYPE_NONE,
+
+ Atom = BEAM_TYPE_ATOM,
+ Bitstring = BEAM_TYPE_BITSTRING,
+ BsMatchState = BEAM_TYPE_BS_MATCHSTATE,
+ Cons = BEAM_TYPE_CONS,
+ Float = BEAM_TYPE_FLOAT,
+ Fun = BEAM_TYPE_FUN,
+ Integer = BEAM_TYPE_INTEGER,
+ Map = BEAM_TYPE_MAP,
+ Nil = BEAM_TYPE_NIL,
+ Pid = BEAM_TYPE_PID,
+ Port = BEAM_TYPE_PORT,
+ Reference = BEAM_TYPE_REFERENCE,
+ Tuple = BEAM_TYPE_TUPLE,
+
+ Any = BEAM_TYPE_ANY,
+
+ Identifier = Pid | Port | Reference,
+ List = Cons | Nil,
+ Number = Float | Integer,
+
+ /** @brief Types that can be boxed, including those that may also be
+ * immediates (e.g. pids, integers). */
+ MaybeBoxed = Bitstring | BsMatchState | Float | Fun | Integer | Map | Pid |
+ Port | Reference | Tuple,
+ /** @brief Types that can be immediates, including those that may also be
+ * boxed (e.g. pids, integers). */
+ MaybeImmediate = Atom | Integer | Nil | Pid | Port,
+
+ /** @brief Types that are _always_ boxed. */
+ AlwaysBoxed = MaybeBoxed & ~(Cons | MaybeImmediate),
+ /** @brief Types that are _always_ immediates. */
+ AlwaysImmediate = MaybeImmediate & ~(Cons | MaybeBoxed),
+};
+
+template<BeamTypeId... T>
+struct BeamTypeIdUnion;
+
+template<>
+struct BeamTypeIdUnion<> {
+ static constexpr BeamTypeId value() {
+ return BeamTypeId::None;
+ }
+};
+
+template<BeamTypeId T, BeamTypeId... Rest>
+struct BeamTypeIdUnion<T, Rest...> : BeamTypeIdUnion<Rest...> {
+ using integral = std::underlying_type_t<BeamTypeId>;
+ using super = BeamTypeIdUnion<Rest...>;
+
+ /* Overlapping type specifications are redundant at best and a subtle error
+ * at worst. We've had several bugs where `Integer | MaybeBoxed` was used
+ * instead of `Integer | AlwaysBoxed` or similar, and erroneously drew the
+ * conclusion that the value is always an integer when not boxed, when it
+ * could also be a pid or port. */
+ static constexpr bool no_overlap =
+ (static_cast<integral>(super::value()) &
+ static_cast<integral>(T)) == BEAM_TYPE_NONE;
+ static constexpr bool no_boxed_overlap =
+ no_overlap || (super::value() != BeamTypeId::MaybeBoxed &&
+ T != BeamTypeId::MaybeBoxed);
+ static constexpr bool no_immed_overlap =
+ no_overlap || (super::value() != BeamTypeId::MaybeImmediate &&
+ T != BeamTypeId::MaybeImmediate);
+
+ static_assert(no_boxed_overlap,
+ "types must not overlap, did you mean to use "
+ "BeamTypeId::AlwaysBoxed here?");
+ static_assert(no_immed_overlap,
+ "types must not overlap, did you mean to use "
+ "BeamTypeId::AlwaysImmediate here?");
+ static_assert(no_overlap || no_boxed_overlap || no_immed_overlap,
+ "types must not overlap");
+
+ static constexpr bool is_single_typed() {
+ constexpr auto V = static_cast<integral>(value());
+ return (static_cast<integral>(V) & (static_cast<integral>(V) - 1)) ==
+ BEAM_TYPE_NONE;
+ }
+
+ static constexpr BeamTypeId value() {
+ return static_cast<BeamTypeId>(static_cast<integral>(super::value()) |
+ static_cast<integral>(T));
+ }
+};
+
+struct BeamArgType : public BeamType {
+ BeamTypeId type() const {
+ return static_cast<BeamTypeId>(BeamType::type_union);
+ }
+
+ bool hasLowerBound() const {
+ return metadata_flags & BEAM_TYPE_HAS_LOWER_BOUND;
+ }
+
+ bool hasUpperBound() const {
+ return metadata_flags & BEAM_TYPE_HAS_UPPER_BOUND;
+ }
+
+ bool hasUnit() const {
+ return metadata_flags & BEAM_TYPE_HAS_UNIT;
+ }
+
+ auto max() const {
+ ASSERT(hasUpperBound());
+ return BeamType::max;
+ }
+
+ auto min() const {
+ ASSERT(hasLowerBound());
+ return BeamType::min;
+ }
+
+ auto unit() const {
+ ASSERT(hasUnit());
+ return BeamType::size_unit;
+ }
+};
+
+static_assert(std::is_standard_layout<BeamArgType>::value);
+
+#endif
diff --git a/erts/emulator/beam/jit/x86/beam_asm.hpp b/erts/emulator/beam/jit/x86/beam_asm.hpp
index 2501f3a3c4..c7f085ee62 100644
--- a/erts/emulator/beam/jit/x86/beam_asm.hpp
+++ b/erts/emulator/beam/jit/x86/beam_asm.hpp
@@ -52,18 +52,20 @@ extern "C"
using namespace asmjit;
-class BeamAssembler : public ErrorHandler {
-protected:
- /* Holds code and relocation information. */
- CodeHolder code;
-
- /* TODO: Want to change this to x86::Builder in order to be able to patch
- * the correct I into the code after code generation */
- x86::Assembler a;
+struct BeamAssembler : public BeamAssemblerCommon {
+ BeamAssembler() : BeamAssemblerCommon(a) {
+ Error err = code.attach(&a);
+ ERTS_ASSERT(!err && "Failed to attach codeHolder");
+ }
- FileLogger logger;
+ BeamAssembler(const std::string &log) : BeamAssembler() {
+ if (erts_jit_asm_dump) {
+ setLogger(log + ".asm");
+ }
+ }
- Section *rodata = nullptr;
+protected:
+ x86::Assembler a;
/* * * * * * * * * */
@@ -170,27 +172,6 @@ protected:
enum Distance { dShort, dLong };
-public:
- static bool hasCpuFeature(uint32_t featureId);
-
- BeamAssembler();
- BeamAssembler(const std::string &log);
-
- ~BeamAssembler();
-
- void *getBaseAddress();
- size_t getOffset();
-
-protected:
- void codegen(JitAllocator *allocator,
- const void **executable_ptr,
- void **writable_ptr);
-
- void *getCode(Label label);
- byte *getCode(char *labelName);
-
- void handleError(Error err, const char *message, BaseEmitter *origin);
-
constexpr x86::Mem getRuntimeStackRef() const {
int base = offsetof(ErtsSchedulerRegisters, aux_regs.d.runtime_stack);
@@ -1039,113 +1020,14 @@ protected:
ASSERT(count == 0);
(void)count;
}
-
-public:
- void embed_rodata(const char *labelName, const char *buff, size_t size);
- void embed_bss(const char *labelName, size_t size);
- void embed_zeros(size_t size);
-
- void setLogger(std::string log);
- void setLogger(FILE *log);
-
- void comment(const char *format) {
- if (logger.file()) {
- a.commentf("# %s", format);
- }
- }
-
- template<typename... Ts>
- void comment(const char *format, Ts... args) {
- if (logger.file()) {
- char buff[1024];
- erts_snprintf(buff, sizeof(buff), format, args...);
- a.commentf("# %s", buff);
- }
- }
-
- struct AsmRange {
- ErtsCodePtr start;
- ErtsCodePtr stop;
- const std::string name;
-
- struct LineData {
- ErtsCodePtr start;
- const std::string file;
- unsigned line;
- };
-
- const std::vector<LineData> lines;
- };
};
#include "beam_asm_global.hpp"
-class BeamModuleAssembler : public BeamAssembler {
- typedef unsigned BeamLabel;
-
- /* Map of label number to asmjit Label */
- typedef std::unordered_map<BeamLabel, const Label> LabelMap;
- LabelMap rawLabels;
-
- struct patch {
- Label where;
- int64_t ptr_offs;
- int64_t val_offs;
- };
-
- struct patch_catch {
- struct patch patch;
- Label handler;
- };
- std::vector<struct patch_catch> catches;
-
- /* Map of import entry to patch labels and mfa */
- struct patch_import {
- std::vector<struct patch> patches;
- ErtsCodeMFA mfa;
- };
- typedef std::unordered_map<unsigned, struct patch_import> ImportMap;
- ImportMap imports;
-
- /* Map of fun entry to trampoline labels and patches */
- struct patch_lambda {
- std::vector<struct patch> patches;
- Label trampoline;
- };
- typedef std::unordered_map<unsigned, struct patch_lambda> LambdaMap;
- LambdaMap lambdas;
-
- /* Map of literals to patch labels */
- struct patch_literal {
- std::vector<struct patch> patches;
- };
- typedef std::unordered_map<unsigned, struct patch_literal> LiteralMap;
- LiteralMap literals;
-
- /* All string patches */
- std::vector<struct patch> strings;
-
- /* All functions that have been seen so far */
- std::vector<BeamLabel> functions;
-
- /* The BEAM file we've been loaded from, if any. */
- const BeamFile *beam;
-
+class BeamModuleAssembler : public BeamAssembler,
+ public BeamModuleAssemblerCommon {
BeamGlobalAssembler *ga;
- Label code_header;
-
- /* Used by emit to populate the labelToMFA map */
- Label current_label;
-
- /* The module's on_load function, if any. */
- Label on_load;
-
- /* The end of the last function. */
- Label code_end;
-
- Eterm mod;
-
/* Save the last PC for an error. */
size_t last_error_offset = 0;
@@ -1302,240 +1184,6 @@ public:
void patchStrings(char *rw_base, const byte *string);
protected:
- const auto &getTypeEntry(const ArgSource &arg) const {
- auto typeIndex =
- arg.isRegister() ? arg.as<ArgRegister>().typeIndex() : 0;
- ASSERT(typeIndex < beam->types.count);
- return beam->types.entries[typeIndex];
- }
-
- auto getTypeUnion(const ArgSource &arg) const {
- if (arg.isRegister()) {
- return static_cast<BeamTypeId>(getTypeEntry(arg).type_union);
- }
-
- Eterm constant =
- arg.isImmed()
- ? arg.as<ArgImmed>().get()
- : beamfile_get_literal(beam,
- arg.as<ArgLiteral>().get());
-
- switch (tag_val_def(constant)) {
- case ATOM_DEF:
- return BeamTypeId::Atom;
- case BINARY_DEF:
- return BeamTypeId::Bitstring;
- case FLOAT_DEF:
- return BeamTypeId::Float;
- case FUN_DEF:
- return BeamTypeId::Fun;
- case BIG_DEF:
- case SMALL_DEF:
- return BeamTypeId::Integer;
- case LIST_DEF:
- return BeamTypeId::Cons;
- case MAP_DEF:
- return BeamTypeId::Map;
- case NIL_DEF:
- return BeamTypeId::Nil;
- case EXTERNAL_PID_DEF:
- case PID_DEF:
- return BeamTypeId::Pid;
- case EXTERNAL_PORT_DEF:
- case PORT_DEF:
- return BeamTypeId::Port;
- case EXTERNAL_REF_DEF:
- case REF_DEF:
- return BeamTypeId::Reference;
- case TUPLE_DEF:
- return BeamTypeId::Tuple;
- default:
- ERTS_ASSERT(!"tag_val_def error");
- }
- }
-
- auto getClampedRange(const ArgSource &arg) const {
- if (arg.isSmall()) {
- Sint value = arg.as<ArgSmall>().getSigned();
- return std::make_pair(value, value);
- } else {
- const auto &entry = getTypeEntry(arg);
-
- if (entry.min <= entry.max) {
- return std::make_pair(entry.min, entry.max);
- } else if (IS_SSMALL(entry.min) && !IS_SSMALL(entry.max)) {
- return std::make_pair(entry.min, MAX_SMALL);
- } else if (!IS_SSMALL(entry.min) && IS_SSMALL(entry.max)) {
- return std::make_pair(MIN_SMALL, entry.max);
- } else {
- return std::make_pair(MIN_SMALL, MAX_SMALL);
- }
- }
- }
-
- int getSizeUnit(const ArgSource &arg) const {
- const auto &entry = getTypeEntry(arg);
- ASSERT(maybe_one_of<BeamTypeId::Bitstring>(arg));
- return entry.size_unit;
- }
-
- bool hasLowerBound(const ArgSource &arg) const {
- const auto &entry = getTypeEntry(arg);
- return IS_SSMALL(entry.min) && !IS_SSMALL(entry.max);
- }
-
- bool hasUpperBound(const ArgSource &arg) const {
- const auto &entry = getTypeEntry(arg);
- return !IS_SSMALL(entry.min) && IS_SSMALL(entry.max);
- }
-
- bool always_small(const ArgSource &arg) const {
- if (arg.isSmall()) {
- return true;
- } else if (!exact_type<BeamTypeId::Integer>(arg)) {
- return false;
- }
-
- const auto &entry = getTypeEntry(arg);
- return entry.min <= entry.max;
- }
-
- bool always_immediate(const ArgSource &arg) const {
- return always_one_of<BeamTypeId::AlwaysImmediate>(arg) ||
- always_small(arg);
- }
-
- bool always_same_types(const ArgSource &lhs, const ArgSource &rhs) const {
- using integral = std::underlying_type_t<BeamTypeId>;
- auto lhs_types = static_cast<integral>(getTypeUnion(lhs));
- auto rhs_types = static_cast<integral>(getTypeUnion(rhs));
-
- /* We can only be certain that the types are the same when there's
- * one possible type. For example, if one is a number and the other
- * is an integer, they could differ if the former is a float. */
- if ((lhs_types & (lhs_types - 1)) == 0) {
- return lhs_types == rhs_types;
- }
-
- return false;
- }
-
- template<BeamTypeId... Types>
- bool never_one_of(const ArgSource &arg) const {
- using integral = std::underlying_type_t<BeamTypeId>;
- auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value());
- auto type_union = static_cast<integral>(getTypeUnion(arg));
- return static_cast<BeamTypeId>(type_union & types) == BeamTypeId::None;
- }
-
- template<BeamTypeId... Types>
- bool maybe_one_of(const ArgSource &arg) const {
- return !never_one_of<Types...>(arg);
- }
-
- template<BeamTypeId... Types>
- bool always_one_of(const ArgSource &arg) const {
- /* Providing a single type to this function is not an error per se, but
- * `exact_type` provides a bit more error checking for that use-case,
- * so we want to encourage its use. */
- static_assert(!BeamTypeIdUnion<Types...>::is_single_typed(),
- "always_one_of expects a union of several primitive "
- "types, use exact_type instead");
-
- using integral = std::underlying_type_t<BeamTypeId>;
- auto types = static_cast<integral>(BeamTypeIdUnion<Types...>::value());
- auto type_union = static_cast<integral>(getTypeUnion(arg));
- return type_union == (type_union & types);
- }
-
- template<BeamTypeId Type>
- bool exact_type(const ArgSource &arg) const {
- /* Rejects `exact_type<BeamTypeId::List>(...)` and similar, as it's
- * almost always an error to exactly match a union of several types.
- *
- * On the off chance that you _do_ need to match a union exactly, use
- * `masked_types<T>(arg) == T` instead. */
- static_assert(BeamTypeIdUnion<Type>::is_single_typed(),
- "exact_type expects exactly one primitive type, use "
- "always_one_of instead");
-
- using integral = std::underlying_type_t<BeamTypeId>;
- auto type_union = static_cast<integral>(getTypeUnion(arg));
- return type_union == (type_union & static_cast<integral>(Type));
- }
-
- template<BeamTypeId Mask>
- BeamTypeId masked_types(const ArgSource &arg) const {
- static_assert((Mask != BeamTypeId::AlwaysBoxed &&
- Mask != BeamTypeId::AlwaysImmediate),
- "using masked_types with AlwaysBoxed or AlwaysImmediate "
- "is almost always an error, use exact_type, "
- "maybe_one_of, or never_one_of instead");
- static_assert((Mask == BeamTypeId::MaybeBoxed ||
- Mask == BeamTypeId::MaybeImmediate ||
- Mask == BeamTypeId::AlwaysBoxed ||
- Mask == BeamTypeId::AlwaysImmediate),
- "masked_types expects a mask type like MaybeBoxed or "
- "MaybeImmediate, use exact_type, maybe_one_of, or"
- "never_one_of instead");
-
- using integral = std::underlying_type_t<BeamTypeId>;
- auto mask = static_cast<integral>(Mask);
- auto type_union = static_cast<integral>(getTypeUnion(arg));
- return static_cast<BeamTypeId>(type_union & mask);
- }
-
- bool is_sum_small_if_args_are_small(const ArgSource &LHS,
- const ArgSource &RHS) {
- Sint min, max;
- auto [min1, max1] = getClampedRange(LHS);
- auto [min2, max2] = getClampedRange(RHS);
- min = min1 + min2;
- max = max1 + max2;
- return IS_SSMALL(min) && IS_SSMALL(max);
- }
-
- bool is_diff_small_if_args_are_small(const ArgSource &LHS,
- const ArgSource &RHS) {
- Sint min, max;
- auto [min1, max1] = getClampedRange(LHS);
- auto [min2, max2] = getClampedRange(RHS);
- min = min1 - max2;
- max = max1 - min2;
- return IS_SSMALL(min) && IS_SSMALL(max);
- }
-
- bool is_product_small_if_args_are_small(const ArgSource &LHS,
- const ArgSource &RHS) {
- auto [min1, max1] = getClampedRange(LHS);
- auto [min2, max2] = getClampedRange(RHS);
- auto mag1 = std::max(std::abs(min1), std::abs(max1));
- auto mag2 = std::max(std::abs(min2), std::abs(max2));
-
- /*
- * mag1 * mag2 <= MAX_SMALL
- * mag1 <= MAX_SMALL / mag2 (when mag2 != 0)
- */
- ERTS_CT_ASSERT(MAX_SMALL < -MIN_SMALL);
- return mag2 == 0 || mag1 <= MAX_SMALL / mag2;
- }
-
- bool is_bsl_small(const ArgSource &LHS, const ArgSource &RHS) {
- if (!(always_small(LHS) && always_small(RHS))) {
- return false;
- } else {
- auto [min1, max1] = getClampedRange(LHS);
- auto [min2, max2] = getClampedRange(RHS);
-
- if (min1 < 0 || max1 == 0 || min2 < 0) {
- return false;
- }
-
- return max2 < Support::clz(max1) - _TAG_IMMED1_SIZE;
- }
- }
-
- /* Helpers */
void emit_gc_test(const ArgWord &Stack,
const ArgWord &Heap,
const ArgWord &Live);
@@ -1722,7 +1370,7 @@ protected:
void make_move_patch(x86::Gp to,
std::vector<struct patch> &patches,
- int64_t offset = 0) {
+ size_t offset = 0) {
const int MOV_IMM64_PAYLOAD_OFFSET = 2;
Label lbl = a.newLabel();
@@ -1906,10 +1554,9 @@ protected:
}
};
-void beamasm_metadata_update(
- std::string module_name,
- ErtsCodePtr base_address,
- size_t code_size,
- const std::vector<BeamAssembler::AsmRange> &ranges);
+void beamasm_metadata_update(std::string module_name,
+ ErtsCodePtr base_address,
+ size_t code_size,
+ const std::vector<AsmRange> &ranges);
void beamasm_metadata_early_init();
void beamasm_metadata_late_init();
diff --git a/erts/emulator/beam/jit/x86/beam_asm_module.cpp b/erts/emulator/beam/jit/x86/beam_asm_module.cpp
index f50398367e..bc8a11e15e 100644
--- a/erts/emulator/beam/jit/x86/beam_asm_module.cpp
+++ b/erts/emulator/beam/jit/x86/beam_asm_module.cpp
@@ -403,79 +403,6 @@ void BeamModuleAssembler::emit_call_error_handler() {
emit_nyi("call_error_handler should never be called");
}
-unsigned BeamModuleAssembler::patchCatches(char *rw_base) {
- unsigned catch_no = BEAM_CATCHES_NIL;
-
- for (const auto &c : catches) {
- const auto &patch = c.patch;
- ErtsCodePtr handler;
-
- handler = (ErtsCodePtr)getCode(c.handler);
- catch_no = beam_catches_cons(handler, catch_no, nullptr);
-
- /* Patch the `mov` instruction with the catch tag */
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (unsigned *)&rw_base[offset + patch.ptr_offs];
-
- ASSERT(0x7fffffff == *where);
- Eterm catch_term = make_catch(catch_no);
-
- /* With the current tag scheme, more than 33 million
- * catches can exist at once. */
- ERTS_ASSERT(catch_term >> 31 == 0);
- *where = (unsigned)catch_term;
- }
-
- return catch_no;
-}
-
-void BeamModuleAssembler::patchImport(char *rw_base,
- unsigned index,
- const Export *import) {
- for (const auto &patch : imports[index].patches) {
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (Eterm *)&rw_base[offset + patch.ptr_offs];
-
- ASSERT(LLONG_MAX == *where);
- *where = reinterpret_cast<Eterm>(import) + patch.val_offs;
- }
-}
-
-void BeamModuleAssembler::patchLambda(char *rw_base,
- unsigned index,
- const ErlFunEntry *fe) {
- for (const auto &patch : lambdas[index].patches) {
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (Eterm *)&rw_base[offset + patch.ptr_offs];
-
- ASSERT(LLONG_MAX == *where);
- *where = reinterpret_cast<Eterm>(fe) + patch.val_offs;
- }
-}
-
-void BeamModuleAssembler::patchLiteral(char *rw_base,
- unsigned index,
- Eterm lit) {
- for (const auto &patch : literals[index].patches) {
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (Eterm *)&rw_base[offset + patch.ptr_offs];
-
- ASSERT(LLONG_MAX == *where);
- *where = lit + patch.val_offs;
- }
-}
-
-void BeamModuleAssembler::patchStrings(char *rw_base,
- const byte *string_table) {
- for (const auto &patch : strings) {
- auto offset = code.labelOffsetFromBase(patch.where);
- auto where = (const byte **)&rw_base[offset + 2];
-
- ASSERT(LLONG_MAX == (Eterm)*where);
- *where = string_table + patch.val_offs;
- }
-}
-
const Label &BeamModuleAssembler::resolve_fragment(void (*fragment)()) {
auto it = _dispatchTable.find(fragment);
diff --git a/erts/emulator/beam/jit/x86/instr_bs.cpp b/erts/emulator/beam/jit/x86/instr_bs.cpp
index 2293ec96fb..36e95df57c 100644
--- a/erts/emulator/beam/jit/x86/instr_bs.cpp
+++ b/erts/emulator/beam/jit/x86/instr_bs.cpp
@@ -1366,7 +1366,7 @@ void BeamModuleAssembler::emit_bs_get_utf8(const ArgRegister &Ctx,
a.bind(multi_byte);
- if (BeamAssembler::hasCpuFeature(CpuFeatures::X86::kBMI2)) {
+ if (hasCpuFeature(CpuFeatures::X86::kBMI2)) {
/* This CPU supports the PEXT and SHRX instructions. */
safe_fragment_call(ga->get_bs_get_utf8_shared());
a.short_().jmp(check);
@@ -1375,7 +1375,7 @@ void BeamModuleAssembler::emit_bs_get_utf8(const ArgRegister &Ctx,
/* Take care of unaligned binaries and binaries with less than 32
* bits left. */
a.bind(fallback);
- if (BeamAssembler::hasCpuFeature(CpuFeatures::X86::kBMI2)) {
+ if (hasCpuFeature(CpuFeatures::X86::kBMI2)) {
/* This CPU supports the PEXT and SHRX instructions. */
safe_fragment_call(ga->get_bs_get_utf8_short_shared());
} else {
diff --git a/erts/emulator/beam/jit/x86/instr_common.cpp b/erts/emulator/beam/jit/x86/instr_common.cpp
index 19149b8e27..99e67c40b2 100644
--- a/erts/emulator/beam/jit/x86/instr_common.cpp
+++ b/erts/emulator/beam/jit/x86/instr_common.cpp
@@ -2307,8 +2307,7 @@ void BeamModuleAssembler::emit_catch(const ArgYRegister &CatchTag,
* with the tagged catch
*/
a.bind(patch_addr);
- a.mov(RETd, imm(0x7fffffff));
-
+ a.mov(RETd, imm(INT_MAX));
mov_arg(CatchTag, RET);
/* Offset = 1 for `mov` payload */