diff options
Diffstat (limited to 'erts/emulator/beam/jit/x86/ops.tab')
| -rw-r--r-- | erts/emulator/beam/jit/x86/ops.tab | 1349 |
1 files changed, 1349 insertions, 0 deletions
diff --git a/erts/emulator/beam/jit/x86/ops.tab b/erts/emulator/beam/jit/x86/ops.tab new file mode 100644 index 0000000000..e33cc110a5 --- /dev/null +++ b/erts/emulator/beam/jit/x86/ops.tab @@ -0,0 +1,1349 @@ +# +# %CopyrightBegin% +# +# Copyright Ericsson AB 1997-2019. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# %CopyrightEnd% +# + +# +# Types that should never be used in specific operations. +# + +FORBIDDEN_TYPES=hQ + +# +# The instructions that follows are only known by the loader and the emulator. +# They can be changed without recompiling old Beam files. +# +# Instructions starting with a "i_" prefix are instructions produced by +# instruction transformations; thus, they never occur in BEAM files. +# + +# The too_old_compiler/0 instruction is specially handled in beam_load.c +# to produce a user-friendly message informing the user that the module +# needs to be re-compiled with a modern compiler. + +too_old_compiler/0 +too_old_compiler | never() => + +# In R9C and earlier, the loader used to insert special instructions inside +# the module_info/0,1 functions. (In R10B and later, the compiler inserts +# an explicit call to an undocumented BIF, so that no loader trickery is +# necessary.) Since the instructions don't work correctly in R12B, simply +# refuse to load the module. + +func_info M=a a==am_module_info A=u==0 | label L | move n x==0 => too_old_compiler +func_info M=a a==am_module_info A=u==1 | label L | move n x==0 => too_old_compiler + +# The undocumented and unsupported guard BIF is_constant/1 was removed +# in R13. The is_constant/2 operation is marked as obsolete in genop.tab, +# so the loader will automatically generate a too_old_compiler message +# it is used, but we need to handle the is_constant/1 BIF specially here. + +bif1 Fail u$func:erlang:is_constant/1 Src Dst => too_old_compiler + +# Since the constant pool was introduced in R12B, empty tuples ({}) +# are literals. Therefore we no longer need to allow put_tuple/2 +# with a tuple size of zero. + +put_tuple u==0 d => too_old_compiler + +# +# All the other instructions. +# + +%cold +# An unaligned label. The address of an unaligned label must never be saved +# on the stack or used in a context where it can be confused with an Erlang term. + +label L + +# An label aligned to a certain boundary. This is used in two cases: +# +# * When the label points to the start of a function, as the ErtsCodeInfo +# struct must be word-aligned. +# * When the address is stored on the stack or otherwise needs to be properly +# tagged as a continuation pointer. +aligned_label L t + +i_func_info I a a I +int_code_end + +i_generic_breakpoint +i_debug_breakpoint +i_return_time_trace +i_return_to_trace +trace_jump W +i_yield +%hot + +return + +# +# A tail call will not refer to the current function on error unless it's a +# BIF, so we can omit the line instruction for non-BIFs. +# + +move S X0=x==0 | line Loc | call_ext_last Ar Func=u$is_not_bif D => \ + move S X0 | call_ext_last Ar Func D +move S X0=x==0 | line Loc | call_ext_only Ar Func=u$is_not_bif => \ + move S X0 | call_ext_only Ar Func + +move S X0=x==0 | line Loc | call_last Ar Func D => \ + move S X0 | call_last Ar Func D +move S X0=x==0 | line Loc | call_only Ar Func => \ + move S X0 | call_only Ar Func + +# The line number in int_func_start/5 can be NIL. +func_line n => empty_func_line + +empty_func_line +func_line I + +line n => +line I + +allocate t t? +allocate_heap t I t? + +deallocate t + +init y + +trim N Remaining => i_trim N + +i_trim t + +test_heap I t? + +# Translate instructions generated by a compiler before OTP 24. +allocate_zero Ns Live => allocate_heap_zero Ns u Live +allocate_heap_zero Ns Nh Live => allocate_heap_zero(Ns, Nh, Live) + +init_yregs I * + +# Selecting values. + +# The size of the dispatch code for a jump table is about 40 +# bytes. Therefore we shouldn't use a jump table if there are too few +# values. + +select_val S Fail=fn Size=u Rest=* | use_jump_tab(Size, Rest, 6) => \ + jump_tab(S, Fail, Size, Rest) +is_integer Fail=f S | select_val S=s Fail=fn Size=u Rest=* | use_jump_tab(Size, Rest, 6) => \ + jump_tab(S, Fail, Size, Rest) + +is_integer TypeFail=f S | select_val S=s Fail=fn Size=u Rest=* | \ + mixed_types(Size, Rest) => \ + split_values(S, TypeFail, Fail, Size, Rest) + +select_val S Fail=fn Size=u Rest=* | mixed_types(Size, Rest) => \ + split_values(S, Fail, Fail, Size, Rest) + +is_integer Fail=f S | select_val S=d Fail=fn Size=u Rest=* | \ + fixed_size_values(Size, Rest) => select_val(S, Fail, Size, Rest) + +is_atom Fail=f S | select_val S=d Fail=fn Size=u Rest=* | \ + fixed_size_values(Size, Rest) => select_val(S, Fail, Size, Rest) + +select_val S Fail=fn Size=u Rest=* | floats_or_bignums(Size, Rest) => \ + select_literals(S, Fail, Size, Rest) + +select_val S Fail=fn Size=u Rest=* | fixed_size_values(Size, Rest) => \ + select_val(S, Fail, Size, Rest) + +is_tuple Fail=f S | select_tuple_arity S=d Fail=f Size=u Rest=* => \ + select_tuple_arity(S, Fail, Size, Rest) + +select_tuple_arity S=d Fail=f Size=u Rest=* => \ + select_tuple_arity(S, Fail, Size, Rest) + +i_select_val_bins s fn I * + +i_select_val_lins s fn I * + +i_select_tuple_arity S f? I * + +i_jump_on_val s fn I I * + +is_number f? s + +jump f + + +# +# List matching instructions. The combination of test for a nonempty list followed +# by get_{list/hd/tl} are common, so we will optimize that. +# +is_nonempty_list Fail nqia => jump Fail +is_nonempty_list Fail Src | get_list Src Hd Tl => is_nonempty_list_get_list Fail Src Hd Tl +is_nonempty_list Fail Src | get_hd Src Hd => is_nonempty_list_get_hd Fail Src Hd +is_nonempty_list Fail Src | get_tl Src Tl => is_nonempty_list_get_tl Fail Src Tl + +is_nonempty_list f? S + +get_list S d d +get_hd S d +get_tl S d + +is_nonempty_list_get_list f S d d +is_nonempty_list_get_hd f S d +is_nonempty_list_get_tl f S d + +# Old-style catch. +catch y f +catch_end y + +# Try/catch. +try Y F => catch Y F + +try_case y +try_end y + +try_case_end s + +# Destructive set tuple element + +set_tuple_element s S P + +# +# Get tuple element. Since this instruction is frequently used, we will try +# to only fetch the pointer to the tuple once for a sequence of BEAM instructions +# that fetch multiple elements from the same tuple. +# + +current_tuple/1 +current_tuple/2 + +is_tuple Fail=f Src | test_arity Fail Src Arity => \ + i_is_tuple_of_arity Fail Src Arity | current_tuple Src + +test_arity Fail Src Arity => i_test_arity Fail Src Arity | current_tuple Src + +is_tuple NotTupleFail Tuple | is_tagged_tuple WrongRecordFail Tuple Arity Atom => \ + i_is_tagged_tuple_ff NotTupleFail WrongRecordFail Tuple Arity Atom | current_tuple Tuple + +is_tagged_tuple Fail Tuple Arity Atom => \ + i_is_tagged_tuple Fail Tuple Arity Atom | current_tuple Tuple + +is_tuple Fail=f Src => i_is_tuple Fail Src | current_tuple Src + +i_is_tuple_of_arity f? s A +i_test_arity f? s A + +i_is_tagged_tuple f? s A a +i_is_tagged_tuple_ff f? f? s A a + +i_is_tuple f? s + +# Generate instruction sequence for fetching the tuple element and remember +# that we have a current tuple pointer. + +get_tuple_element Tuple Pos Dst => \ + load_tuple_ptr Tuple | i_get_tuple_element Tuple Pos Dst | current_tuple Tuple +current_tuple Tuple | get_tuple_element Tuple Pos Dst => \ + i_get_tuple_element Tuple Pos Dst | current_tuple Tuple + +# Drop the current_tuple instruction if the tuple is overwritten. +i_get_tuple_element Tuple Pos Tuple | current_tuple Tuple => i_get_tuple_element Tuple Pos Tuple + +# This is a current_tuple instruction instruction not followed by get_tuple_element. +# Invalidate the current tuple pointer. + +current_tuple Tuple => + +load_tuple_ptr s + +# If both positions and destinations are in consecutive memory, fetch and store +# two words at once. +i_get_tuple_element Tuple Pos1 Dst1 | current_tuple Tuple | \ + get_tuple_element Tuple Pos2 Dst2 | consecutive_words(Pos1, Dst1, Pos2, Dst2) => \ + get_two_tuple_elements Tuple Pos1 Dst1 Dst2 | current_tuple Tuple Dst2 +i_get_tuple_element Tuple Pos1 Dst1 | current_tuple Tuple | \ + get_tuple_element Tuple Pos2 Dst2 | consecutive_words(Pos1, Dst2, Pos2, Dst1) => \ + get_two_tuple_elements Tuple Pos1 Dst1 Dst2 | current_tuple Tuple Dst2 + +# Drop the current_tuple instruction if the tuple is overwritten. +current_tuple Tuple Tuple => +current_tuple Tuple Dst => current_tuple Tuple + +# The first operand will only be used in the debug-compiled runtime +# system to verify that the register holding the tuple pointer agrees +# with the source tuple operand. +i_get_tuple_element s P S +get_two_tuple_elements s P S S + +# +# Expection rasing instructions. Infrequently executed. +# + +%cold +case_end s + +badmatch s + +if_end + +raise s s + +# Workaround the limitation that generators must always return at least one instruction. +delete_me/0 +delete_me => + +system_limit/1 +system_limit p => system_limit_body +system_limit Fail=f => jump Fail + +system_limit_body + +%hot + +# +# Optimize moves of consecutive memory addresses. +# + +move Src=c Dst => i_move Src Dst + +move Src SrcDst | move SrcDst Dst => i_move Src SrcDst | move SrcDst Dst + +# Try to move two words at once. Always arrange the source operands in +# consecutive order; the destination operands may be in consecutive or +# reverse consecutive order. + +move S1=d D1=d | move S2=d D2=d | consecutive_words(S1, D1, S2, D2) => move_two_words S1 D1 S2 D2 +move S1=d D1=d | move S2=d D2=d | consecutive_words(S1, D2, S2, D1) => move_two_words S1 D1 S2 D2 +move S1=d D1=d | move S2=d D2=d | consecutive_words(S2, D1, S1, D2) => move_two_words S2 D2 S1 D1 +move S1=d D1=d | move S2=d D2=d | consecutive_words(S2, D2, S1, D1) => move_two_words S2 D2 S1 D1 + +move Src Dst => i_move Src Dst + +# +# Move instructions. +# + +swap d d +i_move s d +move_two_words s d s d + +# +# Receive operations. We conservatively align all labels before any +# of the receive instructions. +# +# As the labels may be stored in the process structure, we must align them to +# the nearest 4-byte boundary to ensure they're properly tagged as continuation +# pointers. +# + +label L | loop_rec Fail Reg => \ + aligned_label L u=4 | loop_rec Fail Reg +label L | wait_timeout Fail Src => \ + aligned_label L u=4 | wait_timeout Fail Src +label L | wait Fail => \ + aligned_label L u=4 | wait Fail +label L | timeout => \ + aligned_label L u=4 | timeout + +loop_rec Fail x==0 | smp_mark_target_label(Fail) => i_loop_rec Fail + +aligned_label L A | wait_timeout Fail Src | smp_already_locked(L) => \ + aligned_label L A | wait_timeout_locked Src Fail +wait_timeout Fail Src => wait_timeout_unlocked Src Fail + +aligned_label L A | wait Fail | smp_already_locked(L) => \ + aligned_label L A | wait_locked Fail +wait Fail => wait_unlocked Fail + +aligned_label L A | timeout | smp_already_locked(L) => \ + aligned_label L A | timeout_locked + +remove_message +timeout +timeout_locked +i_loop_rec f +loop_rec_end f +wait_locked f +wait_unlocked f + +# Note that a timeout value must fit in 32 bits. +wait_timeout_unlocked s f +wait_timeout_locked s f + +send + +# +# Optimized comparisons with one immediate/literal operand. +# + +is_eq_exact Lbl S S => +is_eq_exact Lbl C=c R=xy => is_eq_exact Lbl R C + +is_eq_exact Lbl R=xy n => is_nil Lbl R +is_eq_exact Lbl R=xy C=ia => i_is_eq_exact_immed Lbl R C +is_eq_exact Lbl R=xy C=q => is_eq_exact_literal(Lbl, R, C) + +is_ne_exact Lbl S S => jump Lbl +is_ne_exact Lbl C=c R=xy => is_ne_exact Lbl R C + +is_ne_exact Lbl R=xy C=ian => i_is_ne_exact_immed Lbl R C +is_ne_exact Lbl R=xy C=q => i_is_ne_exact_literal Lbl R C + +i_is_eq_exact_immed f? s c + +i_is_eq_exact_literal/4 +i_is_eq_exact_literal f? s c I + +i_is_ne_exact_immed f? s c + +i_is_ne_exact_literal f? s c + +is_eq_exact f? s s + +is_ne_exact f? s s + +is_lt f? s s +is_ge f? s s + +is_eq Fail=f Const=c Reg=xy => is_eq Fail Reg Const +is_eq f? s s + +is_ne Fail=f Const=c Reg=xy => is_ne Fail Reg Const +is_ne f? s s + +# +# Putting tuples. +# +# Code compiled with OTP 22 and later uses put_tuple2 to +# to construct a tuple. +# +# Code compiled before OTP 22 uses put_tuple + one put instruction +# per element. Translate to put_tuple2. +# + +i_put_tuple/2 +put_tuple Arity Dst => i_put_tuple Dst u + +i_put_tuple Dst Arity Puts=* | put S1 | put S2 | \ + put S3 | put S4 | put S5 => \ + tuple_append_put5(Arity, Dst, Puts, S1, S2, S3, S4, S5) + +i_put_tuple Dst Arity Puts=* | put S => \ + tuple_append_put(Arity, Dst, Puts, S) + +i_put_tuple Dst Arity Puts=* => put_tuple2 Dst Arity Puts + +put_tuple2 S A * + +# +# Putting lists. +# + +store_cons/2 +put_list Hd Tl Dst => put_cons Hd Tl | store_cons u=1 Dst +store_cons Len Dst | put_list Hd Dst Dst | distinct(Dst, Hd) => combine_conses(Len, Dst, Hd) +store_cons Len Dst | put_list Hd Dst OtherDst => store_cons Len Dst | append_cons u Hd | store_cons u=1 OtherDst + +put_cons s s +append_cons I s +store_cons I d + +# +# Some more only used by the emulator +# + +%cold +normal_exit +continue_exit +call_bif W +call_bif_mfa a a I +call_nif W W W +call_error_handler +error_action_code +return_trace +%hot + +# +# Type tests. Note that the operands for most type tests are `s` to +# ensure that literal operands will work. The BEAM compiler starting +# from OTP 22 will never emit type tests with literal operands even if +# all optimizations are turned off, but loading unoptimized code from +# older releases and code generated by alternative code generators. +# + +is_integer f? s +is_list f? s +is_atom f? s +is_float f? s + +is_nil Fail=f n => +is_nil Fail=f qia => jump Fail +is_nil f? S + +# XXX Deprecated. +is_bitstr Fail Term => is_bitstring Fail Term + +is_binary f? s +is_bitstring f? s + +is_reference f? s +is_pid f? s +is_port f? s + +is_boolean f? s + +is_function2 f? s s + +################################################################# +# External function and bif calls. +################################################################# + +# Expands into call_light_bif/2 +call_light_bif/1 + +# +# The load_nif/2 BIF is an instruction. +# + +call_ext u==2 u$func:erlang:load_nif/2 => \ + i_load_nif +call_ext_last u==2 u$func:erlang:load_nif/2 D => \ + i_load_nif | deallocate D | return +call_ext_only u==2 u$func:erlang:load_nif/2 => \ + i_load_nif | return + +%cold +i_load_nif +%hot + +# +# apply/2 is an instruction, not a BIF. +# + +call_ext u==2 u$func:erlang:apply/2 => i_apply_fun +call_ext_last u==2 u$func:erlang:apply/2 D => i_apply_fun_last D +call_ext_only u==2 u$func:erlang:apply/2 => i_apply_fun_only + +# +# The apply/3 BIF is an instruction. +# + +call_ext u==3 u$func:erlang:apply/3 => i_apply +call_ext_last u==3 u$func:erlang:apply/3 D => i_apply_last D +call_ext_only u==3 u$func:erlang:apply/3 => i_apply_only + +# +# The yield/0 BIF is an instruction +# + +call_ext u==0 u$func:erlang:yield/0 => i_yield +call_ext_last u==0 u$func:erlang:yield/0 D => i_yield | deallocate D | return +call_ext_only u==0 u$func:erlang:yield/0 => i_yield | return + +# +# The hibernate/3 BIF is an instruction. +# +call_ext u==3 u$func:erlang:hibernate/3 => i_hibernate +call_ext_last u==3 u$func:erlang:hibernate/3 D => i_hibernate +call_ext_only u==3 u$func:erlang:hibernate/3 => i_hibernate + +call_ext u==0 u$func:os:perf_counter/0 => \ + i_perf_counter +call_ext_last u==0 u$func:os:perf_counter/0 D => \ + i_perf_counter | deallocate D | return +call_ext_only u==0 u$func:os:perf_counter/0 => \ + i_perf_counter | return + +# +# BIFs like process_info/1,2 require up-to-date information about the current +# emulator state, which the ordinary call_light_bif instruction doesn't save. +# + +call_ext u Bif=u$is_bif | is_heavy_bif(Bif) => \ + i_call_ext Bif +call_ext_last u Bif=u$is_bif D | is_heavy_bif(Bif) => \ + i_call_ext Bif | deallocate D | return +call_ext_only Ar=u Bif=u$is_bif | is_heavy_bif(Bif) => \ + allocate u Ar | i_call_ext Bif | deallocate u | return + +# +# The general case for BIFs that have no special requirements. +# + +call_ext u Bif=u$is_bif => \ + call_light_bif Bif +call_ext_last u Bif=u$is_bif D => \ + call_light_bif Bif | deallocate D | return +call_ext_only Ar=u Bif=u$is_bif => \ + allocate u Ar | call_light_bif Bif | deallocate u | return + +# +# Any remaining calls are calls to Erlang functions, not BIFs. +# We rename the instructions to internal names. This is necessary, +# to avoid an end-less loop, because we want to call a few BIFs +# with call instructions. +# + +call_ext Ar Func => i_call_ext Func +call_ext_last Ar Func D => i_call_ext_last Func D +call_ext_only Ar Func => i_call_ext_only Func + +i_validate t + +i_apply +i_apply_last t +i_apply_only + +i_apply_fun +i_apply_fun_last t +i_apply_fun_only + +call_light_bif Bif => call_light_bif Bif Bif +call_light_bif b e + +%cold + +i_hibernate +i_perf_counter + +%hot + +# +# Calls to non-building and guard BIFs. +# + +bif0 u$bif:erlang:self/0 Dst=d => self Dst +bif0 u$bif:erlang:node/0 Dst=d => node Dst + +bif1 Fail=f Bif=u$bif:erlang:hd/1 Src=n Dst => \ + jump Fail +bif1 Fail=f Bif=u$bif:erlang:hd/1 Src Dst => \ + is_nonempty_list Fail Src | get_hd Src Dst +bif1 Fail Bif=u$bif:erlang:hd/1 Src Dst => \ + bif_hd Fail Src Dst + +bif_hd j? s d + +bif1 Fail=f Bif=u$bif:erlang:tl/1 Src=n Dst => \ + jump Fail +bif1 Fail=f Bif=u$bif:erlang:tl/1 Src Dst => \ + is_nonempty_list Fail Src | get_tl Src Dst + +bif1 Fail Bif=u$bif:erlang:get/1 Src=s Dst=d => get(Src, Dst) + +bif2 Fail u$bif:erlang:element/2 S1=ixy S2 Dst => bif_element Fail S1 S2 Dst + +bif_element j? s s d + +bif1 Fail Bif S1 Dst | never_fails(Bif) => nofail_bif1 S1 Bif Dst +bif2 Fail Bif S1 S2 Dst | never_fails(Bif) => nofail_bif2 S1 S2 Bif Dst + +bif1 Fail Bif S1 Dst => i_bif1 S1 Fail Bif Dst +bif2 Fail Bif S1 S2 Dst => i_bif2 S1 S2 Fail Bif Dst + +nofail_bif2 S1=d S2=ian Bif Dst | is_eq_exact_bif(Bif) => bif_is_eq_exact_immed S1 S2 Dst +nofail_bif2 S1=d S2=ian Bif Dst | is_ne_exact_bif(Bif) => bif_is_ne_exact_immed S1 S2 Dst + +i_get_hash c I d +i_get s d + +self d + +node d + +nofail_bif1 s b d +nofail_bif2 s s b d + +i_bif1 s j? b d +i_bif2 s s j? b d +i_bif3 s s s j? b d + +bif_is_eq_exact_immed S c d +bif_is_ne_exact_immed S c d + +# +# Internal calls. +# + +call Ar Func => i_call Func +call_last Ar Func D => i_call_last Func D +call_only Ar Func => i_call_only Func + +i_call f +i_call_last f t +i_call_only f + +i_call_ext e +i_call_ext_last e t +i_call_ext_only e + +# Fun calls. + +call_fun Arity | deallocate D | return => i_call_fun_last Arity D +call_fun Arity => i_call_fun Arity + +i_call_fun t +i_call_fun_last t t + +# +# A fun with an empty environment can be converted to a literal. +# As a further optimization, the we try to move the fun to its +# final destination directly. + +make_fun2 OldIndex=u => make_fun2(OldIndex) +make_fun3 OldIndex=u Dst=d NumFree=u Env=* => make_fun3(OldIndex, Dst, NumFree, Env) + +%cold + +i_make_fun3 F S t * + +# Psuedo-instruction for signalling lambda load errors. Never actually runs. +i_lambda_error t + +%hot + +is_function f? S +is_function Fail=f c => jump Fail + +# The start and end of a function. +int_func_start/5 +int_func_end/0 + +func_prologue/2 + +int_func_start Func_Label Func_Line M F A | \ + label Entry_Label | line Entry_Line => \ + int_func_start Func_Label Func_Line M F A | \ + func_prologue Entry_Label Entry_Line + +int_func_start Func_Label Func_Line M F A | \ + label Entry_Label => \ + int_func_start Func_Label Func_Line M F A | \ + func_prologue Entry_Label n + +int_func_start Func_Label Func_Line M F A | \ + func_prologue Entry_Label Entry_Line | \ + is_mfa_bif(M, F, A) => \ + aligned_label Func_Label u=8 | \ + func_line Func_Line | \ + i_func_info Func_Label M F A | \ + aligned_label Entry_Label u=8 | \ + line Entry_Line | \ + i_breakpoint_trampoline | \ + call_bif_mfa M F A + +int_func_start Func_Label Func_Line M F A | \ + func_prologue Entry_Label Entry_Line => \ + aligned_label Func_Label u=8 | \ + func_line Func_Line | \ + i_func_info Func_Label M F A | \ + aligned_label Entry_Label u=8 | \ + line Entry_Line | \ + i_breakpoint_trampoline | \ + i_test_yield + + +int_func_end | needs_nif_padding() => i_nif_padding +int_func_end => + +# Handles yielding on function ingress (rather than on each call). +i_test_yield + +# Ensures that the prior function is large enough to allow NIF patching. +i_nif_padding + +# Handles tracing, early NIF calls, and so on. +i_breakpoint_trampoline + +# ================================================================ +# Bit syntax matching obsoleted in OTP 22. +# ================================================================ + +%cold +bs_start_match2 Fail=f ica X Y D => jump Fail +bs_start_match2 Fail Bin X Y D => i_bs_start_match2 Bin Fail X Y D +i_bs_start_match2 S f t t d + +bs_save2 Reg Index => bs_save(Reg, Index) +i_bs_save2 x t + +bs_restore2 Reg Index => bs_restore(Reg, Index) +i_bs_restore2 S t + +bs_context_to_binary S +%warm + +# ================================================================ +# New bit syntax matching (R11B). +# ================================================================ + +%warm + +# Matching integers +bs_match_string Fail Ms Bits Val => i_bs_match_string Ms Fail Bits Val + +i_bs_match_string S f W W + +# Fetching integers from binaries. +bs_get_integer2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \ + get_integer2(Fail, Ms, Live, Sz, Unit, Flags, Dst) + +i_bs_get_integer S f? t t s d + +i_bs_get_integer_8 S t f? d +i_bs_get_integer_16 S t f? d +i_bs_get_integer_32 S t f? d +i_bs_get_integer_64 S t f? t d + +# Fetching binaries from binaries. +bs_get_binary2 Fail=f Ms=xy Live=u Sz=sq Unit=u Flags=u Dst=d => \ + get_binary2(Fail, Ms, Live, Sz, Unit, Flags, Dst) + +i_bs_get_binary2 S f t? s t d +i_bs_get_binary_all2 S f? t t d + +# Fetching float from binaries. +bs_get_float2 Fail=f Ms=xy Live=u Sz=s Unit=u Flags=u Dst=d => \ + get_float2(Fail, Ms, Live, Sz, Unit, Flags, Dst) + +bs_get_float2 Fail=f Ms=x Live=u Sz=q Unit=u Flags=u Dst=d => jump Fail + +i_bs_get_float2 S f? t s t d + +# Miscellanous + +bs_skip_bits2 Fail=f Ms=xy Sz=sq Unit=u Flags=u => \ + skip_bits2(Fail, Ms, Sz, Unit, Flags) + +i_bs_skip_bits_imm2 f? S W +i_bs_skip_bits2 S S f? t + +bs_test_tail2 f? S t + +bs_test_unit f? S t + +# Gets a bitstring from the tail of a context. +bs_get_tail S d t + +# New bs_start_match variant for contexts with external position storage. +# +# bs_get/set_position is used to save positions into registers instead of +# "slots" in the context itself, which lets us continue matching even after +# we've passed it off to another function. + +bs_start_match4 a==am_no_fail Live=u Src=xy Ctx=d => \ + bs_start_match3 p Src Live Ctx +bs_start_match4 Fail=f Live=u Src=xy Ctx=d => \ + bs_start_match3 Fail Src Live Ctx + +%if ARCH_64 + +# This instruction nops on 64-bit platforms +bs_start_match4 a==am_resume Live Same Same => +bs_start_match4 a==am_resume Live Ctx Dst => move Ctx Dst + +%else + +bs_start_match4 a==am_resume Live Ctx Dst => \ + bs_start_match4 a=am_no_fail Live Ctx Dst + +%endif + +bs_start_match3 Fail=j ica Live Dst => jump Fail +bs_start_match3 Fail Bin Live Dst => i_bs_start_match3 Bin Live Fail Dst + +i_bs_start_match3 S t j d + +# Match context position instructions. 64-bit assumes that all positions can +# fit into an unsigned small. + +%if ARCH_64 + bs_get_position Src Dst Live => i_bs_get_position Src Dst + i_bs_get_position S S + bs_set_position S S +%else + bs_get_position S d t? + bs_set_position S S +%endif + +# +# Utf8/utf16/utf32 support. (R12B-5) +# +bs_get_utf8 Fail=f Ms=xy u u Dst=d => i_bs_get_utf8 Ms Fail Dst +i_bs_get_utf8 S f? d + +bs_skip_utf8 Fail=f Ms=xy u u => i_bs_skip_utf8 Ms Fail +i_bs_skip_utf8 S f? + +bs_get_utf16 Fail=f Ms=xy u Flags=u Dst=d => i_bs_get_utf16 Ms Fail Flags Dst +bs_skip_utf16 Fail=f Ms=xy u Flags=u => i_bs_skip_utf16 Ms Fail Flags + +i_bs_get_utf16 S f? t d +i_bs_skip_utf16 S f? t + +bs_get_utf32 Fail=f Ms=xy Live=u Flags=u Dst=d => \ + bs_get_integer2 Fail Ms Live i=32 u=1 Flags Dst | \ + i_bs_validate_unicode_retract Fail Dst Ms +bs_skip_utf32 Fail=f Ms=xy Live=u Flags=u => \ + bs_get_integer2 Fail Ms Live i=32 u=1 Flags x | \ + i_bs_validate_unicode_retract Fail x Ms + +i_bs_validate_unicode_retract j s S +%hot + +# +# Constructing binaries +# +%warm + +bs_init2 Fail Sz Words Regs Flags Dst | binary_too_big(Sz) => system_limit Fail + +bs_init2 Fail Sz=u Words=u==0 Regs Flags Dst => i_bs_init Sz Regs Dst + +bs_init2 Fail Sz=u Words Regs Flags Dst => \ + i_bs_init_heap Sz Words Regs Dst + +bs_init2 Fail Sz Words=u==0 Regs Flags Dst => \ + i_bs_init_fail Sz Fail Regs Dst +bs_init2 Fail Sz Words Regs Flags Dst => \ + i_bs_init_fail_heap Sz Words Fail Regs Dst + +i_bs_init_fail S j? t? S + +i_bs_init_fail_heap s I j? t? S + +i_bs_init W t? S + +i_bs_init_heap W I t? S + + +bs_init_bits Fail Sz=o Words Regs Flags Dst => system_limit Fail + +bs_init_bits Fail Sz=u Words=u==0 Regs Flags Dst => \ + i_bs_init_bits Sz Regs Dst +bs_init_bits Fail Sz=u Words Regs Flags Dst => \ + i_bs_init_bits_heap Sz Words Regs Dst + +bs_init_bits Fail Sz Words=u==0 Regs Flags Dst => \ + i_bs_init_bits_fail Sz Fail Regs Dst +bs_init_bits Fail Sz Words Regs Flags Dst => \ + i_bs_init_bits_fail_heap Sz Words Fail Regs Dst + +i_bs_init_bits_fail S j? t? S + +i_bs_init_bits_fail_heap s I j? t? S + +i_bs_init_bits W t? S +i_bs_init_bits_heap W I t? S + +bs_add Fail S1=i==0 S2 Unit=u==1 D => move S2 D + +bs_add j? s s t? x + +bs_append Fail Size Extra Live Unit Bin Flags Dst => \ + i_bs_append Fail Extra Live Unit Size Bin Dst + +bs_private_append Fail Size Unit Bin Flags Dst => \ + i_bs_private_append Fail Unit Size Bin Dst + +i_bs_private_append Fail Unit Size Bin Dst=y => \ + i_bs_private_append Fail Unit Size Bin x | move x Dst + +bs_init_writable + +i_bs_append j? I t? t s s S +i_bs_private_append j? t s S x + +# +# Storing integers into binaries. +# + +bs_put_integer Fail=j Sz=sq Unit=u Flags=u Src=s => \ + put_integer(Fail, Sz, Unit, Flags, Src) + +i_new_bs_put_integer j? S t s +i_new_bs_put_integer_imm s j? W t + +# +# Utf8/utf16/utf32 support. (R12B-5) +# + +bs_utf8_size j Src Dst=d => i_bs_utf8_size Src Dst +bs_utf16_size j Src Dst=d => i_bs_utf16_size Src Dst + +bs_put_utf8 Fail u Src => i_bs_put_utf8 Fail Src + +bs_put_utf32 Fail=j Flags=u Src=s => \ + i_bs_validate_unicode Fail Src | bs_put_integer Fail i=32 u=1 Flags Src + +i_bs_utf8_size s x +i_bs_utf16_size s x + +i_bs_put_utf8 j? s +bs_put_utf16 j? t s + +i_bs_validate_unicode j? s + +# +# Storing floats into binaries. +# + +# Will fail. No need to keep the instruction, because bs_add or +# bs_init* would already have raised an exception. +bs_put_float Fail Sz=q Unit Flags Val => + +bs_put_float Fail=j Sz=s Unit=u Flags=u Src=s => \ + put_float(Fail, Sz, Unit, Flags, Src) + +i_new_bs_put_float j? S t s +i_new_bs_put_float_imm j? W t s + +# +# Storing binaries into binaries. +# + +bs_put_binary Fail=j Sz=s Unit=u Flags=u Src=s => \ + put_binary(Fail, Sz, Unit, Flags, Src) + +i_new_bs_put_binary j? s t s +i_new_bs_put_binary_imm j? W s +i_new_bs_put_binary_all s j? t + +# +# Warning: The i_bs_put_string and i_new_bs_put_string instructions +# are specially treated in the loader. +# Don't change the instruction format unless you change the loader too. +# + +bs_put_string W W + +# +# New floating point instructions (R8). +# + +fadd p FR1 FR2 FR3 => i_fadd FR1 FR2 FR3 +fsub p FR1 FR2 FR3 => i_fsub FR1 FR2 FR3 +fmul p FR1 FR2 FR3 => i_fmul FR1 FR2 FR3 +fdiv p FR1 FR2 FR3 => i_fdiv FR1 FR2 FR3 +fnegate p FR1 FR2 => i_fnegate FR1 FR2 + +fmove Arg=l Dst=d => fstore Arg Dst +fmove Arg=dq Dst=l => fload Arg Dst + +fstore l d +fload Sq l + +fconv s l + +i_fadd l l l +i_fsub l l l +i_fmul l l l +i_fdiv l l l +i_fnegate l l + +fclearerror => +fcheckerror p => + +%hot + +# +# New apply instructions in R10B. +# + +apply t +apply_last t t + +# +# Handle compatibility with OTP 17 here. +# + +i_put_map_assoc/4 + +# We KNOW that in OTP 20 (actually OTP 18 and higher), a put_map_assoc instruction +# is always preceded by an is_map test. That means that put_map_assoc can never +# fail and does not need any failure label. + +put_map_assoc Fail Map Dst Live Size Rest=* | compiled_with_otp_20_or_higher() => \ + i_put_map_assoc Map Dst Live Size Rest + +# Translate the put_map_assoc instruction if the module was compiled by a compiler +# before 20. This is only necessary if the OTP 17 compiler was used, but we +# have no safe and relatively easy way to know whether OTP 18/19 was used. + +put_map_assoc Fail=p Map Dst Live Size Rest=* => \ + ensure_map Map | i_put_map_assoc Map Dst Live Size Rest +put_map_assoc Fail=f Map Dst Live Size Rest=* => \ + is_map Fail Map | i_put_map_assoc Map Dst Live Size Rest + +ensure_map Lit=q | literal_is_map(Lit) => + +%cold +ensure_map s +%hot + +# +# Map instructions. First introduced in R17. +# + +sorted_put_map_assoc/4 +i_put_map_assoc Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ + sorted_put_map_assoc Map Dst Live Size Rest + +sorted_put_map_exact/5 +put_map_exact F Map Dst Live Size Rest=* | map_key_sort(Size, Rest) => \ + sorted_put_map_exact F Map Dst Live Size Rest + +sorted_put_map_assoc Map Dst Live Size Rest=* | is_empty_map(Map) => \ + new_map Dst Live Size Rest +sorted_put_map_assoc Src=s Dst Live Size Rest=* => \ + update_map_assoc Src Dst Live Size Rest + +sorted_put_map_exact Fail Src Dst Live Size Rest=* => \ + update_map_exact Src Fail Dst Live Size Rest + +new_map Dst Live Size Rest=* | is_small_map_literal_keys(Size, Rest) => \ + new_small_map_lit(Dst, Live, Size, Rest) + +new_map d t I * +i_new_small_map_lit d t q I * +update_map_assoc s d t I * +update_map_exact s j? d t I * + +is_map f? s + +## Transform has_map_fields #{ K1 := _, K2 := _ } to has_map_elements + +has_map_fields Fail Src Size Rest=* => \ + has_map_fields(Fail, Src, Size, Rest) + +## Transform get_map_elements(s) #{ K1 := V1, K2 := V2 } + +get_map_elements Fail Src Size=u==2 Rest=* => \ + get_map_element(Fail, Src, Size, Rest) +get_map_elements Fail Src Size Rest=* | map_key_sort(Size, Rest) => \ + get_map_elements(Fail, Src, Size, Rest) + +i_get_map_elements f? s I * + +i_get_map_element_hash Fail Src=c Key Hash Dst => \ + move Src x | i_get_map_element_hash Fail x Key Hash Dst +i_get_map_element_hash f? S c I S + +i_get_map_element Fail Src=c Key Dst => \ + move Src x | i_get_map_element Fail x Key Dst +i_get_map_element f? S S S + +# +# Convert the plus operations to a generic plus instruction. +# +gen_plus/5 +gen_minus/5 + +gc_bif1 Fail Live u$bif:erlang:splus/1 Src Dst => \ + gen_plus Fail Live Src i Dst +gc_bif2 Fail Live u$bif:erlang:splus/2 S1 S2 Dst => \ + gen_plus Fail Live S1 S2 Dst + +gc_bif1 Fail Live u$bif:erlang:sminus/1 Src Dst => \ + i_unary_minus Src Fail Dst +gc_bif2 Fail Live u$bif:erlang:sminus/2 S1 S2 Dst => \ + gen_minus Fail Live S1 S2 Dst + +# +# Optimize addition and subtraction of small literals using +# the i_increment/3 instruction (in bodies, not in guards). +# + +gen_plus p Live Int=i Reg=d Dst => \ + increment(Reg, Int, Dst) +gen_plus p Live Reg=d Int=i Dst => \ + increment(Reg, Int, Dst) + +gen_minus p Live Reg=d Int=i Dst | negation_is_small(Int) => \ + increment_from_minus(Reg, Int, Dst) + +# +# Arithmetic instructions. +# + +# It is OK to swap arguments for '+' in a guard. It is also +# OK to turn minus into plus in a guard. +gen_plus Fail=f Live S1=c S2 Dst => i_plus S2 S1 Fail Dst +gen_minus Fail=f Live S1 S2=i Dst | negation_is_small(S2) => \ + plus_from_minus(Fail, Live, S1, S2, Dst) + +gen_plus Fail Live S1 S2 Dst => i_plus S1 S2 Fail Dst + +gen_minus Fail Live S1 S2 Dst => i_minus S1 S2 Fail Dst + +gc_bif2 Fail Live u$bif:erlang:stimes/2 S1 S2 Dst => \ + i_times Fail S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:div/2 S1 S2 Dst => \ + i_m_div Fail S1 S2 Dst + +# Fused 'rem'/'div' pair. +gc_bif2 Fail Live u$bif:erlang:rem/2 LHS RHS Remainder | \ + gc_bif2 A B u$bif:erlang:intdiv/2 LHS RHS Quotient | \ + distinct(LHS, Remainder) | \ + distinct(RHS, Remainder) => \ + i_rem_div LHS RHS Fail Remainder Quotient + +# As above but with a `line` in between +gc_bif2 Fail Live u$bif:erlang:rem/2 LHS RHS Remainder | \ + line Loc | \ + gc_bif2 A B u$bif:erlang:intdiv/2 LHS RHS Quotient | \ + distinct(LHS, Remainder) | \ + distinct(RHS, Remainder) => \ + i_rem_div LHS RHS Fail Remainder Quotient + +# Fused 'div'/'rem' pair +gc_bif2 Fail Live u$bif:erlang:intdiv/2 LHS RHS Quotient | \ + gc_bif2 A B u$bif:erlang:rem/2 LHS RHS Remainder | \ + distinct(LHS, Quotient) | \ + distinct(RHS, Quotient) => \ + i_div_rem LHS RHS Fail Quotient Remainder + +# As above but with a `line` in between +gc_bif2 Fail Live u$bif:erlang:intdiv/2 LHS RHS Quotient | \ + line Loc | \ + gc_bif2 A B u$bif:erlang:rem/2 LHS RHS Remainder | \ + distinct(LHS, Quotient) | \ + distinct(RHS, Quotient) => \ + i_div_rem LHS RHS Fail Quotient Remainder + +gc_bif2 Fail Live u$bif:erlang:intdiv/2 S1 S2 Dst => \ + i_int_div Fail S1 S2 Dst +gc_bif2 Fail Live u$bif:erlang:rem/2 S1 S2 Dst => \ + i_rem S1 S2 Fail Dst + +gc_bif2 Fail Live u$bif:erlang:bsl/2 S1 S2 Dst => \ + i_bsl S1 S2 Fail Dst +gc_bif2 Fail Live u$bif:erlang:bsr/2 S1 S2 Dst => \ + i_bsr S1 S2 Fail Dst + +gc_bif2 Fail Live u$bif:erlang:band/2 S1 S2 Dst => \ + i_band S1 S2 Fail Dst + +gc_bif2 Fail Live u$bif:erlang:bor/2 S1 S2 Dst => \ + i_bor Fail S1 S2 Dst + +gc_bif2 Fail Live u$bif:erlang:bxor/2 S1 S2 Dst => \ + i_bxor Fail S1 S2 Dst + +gc_bif1 Fail Live u$bif:erlang:bnot/1 Src Dst => \ + i_bnot Fail Src Dst + +i_increment S W d + +i_plus s s j? d +i_minus s s j? d + +i_unary_minus s j? d + +i_times j? s s d + +i_m_div j? s s d + +i_rem_div s s j? d d +i_div_rem s s j? d d +i_int_div j? s s d +i_rem s s j? d + +i_bsl s s j? d +i_bsr s s j? d + +i_band s s j? d +i_bor j? s s d +i_bxor j? s s d + +i_bnot j? s d + +# +# Old guard BIFs that creates heap fragments are no longer allowed. +# +bif1 Fail u$bif:erlang:length/1 s d => too_old_compiler +bif1 Fail u$bif:erlang:size/1 s d => too_old_compiler +bif1 Fail u$bif:erlang:abs/1 s d => too_old_compiler +bif1 Fail u$bif:erlang:float/1 s d => too_old_compiler +bif1 Fail u$bif:erlang:round/1 s d => too_old_compiler +bif1 Fail u$bif:erlang:trunc/1 s d => too_old_compiler + +# +# Handle the length/1 guard BIF specially to make it trappable. +# + +gc_bif1 Fail=j Live u$bif:erlang:length/1 Src Dst => \ + i_length_setup Fail Live Src | i_length Fail Live Dst + +i_length_setup j? t s +i_length j? t d + +# +# Guard BIFs. +# +gc_bif1 Fail Live Bif Src Dst => i_bif1 Src Fail Bif Dst +gc_bif2 Fail Live Bif S1 S2 Dst => i_bif2 S1 S2 Fail Bif Dst +gc_bif3 Fail Live Bif S1 S2 S3 Dst => i_bif3 S1 S2 S3 Fail Bif Dst + +# +# The following instruction is specially handled in beam_load.c +# to produce a user-friendly message if an unsupported guard BIF is +# encountered. +# +unsupported_guard_bif/3 +unsupported_guard_bif A B C | never() => + +# +# R13B03 +# +on_load + +# +# R14A. +# +# Superseded in OTP 24 by 'recv_marker_reserve' and friends. +# + +recv_mark f => i_recv_mark +i_recv_mark + +recv_set Fail | label Lbl | loop_rec Lf Reg => \ + i_recv_set | label Lbl | loop_rec Lf Reg +i_recv_set + +# +# OTP 21. +# + +build_stacktrace +raw_raise + +# +# Specialized move instructions. Since they don't require a second +# instruction, we have intentionally placed them after any other +# transformation rules that starts with a move instruction in order to +# produce better code for the transformation engine. +# + +move n D=y => init D + +# +# OTP 24 +# + +recv_marker_reserve S +recv_marker_bind S S +recv_marker_clear S +recv_marker_use S |