path: root/erts/emulator/beam/emu/beam_emu.c

/*
 * %CopyrightBegin%
 *
 * Copyright Ericsson AB 1996-2020. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * %CopyrightEnd%
 */

#ifdef HAVE_CONFIG_H
#  include "config.h"
#endif

#include <stddef.h> /* offsetof() */
#include "sys.h"
#include "erl_vm.h"
#include "global.h"
#include "erl_process.h"
#include "error.h"
#include "bif.h"
#include "big.h"
#include "beam_code.h"
#include "erl_binary.h"
#include "erl_map.h"
#include "erl_bits.h"
#include "dist.h"
#include "beam_bp.h"
#include "beam_catches.h"
#include "erl_thr_progress.h"
#include "erl_nfunc_sched.h"
#include "dtrace-wrapper.h"
#include "erl_proc_sig_queue.h"
#include "beam_common.h"

/* #define HARDDEBUG 1 */

/* The x86 and AMD64 optimization guides recommend that branch targets are
 * paragraph aligned (16 bytes). Neither GCC nor Clang align the addresses of
 * first class labels as used by the OpCase construction when the jump table is
 * enabled. Aligning the OpCase labels looks like a worthwhile optimization, but
 * despite the recommendation in the optimization guides, forcing alignment
 * leads to a ~5% slowdown of the emulator. Therefore alignment of OpCase labels
 * are not attempted.
 */

#if defined(NO_JUMP_TABLE)
#  define OpCase(OpCode)    case op_##OpCode
#  define CountCase(OpCode) case op_count_##OpCode
#  define IsOpCode(InstrWord, OpCode)  (BeamCodeAddr(InstrWord) == (BeamInstr)op_##OpCode)
#  define Goto(Rel)         {Go = BeamCodeAddr(Rel); goto emulator_loop;}
#  define GotoPF(Rel)       Goto(Rel)
#else
#  define OpCase(OpCode)    lb_##OpCode
#  define CountCase(OpCode) lb_count_##OpCode
#  define IsOpCode(InstrWord, OpCode)  (BeamCodeAddr(InstrWord) == (BeamInstr)&&lb_##OpCode)
#  define Goto(Rel)         goto *((void *)BeamCodeAddr(Rel))
#  define GotoPF(Rel)       goto *((void *)Rel)
#  define LabelAddr(Label)  &&Label
#endif

/*
 * Define macros for deep checking of terms.
 */

#define GET_EXPORT_MODULE(p)  ((p)->info.mfa.module)
#define GET_EXPORT_FUNCTION(p)  ((p)->info.mfa.function)
#define GET_EXPORT_ARITY(p)  ((p)->info.mfa.arity)

/*
 * Add a byte offset to a pointer to Eterm.  This is useful when the
 * the loader has precalculated a byte offset.
 */
#define ADD_BYTE_OFFSET(ptr, offset) \
   ((Eterm *) (((unsigned char *)ptr) + (offset)))

/* We don't check the range if an ordinary switch is used */
#ifdef NO_JUMP_TABLE
#  define VALID_INSTR(IP) (BeamCodeAddr(IP) < (NUMBER_OF_OPCODES*2+10))
#else
#  define VALID_INSTR(IP) \
    ((BeamInstr)LabelAddr(emulator_loop) <= BeamCodeAddr(IP) && \
     BeamCodeAddr(IP) < (BeamInstr)LabelAddr(end_emulator_loop))
#endif /* NO_JUMP_TABLE */

#define SET_I(ip) \
   ASSERT(VALID_INSTR(* (Eterm *)(ip))); \
   I = (ip)

/*
 * Register target (X or Y register).
 */

#define REG_TARGET_PTR(Target) (((Target) & 1) ? &yb((Target)-1) : &xb(Target))

/*
 * Special Beam instructions.
 */

static BeamInstr beam_run_process_[1];
ErtsCodePtr beam_run_process;

static BeamInstr beam_normal_exit_[1];
ErtsCodePtr beam_normal_exit;

static BeamInstr beam_exit_[1];
ErtsCodePtr beam_exit;

static BeamInstr beam_continue_exit_[1];
ErtsCodePtr beam_continue_exit;


/* NOTE These should be the only variables containing trace instructions.
**      Sometimes tests are for the instruction value, and sometimes
**      for the referring variable (one of these), and rouge references
**      will most likely cause chaos.
*/

/* OpCode(i_return_to_trace) */
static BeamInstr beam_return_to_trace_[1];
ErtsCodePtr beam_return_to_trace;

/* OpCode(i_return_trace) */
static BeamInstr beam_return_trace_[1];
ErtsCodePtr beam_return_trace;

/* UGLY also OpCode(i_return_trace) */
static BeamInstr beam_exception_trace_[1];
ErtsCodePtr beam_exception_trace;

/* OpCode(i_return_time_trace) */
static BeamInstr beam_return_time_trace_[1];
ErtsCodePtr beam_return_time_trace;

/* The address field of every fun that has no loaded code will point to
 * beam_unloaded_fun[]. The -1 in beam_unloaded_fun[0] will be interpreted
 * as an illegal arity when attempting to call a fun. */
static BeamInstr unloaded_fun_code[4] = {NIL, NIL, -1, 0};
ErtsCodePtr beam_unloaded_fun = &unloaded_fun_code[3];

/*
 * All Beam instructions in numerical order.
 */

#ifndef NO_JUMP_TABLE
void** beam_ops;
#endif

#define db(N) (N)
#define fb(N) ((Sint)(Sint32)(N))
#define jb(N) ((Sint)(Sint32)(N))
#define tb(N) (N)
#define xb(N) (*ADD_BYTE_OFFSET(reg, N))
#define yb(N) (*ADD_BYTE_OFFSET(E, N))
#define Sb(N) (*REG_TARGET_PTR(N))
#define lb(N) (*(double *) (((unsigned char *)&(freg[0].fd)) + (N)))
#define Qb(N) (N)
#define Ib(N) (N)

/*
 * Use LIGHT_SWAPOUT when the called function
 * will call HeapOnlyAlloc() (and never HAlloc()).
 */
#ifdef DEBUG
#  /* The stack pointer is used in an assertion. */
#  define LIGHT_SWAPOUT SWAPOUT
#  define DEBUG_SWAPOUT SWAPOUT
#  define DEBUG_SWAPIN  SWAPIN
#else
#  define LIGHT_SWAPOUT HEAP_TOP(c_p) = HTOP
#  define DEBUG_SWAPOUT
#  define DEBUG_SWAPIN
#endif

/*
 * Use LIGHT_SWAPIN when we know that c_p->stop cannot
 * have been updated (i.e. if there cannot have been
 * a garbage-collection).
 */

#define LIGHT_SWAPIN HTOP = HEAP_TOP(c_p)

#ifdef FORCE_HEAP_FRAGS
#  define HEAP_SPACE_VERIFIED(Words) do { \
      c_p->space_verified = (Words);	  \
      c_p->space_verified_from = HTOP;	  \
    }while(0)
#else
#  define HEAP_SPACE_VERIFIED(Words) ((void)0)
#endif

#define PRE_BIF_SWAPOUT(P)						\
     HEAP_TOP((P)) = HTOP;  						\
     (P)->stop = E;  							\
     PROCESS_MAIN_CHK_LOCKS((P));					\
     ERTS_UNREQ_PROC_MAIN_LOCK((P))

#ifdef DEBUG
/* Better static type testing by the C compiler */
#  define BEAM_IS_TUPLE(Src) is_tuple(Src)
#else
/* Better performance */
# define BEAM_IS_TUPLE(Src) is_boxed(Src)
#endif

/*
 * process_main() is already huge, so we want to avoid inlining
 * seldom used functions into it.
 */
static void init_emulator_finish(void) ERTS_NOINLINE;

void
init_emulator(void)
{
    process_main(0);
}

/*
 * On certain platforms, make sure that the main variables really are placed
 * in registers.
 */

#if defined(__GNUC__) && defined(sparc) && !defined(DEBUG)
#  define REG_xregs asm("%l1")
#  define REG_htop asm("%l2")
#  define REG_stop asm("%l3")
#  define REG_I asm("%l4")
#  define REG_fcalls asm("%l5")
#elif defined(__GNUC__) && defined(__amd64__) && !defined(DEBUG)
#  define REG_xregs asm("%r12")
#  define REG_htop
#  define REG_stop asm("%r13")
#  define REG_I asm("%rbx")
#  define REG_fcalls asm("%r14")
#else
#  define REG_xregs
#  define REG_htop
#  define REG_stop
#  define REG_I
#  define REG_fcalls
#endif

/*
 * process_main() is called twice:
 * The first call performs some initialisation, including exporting
 * the instructions' C labels to the loader.
 * The second call starts execution of BEAM code. This call never returns.
 */
ERTS_NO_RETPOLINE
void process_main(ErtsSchedulerData *esdp)
{
    static int init_done = 0;
    Process* c_p = NULL;
    int reds_used;
#ifdef DEBUG
    ERTS_DECLARE_DUMMY(Eterm pid);
#endif

    /* Pointer to X registers: x(1)..x(N); reg[0] is used when doing GC,
     * in all other cases x0 is used.
     */
    register Eterm* reg = NULL;

    /*
     * Top of heap (next free location); grows upwards.
     */
    register Eterm* HTOP REG_htop = NULL;

    /* Stack pointer.  Grows downwards; points
     * to last item pushed (normally a saved
     * continuation pointer).
     */
    register Eterm* E REG_stop = NULL;

    /*
     * Pointer to next threaded instruction.
     */
    register const BeamInstr *I REG_I = NULL;

    /* Number of reductions left.  This function
     * returns to the scheduler when FCALLS reaches zero.
     */
    register Sint FCALLS REG_fcalls = 0;

    /*
     * X registers and floating point registers are located in
     * scheduler specific data.
     */
    register FloatDef *freg = NULL;

    /*
     * For keeping the negative old value of 'reds' when call saving is active.
     */
    int neg_o_reds = 0;

#ifdef ERTS_OPCODE_COUNTER_SUPPORT
    static void* counting_opcodes[] = { DEFINE_COUNTING_OPCODES };
#else
#ifndef NO_JUMP_TABLE
    static void* opcodes[] = { DEFINE_OPCODES };
#else
    register BeamInstr Go;
#endif
#endif

    Uint64 start_time = 0;          /* Monitor long schedule */
    ErtsCodePtr start_time_i = NULL;

    ERTS_MSACC_DECLARE_CACHE_X() /* a cached value of the tsd pointer for msacc */

    ERL_BITS_DECLARE_STATEP; /* Has to be last declaration */

    /*
     * Note: In this function, we attempt to place rarely executed code towards
     * the end of the function, in the hope that the cache hit rate will be better.
     * The initialization code is only run once, so it is at the very end.
     *
     * Note: c_p->arity must be set to reflect the number of useful terms in
     * c_p->arg_reg before calling the scheduler.
     */
    if (ERTS_UNLIKELY(!init_done)) {
       /* This should only be reached during the init phase when only the main
        * process is running. I.e. there is no race for init_done.
        */
	init_done = 1;
	goto init_emulator;
    }

    reg = (esdp->registers)->x_reg_array.d;
    freg = (esdp->registers)->f_reg_array.d;

    c_p = NULL;
    reds_used = 0;

    goto do_schedule1;

 do_schedule:
    ASSERT(c_p->arity < 6);
    ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
    if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
	reds_used = REDS_IN(c_p) - FCALLS;
    else
	reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
    ASSERT(reds_used >= 0);
 do_schedule1:

    if (start_time != 0) {
        check_monitor_long_schedule(c_p, start_time, start_time_i);
    }

    PROCESS_MAIN_CHK_LOCKS(c_p);
    ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
    c_p = erts_schedule(NULL, c_p, reds_used);
    ERTS_VERIFY_UNUSED_TEMP_ALLOC(c_p);
    start_time = 0;
#ifdef DEBUG
    pid = c_p->common.id; /* Save for debugging purposes */
#endif
    ERTS_REQ_PROC_MAIN_LOCK(c_p);
    PROCESS_MAIN_CHK_LOCKS(c_p);

    ERTS_MSACC_UPDATE_CACHE_X();

    if (erts_system_monitor_long_schedule != 0) {
	start_time = erts_timestamp_millis();
	start_time_i = c_p->i;
    }

    ERL_BITS_RELOAD_STATEP(c_p);
    {
	int reds;
	BeamInstr next;

        copy_in_registers(c_p, reg);

	/*
	 * We put the original reduction count in the process structure, to reduce
	 * the code size (referencing a field in a struct through a pointer stored
	 * in a register gives smaller code than referencing a global variable).
	 */

	SET_I(c_p->i);

	REDS_IN(c_p) = reds = c_p->fcalls;
#ifdef DEBUG
	c_p->debug_reds_in = reds;
#endif

	if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)) {
	    neg_o_reds = -CONTEXT_REDS;
	    FCALLS = neg_o_reds + reds;
	} else {
	    neg_o_reds = 0;
	    FCALLS = reds;
	}

	ERTS_DBG_CHK_REDS(c_p, FCALLS);

	next = *I;
	SWAPIN;
	ASSERT(VALID_INSTR(next));

#ifdef USE_VM_PROBES
        if (DTRACE_ENABLED(process_scheduled)) {
            DTRACE_CHARBUF(process_buf, DTRACE_TERM_BUF_SIZE);
            DTRACE_CHARBUF(fun_buf, DTRACE_TERM_BUF_SIZE);
            dtrace_proc_str(c_p, process_buf);

            if (ERTS_PROC_IS_EXITING(c_p)) {
                sys_strcpy(fun_buf, "<exiting>");
            } else {
                ErtsCodeMFA *cmfa = erts_find_function_from_pc(c_p->i);
                if (cmfa) {
                    dtrace_fun_decode(c_p, cmfa,
                                      NULL, fun_buf);
                } else {
                    erts_snprintf(fun_buf, sizeof(DTRACE_CHARBUF_NAME(fun_buf)),
                                  "<unknown/%p>", next);
                }
            }

            DTRACE2(process_scheduled, process_buf, fun_buf);
        }
#endif
	Goto(next);
    }

#if defined(DEBUG) || defined(NO_JUMP_TABLE)
 emulator_loop:
#endif

#ifdef NO_JUMP_TABLE
    switch (Go) {
#endif

#include "beam_hot.h"
    /*
     * The labels are jumped to from the $DISPATCH() macros when the reductions
     * are used up.
     *
     * Since the I register points just beyond the FuncBegin instruction, we
     * can get the module, function, and arity for the function being
     * called from I[-3], I[-2], and I[-1] respectively.
     */
 context_switch_fun:
    /* Add one for the environment of the fun */
    c_p->arity = erts_code_to_codemfa(I)->arity + 1;
    goto context_switch2;

 context_switch:
    c_p->arity = erts_code_to_codemfa(I)->arity;

 context_switch2: 		/* Entry for fun calls. */
    c_p->current = erts_code_to_codemfa(I);

 context_switch3:

 {

     if (erts_atomic32_read_nob(&c_p->state) & ERTS_PSFLG_EXITING) {
         c_p->i = beam_exit;
         c_p->arity = 0;
         c_p->current = NULL;
         goto do_schedule;
     }

     /*
      * Since REDS_IN(c_p) is stored in the save area (c_p->arg_reg) we must read it
      * now before saving registers.
      */

     ASSERT(c_p->debug_reds_in == REDS_IN(c_p));
     if (!ERTS_PROC_GET_SAVED_CALLS_BUF(c_p))
         reds_used = REDS_IN(c_p) - FCALLS;
     else
         reds_used = REDS_IN(c_p) - (CONTEXT_REDS + FCALLS);
     ASSERT(reds_used >= 0);

     copy_out_registers(c_p, reg);

     SWAPOUT;
     c_p->i = I;
     goto do_schedule1;
 }

#include "beam_warm.h"

 OpCase(normal_exit): {
     HEAVY_SWAPOUT;
     c_p->freason = EXC_NORMAL;
     c_p->arity = 0; /* In case this process will ever be garbed again. */
     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
     erts_do_exit_process(c_p, am_normal);
     ERTS_REQ_PROC_MAIN_LOCK(c_p);
     HEAVY_SWAPIN;
     goto do_schedule;
 }

 OpCase(continue_exit): {
     HEAVY_SWAPOUT;
     ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
     erts_continue_exit_process(c_p);
     ERTS_REQ_PROC_MAIN_LOCK(c_p);
     HEAVY_SWAPIN;
     goto do_schedule;
 }

 find_func_info: {
     SWAPOUT;
     I = handle_error(c_p, I, reg, NULL);
     goto post_error_handling;
 }

 OpCase(call_error_handler): {
        /*
         * At this point, I points to the code[3] in the export entry for
         * a function which is not loaded.
         *
         * code[0]: Module
         * code[1]: Function
         * code[2]: Arity
         * code[3]: &&call_error_handler
         * code[4]: Not used
         */
        Export *error_handler;

        HEAVY_SWAPOUT;
        error_handler = call_error_handler(c_p, erts_code_to_codemfa(I),
                                           reg, am_undefined_function);
        HEAVY_SWAPIN;

        if (error_handler) {
            I = error_handler->addresses[erts_active_code_ix()];
            Goto(*I);
        }
    }
 /* Fall through */
 OpCase(error_action_code): {
    handle_error:
     SWAPOUT;
     I = handle_error(c_p, NULL, reg, NULL);
 post_error_handling:
     if (I == 0) {
	 goto do_schedule;
     } else {
	 ASSERT(!is_value(r(0)));
	 SWAPIN;
	 Goto(*I);
     }
 }

 OpCase(i_func_info_IaaI): {
     ErtsCodeInfo *ci = (ErtsCodeInfo*)I;
     c_p->freason = EXC_FUNCTION_CLAUSE;
     c_p->current = &ci->mfa;
     goto handle_error;
 }

#include "beam_cold.h"

#ifdef ERTS_OPCODE_COUNTER_SUPPORT
    DEFINE_COUNTING_LABELS;
#endif

#ifndef NO_JUMP_TABLE
#ifdef DEBUG
 end_emulator_loop:
#endif
#endif

 OpCase(int_code_end):
 OpCase(label_L):
 OpCase(on_load):
 OpCase(line_I):
 OpCase(i_nif_padding):
    erts_exit(ERTS_ERROR_EXIT, "meta op\n");

    /*
     * One-time initialization of Beam emulator.
     */

 init_emulator:
 {
#ifndef NO_JUMP_TABLE
#ifdef ERTS_OPCODE_COUNTER_SUPPORT
#ifdef DEBUG
     counting_opcodes[op_catch_end_y] = LabelAddr(lb_catch_end_y);
#endif
     counting_opcodes[op_i_func_info_IaaI] = LabelAddr(lb_i_func_info_IaaI);
     beam_ops = counting_opcodes;
#else /* #ifndef ERTS_OPCODE_COUNTER_SUPPORT */
     beam_ops = opcodes;
#endif /* ERTS_OPCODE_COUNTER_SUPPORT */
#endif /* NO_JUMP_TABLE */

     init_emulator_finish();
     return;
 }
#ifdef NO_JUMP_TABLE
 default:
    erts_exit(ERTS_ERROR_EXIT, "unexpected op code %d\n",Go);
  }
#endif
    return;			/* Never executed */
}

/*
 * Enter all BIFs into the export table.
 *
 * Note that they will all call the error_handler until their modules have been
 * loaded, which may prevent the system from booting if BIFs from non-preloaded
 * modules are apply/3'd while loading code. Ordinary BIF calls will work fine
 * however since they won't go through export entries.
 */
static void install_bifs(void) {
    int i;

    for (i = 0; i < BIF_SIZE; i++) {
        BifEntry *entry;
        Export *ep;
        int j;

        entry = &bif_table[i];

        ep = erts_export_put(entry->module, entry->name, entry->arity);

        ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
        ep->info.mfa.module = entry->module;
        ep->info.mfa.function = entry->name;
        ep->info.mfa.arity = entry->arity;
        ep->bif_number = i;

        memset(&ep->trampoline, 0, sizeof(ep->trampoline));
        ep->trampoline.common.op = BeamOpCodeAddr(op_call_error_handler);

        for (j = 0; j < ERTS_NUM_CODE_IX; j++) {
            erts_activate_export_trampoline(ep, j);
        }

        /* Set up a hidden export entry so we can trap to this BIF without
         * it being seen when tracing. */
        erts_init_trap_export(BIF_TRAP_EXPORT(i),
                              entry->module, entry->name, entry->arity,
                              entry->f);
    }
}

/*
 * One-time initialization of emulator. Does not need to be
 * in process_main().
 */
static void
init_emulator_finish(void)
{
#if defined(ARCH_64) && defined(CODE_MODEL_SMALL)
    int i;

    for (i = 0; i < NUMBER_OF_OPCODES; i++) {
        BeamInstr instr = BeamOpCodeAddr(i);
        if (instr >= (1ull << 32)) {
            erts_exit(ERTS_ERROR_EXIT,
                      "This run-time was supposed be compiled with all code below 2Gb,\n"
                      "but the instruction '%s' is located at %016lx.\n",
                      opc[i].name, instr);
        }
    }
#endif

    beam_run_process_[0]       = BeamOpCodeAddr(op_i_apply_only);
    beam_run_process = (ErtsCodePtr)&beam_run_process_[0];

    beam_normal_exit_[0]       = BeamOpCodeAddr(op_normal_exit);
    beam_normal_exit = (ErtsCodePtr)&beam_normal_exit_[0];

    beam_exit_[0]              = BeamOpCodeAddr(op_error_action_code);
    beam_exit = (ErtsCodePtr)&beam_exit_[0];

    beam_continue_exit_[0]     = BeamOpCodeAddr(op_continue_exit);
    beam_continue_exit = (ErtsCodePtr)&beam_continue_exit_[0];

    beam_return_to_trace_[0]   = BeamOpCodeAddr(op_i_return_to_trace);
    beam_return_to_trace = (ErtsCodePtr)&beam_return_to_trace_[0];

    beam_return_trace_[0]      = BeamOpCodeAddr(op_return_trace);
    beam_return_trace = (ErtsCodePtr)&beam_return_trace_[0];

    beam_exception_trace_[0]   = BeamOpCodeAddr(op_return_trace); /* UGLY */
    beam_exception_trace = (ErtsCodePtr)&beam_exception_trace_[0];

    beam_return_time_trace_[0] = BeamOpCodeAddr(op_i_return_time_trace);
    beam_return_time_trace = (ErtsCodePtr)&beam_return_time_trace_[0];

    install_bifs();
}

int
erts_beam_jump_table(void)
{
#if defined(NO_JUMP_TABLE)
    return 0;
#else
    return 1;
#endif
}