Merge pull request #2381 from jhogberg/john/erts/refactor-cif-tracing/OTP-14734

Refactor BIF tracing

1 files changed, 158 insertions, 266 deletions

diff --git a/erts/emulator/beam/beam_emu.c b/erts/emulator/beam/beam_emu.c
index 9f8b56a5d5..e8a0c666e3 100644
--- a/erts/emulator/beam/beam_emu.c
+++ b/erts/emulator/beam/beam_emu.c
@@ -111,10 +111,9 @@ do {                                     \
 
 #define CHECK_ALIGNED(Dst) ASSERT((((Uint)&Dst) & (sizeof(Uint)-1)) == 0)
 
-#define GET_BIF_MODULE(p)  (p->info.mfa.module)
-#define GET_BIF_FUNCTION(p)  (p->info.mfa.function)
-#define GET_BIF_ARITY(p)  (p->info.mfa.arity)
-#define GET_BIF_ADDRESS(p) ((BifFunction) (p->beam[1]))
+#define GET_EXPORT_MODULE(p)  ((p)->info.mfa.module)
+#define GET_EXPORT_FUNCTION(p)  ((p)->info.mfa.function)
+#define GET_EXPORT_ARITY(p)  ((p)->info.mfa.arity)
 #define TermWords(t) (((t) / (sizeof(BeamInstr)/sizeof(Eterm))) + !!((t) % (sizeof(BeamInstr)/sizeof(Eterm))))
 
 
@@ -250,72 +249,6 @@ void** beam_ops;
 #define Q(N) (N*sizeof(Eterm *))
 #define l(N) (freg[N].fd)
 
-/*
- * Check that we haven't used the reductions and jump to function pointed to by
- * the I register.  If we are out of reductions, do a context switch.
- */
-
-#define DispatchMacro()				\
-  do {						\
-     BeamInstr dis_next;                        \
-     dis_next = *I;                             \
-     CHECK_ARGS(I);				\
-     if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
-        FCALLS--;				\
-        Goto(dis_next);				\
-     } else {					\
-	goto context_switch;			\
-     }						\
- } while (0)                                    \
-
-#define DispatchMacroFun()			\
-  do {						\
-     BeamInstr dis_next;                        \
-     dis_next = *I;                             \
-     CHECK_ARGS(I);				\
-     if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
-        FCALLS--;				\
-        Goto(dis_next);				\
-     } else {					\
-	goto context_switch_fun;		\
-     }						\
- } while (0)
-
-#define DispatchMacrox()                                                \
-  do {                                                                  \
-     if (FCALLS > 0) {                                                  \
-        BeamInstr dis_next;                                             \
-        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);    \
-        dis_next = *I;                                                  \
-        FCALLS--;                                                       \
-        CHECK_ARGS(I);                                                  \
-        Goto(dis_next);                                                 \
-     } else if (ERTS_PROC_GET_SAVED_CALLS_BUF(c_p)                      \
-		&& FCALLS > neg_o_reds) {                               \
-        goto save_calls1;                                               \
-     } else {                                                           \
-        SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);    \
-        CHECK_ARGS(I);                                                  \
-	goto context_switch;                                            \
-     }                                                                  \
- } while (0)
-
-#ifdef DEBUG
-/*
- * To simplify breakpoint setting, put the code in one place only and jump to it.
- */
-#  define Dispatch() goto do_dispatch
-#  define Dispatchx() goto do_dispatchx
-#  define Dispatchfun() goto do_dispatchfun
-#else
-/*
- * Inline for speed.
- */
-#  define Dispatch() DispatchMacro()
-#  define Dispatchx() DispatchMacrox()
-#  define Dispatchfun() DispatchMacroFun()
-#endif
-
 #define Arg(N)       I[(N)+1]
 
 #define GetSource(raw, dst)			\
@@ -348,19 +281,6 @@ do {						\
     }						\
 } while(0)
 
-#define DispatchReturn                          \
-do {                                            \
-    if (FCALLS > 0 || FCALLS > neg_o_reds) {	\
-        FCALLS--;				\
-        Goto(*I);                               \
-    }                                           \
-    else {					\
-        c_p->current = NULL;                    \
-        c_p->arity = 1;                         \
-        goto context_switch3;			\
-    }						\
-} while (0)
-
 #ifdef DEBUG
 /* Better static type testing by the C compiler */
 #  define BEAM_IS_TUPLE(Src) is_tuple(Src)
@@ -768,27 +688,9 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 #endif
 
 #include "beam_hot.h"
-
-#ifdef DEBUG
-    /*
-     * Set a breakpoint here to get control just after a call instruction.
-     * I points to the first instruction in the called function.
-     *
-     * In gdb, use 'call dis(I-5, 1)' to show the name of the function.
-     */
- do_dispatch:
-     DispatchMacro();
-
- do_dispatchx:
-     DispatchMacrox();
-
- do_dispatchfun:
-     DispatchMacroFun();
-
-#endif
-
     /*
-     * Jumped to from the Dispatch() macro when the reductions are used up.
+     * The labels are jumped to from the $DISPATCH() macros when the reductions
+     * are used up.
      *
      * Since the I register points just beyond the FuncBegin instruction, we
      * can get the module, function, and arity for the function being
@@ -982,18 +884,42 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
   }
 #endif
     return;			/* Never executed */
+}
 
-  save_calls1:
-    {
-	BeamInstr dis_next;
+/*
+ * Enter all BIFs into the export table.
+ *
+ * Note that they will all call the error_handler until their modules have been
+ * loaded, which may prevent the system from booting if BIFs from non-preloaded
+ * modules are apply/3'd while loading code. Ordinary BIF calls will work fine
+ * however since they won't go through export entries.
+ */
+static void install_bifs(void) {
+    int i;
+
+    for (i = 0; i < BIF_SIZE; i++) {
+        BifEntry *entry;
+        Export *ep;
+        int j;
+
+        entry = &bif_table[i];
+
+        ep = erts_export_put(entry->module, entry->name, entry->arity);
 
-	save_calls(c_p, (Export *) Arg(0));
+        ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
+        ep->info.mfa.module = entry->module;
+        ep->info.mfa.function = entry->name;
+        ep->info.mfa.arity = entry->arity;
+        ep->bif_table_index = i;
 
-	SET_I(((Export *) Arg(0))->addressv[erts_active_code_ix()]);
+        memset(&ep->trampoline, 0, sizeof(ep->trampoline));
+        ep->trampoline.op = BeamOpCodeAddr(op_call_error_handler);
 
-	dis_next = *I;
-	FCALLS--;
-	Goto(dis_next);
+        for (j = 0; j < ERTS_NUM_CODE_IX; j++) {
+            ep->addressv[j] = ep->trampoline.raw;
+        }
+
+        bif_export[i] = ep;
     }
 }
 
@@ -1004,43 +930,30 @@ void process_main(Eterm * x_reg_array, FloatDef* f_reg_array)
 static void
 init_emulator_finish(void)
 {
-     int i;
-     Export* ep;
-
 #if defined(ARCH_64) && defined(CODE_MODEL_SMALL)
-     for (i = 0; i < NUMBER_OF_OPCODES; i++) {
-         BeamInstr instr = BeamOpCodeAddr(i);
-         if (instr >= (1ull << 32)) {
-             erts_exit(ERTS_ERROR_EXIT,
-                       "This run-time was supposed be compiled with all code below 2Gb,\n"
-                       "but the instruction '%s' is located at %016lx.\n",
-                       opc[i].name, instr);
-         }
-     }
+    int i;
+
+    for (i = 0; i < NUMBER_OF_OPCODES; i++) {
+        BeamInstr instr = BeamOpCodeAddr(i);
+        if (instr >= (1ull << 32)) {
+            erts_exit(ERTS_ERROR_EXIT,
+                      "This run-time was supposed be compiled with all code below 2Gb,\n"
+                      "but the instruction '%s' is located at %016lx.\n",
+                      opc[i].name, instr);
+        }
+    }
 #endif
 
-     beam_apply[0]             = BeamOpCodeAddr(op_i_apply);
-     beam_apply[1]             = BeamOpCodeAddr(op_normal_exit);
-     beam_exit[0]              = BeamOpCodeAddr(op_error_action_code);
-     beam_continue_exit[0]     = BeamOpCodeAddr(op_continue_exit);
-     beam_return_to_trace[0]   = BeamOpCodeAddr(op_i_return_to_trace);
-     beam_return_trace[0]      = BeamOpCodeAddr(op_return_trace);
-     beam_exception_trace[0]   = BeamOpCodeAddr(op_return_trace); /* UGLY */
-     beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace);
+    beam_apply[0]             = BeamOpCodeAddr(op_i_apply);
+    beam_apply[1]             = BeamOpCodeAddr(op_normal_exit);
+    beam_exit[0]              = BeamOpCodeAddr(op_error_action_code);
+    beam_continue_exit[0]     = BeamOpCodeAddr(op_continue_exit);
+    beam_return_to_trace[0]   = BeamOpCodeAddr(op_i_return_to_trace);
+    beam_return_trace[0]      = BeamOpCodeAddr(op_return_trace);
+    beam_exception_trace[0]   = BeamOpCodeAddr(op_return_trace); /* UGLY */
+    beam_return_time_trace[0] = BeamOpCodeAddr(op_i_return_time_trace);
 
-     /*
-      * Enter all BIFs into the export table.
-      */
-     for (i = 0; i < BIF_SIZE; i++) {
-	 ep = erts_export_put(bif_table[i].module,
-			      bif_table[i].name,
-			      bif_table[i].arity);
-	 bif_export[i] = ep;
-	 ep->beam[0] = BeamOpCodeAddr(op_apply_bif);
-	 ep->beam[1] = (BeamInstr) bif_table[i].f;
-	 /* XXX: set func info for bifs */
-	 ep->info.op = BeamOpCodeAddr(op_i_func_info_IaaI);
-     }
+    install_bifs();
 }
 
 /*
@@ -1253,7 +1166,7 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	 * I[2]: Pointer to erl_module_nif
 	 * I[3]: Function pointer to dirty NIF
 	 *
-	 * This layout is determined by the NifExport struct
+	 * This layout is determined by the ErtsNativeFunc struct
 	 */
 
 	ERTS_MSACC_SET_STATE_CACHED_M_X(ERTS_MSACC_STATE_NIF);
@@ -1267,11 +1180,11 @@ void erts_dirty_process_main(ErtsSchedulerData *esdp)
 	ERTS_UNREQ_PROC_MAIN_LOCK(c_p);
 
 	ASSERT(!ERTS_PROC_IS_EXITING(c_p));
-	if (BeamIsOpCode(*I, op_apply_bif)) {
+	if (BeamIsOpCode(*I, op_call_bif_W)) {
 	    exiting = erts_call_dirty_bif(esdp, c_p, I, reg);
 	}
 	else {
-	    ASSERT(BeamIsOpCode(*I, op_call_nif));
+	    ASSERT(BeamIsOpCode(*I, op_call_nif_WWW));
             exiting = erts_call_dirty_nif(esdp, c_p, I, reg);
 	}
 
@@ -1340,6 +1253,33 @@ Eterm error_atom[NUMBER_EXIT_CODES] = {
   am_badkey,		/* 19 */
 };
 
+/* Returns the return address at E[0] in printable form, skipping tracing in
+ * the same manner as gather_stacktrace.
+ *
+ * This is needed to generate correct stacktraces when throwing errors from
+ * instructions that return like an ordinary function, such as call_nif. */
+BeamInstr *erts_printable_return_address(Process* p, Eterm *E) {
+    Eterm *ptr = E;
+
+    ASSERT(is_CP(*ptr));
+
+    while (ptr < STACK_START(p)) {
+        BeamInstr *cp = cp_val(*ptr);
+
+        if (cp == beam_exception_trace || cp == beam_return_trace) {
+            ptr += 3;
+        } else if (cp == beam_return_time_trace) {
+            ptr += 2;
+        } else if (cp == beam_return_to_trace) {
+            ptr += 1;
+        } else {
+            return cp;
+        }
+    }
+
+    ERTS_ASSERT(!"No continuation pointer on stack");
+}
+
 /*
  * To fully understand the error handling, one must keep in mind that
  * when an exception is thrown, the search for a handler can jump back
@@ -1369,14 +1309,14 @@ handle_error(Process* c_p, BeamInstr* pc, Eterm* reg, ErtsCodeMFA *bif_mfa)
 
     ASSERT(c_p->freason != TRAP); /* Should have been handled earlier. */
 
-    if (c_p->freason & EXF_RESTORE_NIF)
-	erts_nif_export_restore_error(c_p, &pc, reg, &bif_mfa);
+    if (c_p->freason & EXF_RESTORE_NFUNC)
+	erts_nfunc_restore_error(c_p, &pc, reg, &bif_mfa);
 
 #ifdef DEBUG
     if (bif_mfa) {
-	/* Verify that bif_mfa does not point into our nif export */
-	NifExport *nep = ERTS_PROC_GET_NIF_TRAP_EXPORT(c_p);
-	ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(NifExport)));
+	/* Verify that bif_mfa does not point into our native function wrapper */
+	ErtsNativeFunc *nep = ERTS_PROC_GET_NFUNC_TRAP_WRAPPER(c_p);
+	ASSERT(!nep || !ErtsInArea(bif_mfa, (char *)nep, sizeof(ErtsNativeFunc)));
     }
 #endif
 
@@ -1633,19 +1573,17 @@ expand_error_value(Process* c_p, Uint freason, Eterm Value) {
 
 
 static void
-gather_stacktrace(Process* p, Eterm *ptr, struct StackTrace* s, int depth)
+gather_stacktrace(Process* p, struct StackTrace* s, int depth)
 {
     BeamInstr *prev;
-    BeamInstr i_return_trace;
-    BeamInstr i_return_to_trace;
+    Eterm *ptr;
 
     if (depth == 0) {
         return;
     }
 
-    prev = s->depth ? s->trace[s->depth-1] : s->pc;
-    i_return_trace = beam_return_trace[0];
-    i_return_to_trace = beam_return_to_trace[0];
+    prev = s->depth ? s->trace[s->depth - 1] : s->pc;
+    ptr = p->stop;
 
     /*
      * Traverse the stack backwards and add all unique continuation
@@ -1658,16 +1596,15 @@ gather_stacktrace(Process* p, Eterm *ptr, struct StackTrace* s, int depth)
 
     while (ptr < STACK_START(p) && depth > 0) {
         if (is_CP(*ptr)) {
-            if (*cp_val(*ptr) == i_return_trace) {
-                /* Skip stack frame variables */
-                do ++ptr; while (is_not_CP(*ptr));
-                /* Skip return_trace parameters */
+            BeamInstr *cp = cp_val(*ptr);
+
+            if (cp == beam_exception_trace || cp == beam_return_trace) {
+                ptr += 3;
+            } else if (cp == beam_return_time_trace) {
                 ptr += 2;
-            } else if (*cp_val(*ptr) == i_return_to_trace) {
-                /* Skip stack frame variables */
-                do ++ptr; while (is_not_CP(*ptr));
+            } else if (cp == beam_return_to_trace) {
+                ptr += 1;
             } else {
-                BeamInstr *cp = cp_val(*ptr);
                 if (cp != prev) {
                     /* Record non-duplicates only */
                     prev = cp;
@@ -1720,7 +1657,6 @@ static void
 save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
 		ErtsCodeMFA *bif_mfa, Eterm args) {
     struct StackTrace* s;
-    Eterm *stack_start;
     int sz;
     int depth = erts_backtrace_depth;    /* max depth (never negative) */
 
@@ -1739,33 +1675,6 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
     s->depth = 0;
 
     /*
-     * If we crash on an instruction that returns to a return/exception trace
-     * instruction, we must set the stacktrace 'pc' to the actual return
-     * address or we'll lose the top stackframe when gathering the stack
-     * trace.
-     */
-    stack_start = STACK_TOP(c_p);
-    if (stack_start < STACK_START(c_p) && is_CP(*stack_start)) {
-        BeamInstr *cp = cp_val(*stack_start);
-
-        if (cp == pc) {
-            if (pc == beam_exception_trace || pc == beam_return_trace) {
-                ASSERT(&stack_start[3] <= STACK_START(c_p));
-                /* Fake having failed on the first instruction in the function
-                 * pointed to by the tag. */
-                pc = cp_val(stack_start[1]);
-                stack_start += 3;
-            } else if (pc == beam_return_to_trace) {
-                ASSERT(&stack_start[2] <= STACK_START(c_p));
-                pc = cp_val(stack_start[1]);
-                /* Skip both the trace tag and the new 'pc' to avoid
-                 * duplicated entries. */
-                stack_start += 2;
-            }
-        }
-    }
-
-    /*
      * If the failure was in a BIF other than 'error/1', 'error/2',
      * 'exit/1' or 'throw/1', save BIF-MFA and save the argument
      * registers by consing up an arglist.
@@ -1827,13 +1736,13 @@ save_stacktrace(Process* c_p, BeamInstr* pc, Eterm* reg,
     }
 
     /* Save the actual stack trace */
-    gather_stacktrace(c_p, stack_start, s, depth);
+    gather_stacktrace(c_p, s, depth);
 }
 
 void
 erts_save_stacktrace(Process* p, struct StackTrace* s, int depth)
 {
-    gather_stacktrace(p, STACK_TOP(p), s, depth);
+    gather_stacktrace(p, s, depth);
 }
 
 /*
@@ -2092,83 +2001,66 @@ apply_bif_error_adjustment(Process *p, Export *ep,
 			   Eterm *reg, Uint arity,
 			   BeamInstr *I, Uint stack_offset)
 {
+    int apply_only;
+    Uint need;
+
+    need = stack_offset /* bytes */ / sizeof(Eterm);
+    apply_only = stack_offset == 0;
+
     /*
      * I is only set when the apply is a tail call, i.e.,
      * from the instructions i_apply_only, i_apply_last_P,
      * and apply_last_IP.
      */
-    if (I
-	&& BeamIsOpCode(ep->beam[0], op_apply_bif)
-        && (ep == bif_export[BIF_error_1]
-	    || ep == bif_export[BIF_error_2]
-	    || ep == bif_export[BIF_exit_1]
-	    || ep == bif_export[BIF_throw_1])) {
-	/*
-	 * We are about to tail apply one of the BIFs
-	 * erlang:error/1, erlang:error/2, erlang:exit/1,
-	 * or erlang:throw/1. Error handling of these BIFs is
-	 * special!
-         *
-	 * We need the topmost continuation pointer to point into the
-	 * calling function when handling the error after the BIF has
-	 * been applied. This in order to get the topmost stackframe
-	 * correct.
-	 *
-	 * Note that these BIFs will unconditionally cause an
-	 * exception to be raised. That is, our modifications of the
-	 * stack will be corrected by the error handling code.
-	 */
-	int apply_only = stack_offset == 0;
-	BeamInstr *cpp;
-        Eterm *E;
+    if (!(I && (ep == bif_export[BIF_error_1] ||
+                ep == bif_export[BIF_error_2] ||
+                ep == bif_export[BIF_exit_1] ||
+                ep == bif_export[BIF_throw_1]))) {
+        return;
+    }
 
-        E = p->stop;
+    /*
+     * We are about to tail apply one of the BIFs erlang:error/1,
+     * erlang:error/2, erlang:exit/1, or erlang:throw/1. Error handling of
+     * these BIFs is special!
+     *
+     * We need the topmost continuation pointer to point into the calling
+     * function when handling the error after the BIF has been applied. This in
+     * order to get the topmost stackframe correct.
+     *
+     * Note that these BIFs will unconditionally cause an exception to be
+     * raised. That is, our modifications of the stack will be corrected by the
+     * error handling code.
+     */
+    if (need == 0) {
+        need = 1; /* i_apply_only */
+    }
 
-        while (is_not_CP(*E)) {
-            E++;
-        }
-        cpp = cp_val(E[0]);
+    if (p->stop - p->htop < need) {
+        erts_garbage_collect(p, (int) need, reg, arity+1);
+    }
 
+    if (apply_only) {
         /*
-	 * If we find an exception/return-to trace continuation
-	 * pointer as the topmost continuation pointer, we do not
-	 * need to do anything since the information will already
-	 * be available for generation of the stacktrace.
+         * Called from the i_apply_only instruction.
+         *
+         * Push the continuation pointer for the current function to the stack.
          */
-
-	if (cpp != beam_exception_trace
-	    && cpp != beam_return_trace
-	    && cpp != beam_return_to_trace) {
-	    Uint need = stack_offset /* bytes */ / sizeof(Eterm);
-	    if (need == 0)
-		need = 1; /* i_apply_only */
-	    if (p->stop - p->htop < need)
-		erts_garbage_collect(p, (int) need, reg, arity+1);
-	    if (apply_only) {
-		/*
-		 * Called from the i_apply_only instruction.
-		 *
-                 * Push the continuation pointer for the current
-                 * function to the stack.
-		 */
-                p->stop -= need;
-                p->stop[0] = make_cp(I);
-	    } else {
-		/*
-		 * Called from an i_apply_last_* instruction.
-		 *
-                 * The calling instruction will deallocate a stack
-                 * frame of size 'stack_offset'.
-		 *
-                 * Push the continuation pointer for the current
-                 * function to the stack, and then add a dummy
-                 * stackframe for the i_apply_last* instruction
-                 * to discard.
-		 */
-                p->stop[0] = make_cp(I);
-                p->stop -= need;
-	    }
-	}
+        p->stop -= need;
+        p->stop[0] = make_cp(I);
+    } else {
+        /*
+         * Called from an i_apply_last_* instruction.
+         *
+         * The calling instruction will deallocate a stack frame of size
+         * 'stack_offset'.
+         *
+         * Push the continuation pointer for the current function to the stack,
+         * and then add a dummy stackframe for the i_apply_last* instruction
+         * to discard.
+         */
+        p->stop[0] = make_cp(I);
+        p->stop -= need;
     }
 }
 
@@ -3220,10 +3112,10 @@ erts_is_builtin(Eterm Mod, Eterm Name, int arity)
     e.info.mfa.arity = arity;
 
     if ((ep = export_get(&e)) == NULL) {
-	return 0;
+        return 0;
     }
-    return ep->addressv[erts_active_code_ix()] == ep->beam &&
-	BeamIsOpCode(ep->beam[0], op_apply_bif);
+
+    return ep->bif_table_index != -1;
 }